From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/plugins/Makefile.am | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/plugins/Makefile.am (limited to 'src/plugins/Makefile.am') diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am new file mode 100644 index 00000000..ffc4b3ab --- /dev/null +++ b/src/plugins/Makefile.am @@ -0,0 +1,60 @@ + +# Copyright (c) +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AUTOMAKE_OPTIONS = foreign subdir-objects + +AM_CFLAGS = -Wall -I${top_srcdir} -I${top_builddir} +AM_LDFLAGS = -module -shared -avoid-version +SUFFIXES = .api.h .api .api.json +API_FILES = +BUILT_SOURCES = +vppplugins_LTLIBRARIES = +vppapitestplugins_LTLIBRARIES = +noinst_HEADERS = + +vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins +vpppluginsdir = ${libdir}/vpp_plugins + +if ENABLE_FLOWPERPKT_PLUGIN +include flowperpkt.am +endif + +if ENABLE_ILA_PLUGIN +include ila.am +endif + +if ENABLE_SIXRD_PLUGIN +include sixrd.am +endif + +include ../suffix-rules.mk + +# Remove *.la files +install-data-hook: + @-(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) + @-(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) + +############################################################################### +# API +############################################################################### + +apidir = $(prefix)/share/vpp/api/plugins + +api_DATA = \ + $(patsubst %.api,%.api.json,$(API_FILES)) + +BUILT_SOURCES += \ + $(patsubst %.api,%.api.json,$(API_FILES)) \ + $(patsubst %.api,%.api.h,$(API_FILES)) + -- cgit 1.2.3-korg From cb034b9b374927c7552e36dcbc306d8456b2a0cb Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 28 Dec 2016 18:38:59 +0100 Subject: Move java,lua api and remaining plugins to src/ Change-Id: I1c3b87e886603678368428ae56a6bd3327cbc90d Signed-off-by: Damjan Marion --- .gitignore | 10 +- Makefile | 21 +- build-data/packages/plugins.mk | 34 - build-data/packages/vnet.mk | 47 - build-data/packages/vpp-api.mk | 9 - build-data/platforms.mk | 20 +- build-data/platforms/vpp.mk | 4 +- build-data/platforms/vpp_lite.mk | 2 +- build-root/deb/debian/.gitignore | 4 +- build-root/deb/debian/control | 16 +- build-root/packages/tools.mk | 2 +- build-root/rpm/vpp.spec | 70 +- build-root/scripts/find-api-core-contents | 2 +- build-root/scripts/find-dev-contents | 8 +- build-root/scripts/find-plugins-contents | 4 +- build-root/scripts/find-python-api-contents | 8 - build-root/scripts/find-vpp-api-java-contents | 8 + build-root/scripts/find-vpp-api-lua-contents | 6 + build-root/scripts/find-vpp-api-python-contents | 8 + doxygen/Makefile | 7 +- plugins/Makefile.am | 15 - plugins/acl-plugin/Makefile.am | 114 -- plugins/acl-plugin/acl/acl.api | 444 ----- plugins/acl-plugin/acl/acl.c | 1901 ------------------- plugins/acl-plugin/acl/acl.h | 148 -- plugins/acl-plugin/acl/acl_all_api_h.h | 321 ---- plugins/acl-plugin/acl/acl_msg_enum.h | 28 - plugins/acl-plugin/acl/acl_test.c | 1024 ---------- .../fd/vpp/jvpp/acl/test/AclExpectedDumpData.java | 135 -- .../jvpp/io/fd/vpp/jvpp/acl/test/AclTestData.java | 101 - .../io/fd/vpp/jvpp/acl/test/AclTestRequests.java | 141 -- .../io/fd/vpp/jvpp/acl/test/FutureApiTest.java | 68 - .../acl/jvpp/io/fd/vpp/jvpp/acl/test/Readme.txt | 1 - plugins/acl-plugin/acl/jvpp_acl.c | 124 -- plugins/acl-plugin/acl/jvpp_acl.h | 45 - plugins/acl-plugin/acl/l2sess.c | 243 --- plugins/acl-plugin/acl/l2sess.h | 150 -- plugins/acl-plugin/acl/l2sess_node.c | 816 -------- plugins/acl-plugin/acl/node_in.c | 168 -- plugins/acl-plugin/acl/node_in.h | 12 - plugins/acl-plugin/acl/node_out.c | 175 -- plugins/acl-plugin/acl/node_out.h | 12 - plugins/acl-plugin/configure.ac | 24 - plugins/acl-plugin/test/run-python | 28 - plugins/acl-plugin/test/run-scapy | 26 - plugins/acl-plugin/test/test_acl_plugin.py | 118 -- plugins/configure.ac | 5 - plugins/ioam-plugin/Makefile.am | 338 ---- plugins/ioam-plugin/configure.ac | 25 - plugins/ioam-plugin/ioam/dir.dox | 18 - plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.c | 232 --- plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.h | 47 - plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c | 276 --- plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.c | 109 -- plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.h | 70 - .../ioam/encap/ip6_ioam_seqno_analyse.c | 141 -- plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c | 438 ----- .../ioam-plugin/ioam/export-common/ioam_export.h | 616 ------ .../export-vxlan-gpe/vxlan_gpe_ioam_export.api | 42 - .../ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c | 271 --- .../vxlan_gpe_ioam_export_all_api_h.h | 16 - .../vxlan_gpe_ioam_export_msg_enum.h | 28 - .../export-vxlan-gpe/vxlan_gpe_ioam_export_test.c | 215 --- .../vxlan_gpe_ioam_export_thread.c | 49 - .../ioam/export-vxlan-gpe/vxlan_gpe_node.c | 162 -- plugins/ioam-plugin/ioam/export/ioam_export.api | 42 - plugins/ioam-plugin/ioam/export/ioam_export.c | 282 --- .../ioam/export/ioam_export_all_api_h.h | 16 - .../ioam-plugin/ioam/export/ioam_export_msg_enum.h | 28 - plugins/ioam-plugin/ioam/export/ioam_export_test.c | 206 --- .../ioam-plugin/ioam/export/ioam_export_thread.c | 38 - plugins/ioam-plugin/ioam/export/jvpp_ioam_export.c | 124 -- plugins/ioam-plugin/ioam/export/jvpp_ioam_export.h | 45 - plugins/ioam-plugin/ioam/export/node.c | 151 -- plugins/ioam-plugin/ioam/ioam_plugin_doc.md | 464 ----- .../jvpp/ioamexport/test/IoamExportApiTest.java | 56 - .../jvpp/io/fd/vpp/jvpp/ioamexport/test/Readme.txt | 1 - .../fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java | 75 - .../jvpp/io/fd/vpp/jvpp/ioampot/test/Readme.txt | 1 - .../vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java | 77 - .../jvpp/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt | 1 - plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.c | 124 -- plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.h | 45 - plugins/ioam-plugin/ioam/lib-pot/math64.h | 159 -- plugins/ioam-plugin/ioam/lib-pot/pot.api | 133 -- plugins/ioam-plugin/ioam/lib-pot/pot_all_api_h.h | 16 - plugins/ioam-plugin/ioam/lib-pot/pot_api.c | 292 --- plugins/ioam-plugin/ioam/lib-pot/pot_msg_enum.h | 28 - plugins/ioam-plugin/ioam/lib-pot/pot_test.c | 365 ---- plugins/ioam-plugin/ioam/lib-pot/pot_util.c | 445 ----- plugins/ioam-plugin/ioam/lib-pot/pot_util.h | 195 -- .../ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.c | 124 -- .../ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.h | 45 - plugins/ioam-plugin/ioam/lib-trace/trace.api | 92 - .../ioam-plugin/ioam/lib-trace/trace_all_api_h.h | 16 - plugins/ioam-plugin/ioam/lib-trace/trace_api.c | 252 --- .../ioam-plugin/ioam/lib-trace/trace_msg_enum.h | 28 - plugins/ioam-plugin/ioam/lib-trace/trace_test.c | 292 --- plugins/ioam-plugin/ioam/lib-trace/trace_util.c | 206 --- plugins/ioam-plugin/ioam/lib-trace/trace_util.h | 247 --- .../ioam-plugin/ioam/lib-vxlan-gpe/ioam_decap.c | 223 --- .../ioam-plugin/ioam/lib-vxlan-gpe/ioam_encap.c | 194 -- plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_pop.c | 353 ---- .../ioam-plugin/ioam/lib-vxlan-gpe/ioam_transit.c | 188 -- .../ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe.api | 181 -- .../ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h | 16 - .../ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_api.c | 378 ---- .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c | 773 -------- .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h | 183 -- .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h | 61 - .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c | 552 ------ .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h | 172 -- .../ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h | 28 - .../ioam/lib-vxlan-gpe/vxlan_gpe_test.c | 600 ------ plugins/lb-plugin/Makefile.am | 55 - plugins/lb-plugin/configure.ac | 9 - plugins/lb-plugin/lb/api.c | 228 --- plugins/lb-plugin/lb/cli.c | 250 --- plugins/lb-plugin/lb/lb.api | 71 - plugins/lb-plugin/lb/lb.c | 844 --------- plugins/lb-plugin/lb/lb.h | 333 ---- plugins/lb-plugin/lb/lb_test.c | 293 --- plugins/lb-plugin/lb/lbhash.h | 216 --- plugins/lb-plugin/lb/node.c | 419 ----- plugins/lb-plugin/lb/refcount.c | 41 - plugins/lb-plugin/lb/refcount.h | 67 - plugins/lb-plugin/lb/util.c | 72 - plugins/lb-plugin/lb/util.h | 40 - plugins/lb-plugin/lb_plugin_doc.md | 141 -- plugins/sample-plugin/Makefile.am | 56 - plugins/sample-plugin/configure.ac | 9 - plugins/sample-plugin/sample/node.c | 295 --- plugins/sample-plugin/sample/sample.api | 39 - plugins/sample-plugin/sample/sample.c | 255 --- plugins/sample-plugin/sample/sample.h | 40 - plugins/sample-plugin/sample/sample_all_api_h.h | 16 - plugins/sample-plugin/sample/sample_msg_enum.h | 28 - plugins/sample-plugin/sample/sample_test.c | 213 --- plugins/snat-plugin/Makefile.am | 113 -- plugins/snat-plugin/configure.ac | 32 - plugins/snat-plugin/snat/in2out.c | 1597 ---------------- .../io/fd/vpp/jvpp/snat/test/CallbackApiTest.java | 68 - .../snat/jvpp/io/fd/vpp/jvpp/snat/test/Readme.txt | 1 - plugins/snat-plugin/snat/jvpp_snat.c | 124 -- plugins/snat-plugin/snat/jvpp_snat.h | 45 - plugins/snat-plugin/snat/out2in.c | 1261 ------------- plugins/snat-plugin/snat/snat.api | 283 --- plugins/snat-plugin/snat/snat.c | 1957 -------------------- plugins/snat-plugin/snat/snat.h | 259 --- plugins/snat-plugin/snat/snat_all_api_h.h | 19 - plugins/snat-plugin/snat/snat_msg_enum.h | 31 - plugins/snat-plugin/snat/snat_test.c | 602 ------ src/Makefile.am | 10 +- src/configure.ac | 42 +- src/examples/sample-plugin/Makefile.am | 56 + src/examples/sample-plugin/configure.ac | 9 + src/examples/sample-plugin/sample/node.c | 295 +++ src/examples/sample-plugin/sample/sample.api | 39 + src/examples/sample-plugin/sample/sample.c | 255 +++ src/examples/sample-plugin/sample/sample.h | 40 + .../sample-plugin/sample/sample_all_api_h.h | 16 + .../sample-plugin/sample/sample_msg_enum.h | 28 + src/examples/sample-plugin/sample/sample_test.c | 213 +++ src/m4/ax_vpp_find_jdk8.m4 | 29 + src/plugins/Makefile.am | 18 + src/plugins/acl.am | 35 + src/plugins/acl/acl.api | 444 +++++ src/plugins/acl/acl.c | 1901 +++++++++++++++++++ src/plugins/acl/acl.h | 148 ++ src/plugins/acl/acl_all_api_h.h | 321 ++++ src/plugins/acl/acl_msg_enum.h | 28 + src/plugins/acl/acl_test.c | 1024 ++++++++++ src/plugins/acl/l2sess.c | 243 +++ src/plugins/acl/l2sess.h | 150 ++ src/plugins/acl/l2sess_node.c | 816 ++++++++ src/plugins/acl/node_in.c | 168 ++ src/plugins/acl/node_in.h | 12 + src/plugins/acl/node_out.c | 175 ++ src/plugins/acl/node_out.h | 12 + src/plugins/acl/test/run-python | 28 + src/plugins/acl/test/run-scapy | 26 + src/plugins/acl/test/test_acl_plugin.py | 118 ++ src/plugins/ioam.am | 150 ++ src/plugins/ioam/dir.dox | 18 + src/plugins/ioam/encap/ip6_ioam_e2e.c | 232 +++ src/plugins/ioam/encap/ip6_ioam_e2e.h | 47 + src/plugins/ioam/encap/ip6_ioam_pot.c | 276 +++ src/plugins/ioam/encap/ip6_ioam_seqno.c | 109 ++ src/plugins/ioam/encap/ip6_ioam_seqno.h | 70 + src/plugins/ioam/encap/ip6_ioam_seqno_analyse.c | 141 ++ src/plugins/ioam/encap/ip6_ioam_trace.c | 438 +++++ src/plugins/ioam/export-common/ioam_export.h | 616 ++++++ .../export-vxlan-gpe/vxlan_gpe_ioam_export.api | 42 + .../ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c | 271 +++ .../vxlan_gpe_ioam_export_all_api_h.h | 16 + .../vxlan_gpe_ioam_export_msg_enum.h | 28 + .../export-vxlan-gpe/vxlan_gpe_ioam_export_test.c | 215 +++ .../vxlan_gpe_ioam_export_thread.c | 49 + src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_node.c | 162 ++ src/plugins/ioam/export/ioam_export.api | 42 + src/plugins/ioam/export/ioam_export.c | 282 +++ src/plugins/ioam/export/ioam_export_all_api_h.h | 16 + src/plugins/ioam/export/ioam_export_msg_enum.h | 28 + src/plugins/ioam/export/ioam_export_test.c | 206 +++ src/plugins/ioam/export/ioam_export_thread.c | 38 + src/plugins/ioam/export/node.c | 151 ++ src/plugins/ioam/ioam_plugin_doc.md | 464 +++++ src/plugins/ioam/lib-pot/math64.h | 159 ++ src/plugins/ioam/lib-pot/pot.api | 133 ++ src/plugins/ioam/lib-pot/pot_all_api_h.h | 16 + src/plugins/ioam/lib-pot/pot_api.c | 292 +++ src/plugins/ioam/lib-pot/pot_msg_enum.h | 28 + src/plugins/ioam/lib-pot/pot_test.c | 365 ++++ src/plugins/ioam/lib-pot/pot_util.c | 445 +++++ src/plugins/ioam/lib-pot/pot_util.h | 195 ++ src/plugins/ioam/lib-trace/trace.api | 92 + src/plugins/ioam/lib-trace/trace_all_api_h.h | 16 + src/plugins/ioam/lib-trace/trace_api.c | 252 +++ src/plugins/ioam/lib-trace/trace_msg_enum.h | 28 + src/plugins/ioam/lib-trace/trace_test.c | 292 +++ src/plugins/ioam/lib-trace/trace_util.c | 206 +++ src/plugins/ioam/lib-trace/trace_util.h | 247 +++ src/plugins/ioam/lib-vxlan-gpe/ioam_decap.c | 223 +++ src/plugins/ioam/lib-vxlan-gpe/ioam_encap.c | 194 ++ src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c | 353 ++++ src/plugins/ioam/lib-vxlan-gpe/ioam_transit.c | 188 ++ src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api | 181 ++ .../ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h | 16 + src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_api.c | 378 ++++ src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c | 773 ++++++++ src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h | 183 ++ .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h | 61 + .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c | 552 ++++++ .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h | 172 ++ .../ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h | 28 + src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_test.c | 600 ++++++ src/plugins/lb.am | 42 + src/plugins/lb/api.c | 228 +++ src/plugins/lb/cli.c | 250 +++ src/plugins/lb/lb.api | 71 + src/plugins/lb/lb.c | 844 +++++++++ src/plugins/lb/lb.h | 333 ++++ src/plugins/lb/lb_plugin_doc.md | 141 ++ src/plugins/lb/lb_test.c | 293 +++ src/plugins/lb/lbhash.h | 216 +++ src/plugins/lb/node.c | 419 +++++ src/plugins/lb/refcount.c | 41 + src/plugins/lb/refcount.h | 67 + src/plugins/lb/util.c | 72 + src/plugins/lb/util.h | 40 + src/plugins/snat.am | 33 + src/plugins/snat/in2out.c | 1597 ++++++++++++++++ src/plugins/snat/out2in.c | 1261 +++++++++++++ src/plugins/snat/snat.api | 283 +++ src/plugins/snat/snat.c | 1957 ++++++++++++++++++++ src/plugins/snat/snat.h | 259 +++ src/plugins/snat/snat_all_api_h.h | 19 + src/plugins/snat/snat_msg_enum.h | 31 + src/plugins/snat/snat_test.c | 602 ++++++ src/suffix-rules.mk | 6 +- src/tools/g2/configure.ac | 12 - src/tools/perftool/configure.ac | 12 - src/tools/vppapigen/configure.ac | 14 - src/vpp-api/java/Makefile.am | 199 ++ src/vpp-api/java/Readme.txt | 236 +++ .../fd/vpp/jvpp/acl/test/AclExpectedDumpData.java | 135 ++ .../io/fd/vpp/jvpp/acl/test/AclTestData.java | 101 + .../io/fd/vpp/jvpp/acl/test/AclTestRequests.java | 141 ++ .../io/fd/vpp/jvpp/acl/test/FutureApiTest.java | 68 + .../jvpp-acl/io/fd/vpp/jvpp/acl/test/Readme.txt | 1 + src/vpp-api/java/jvpp-acl/jvpp_acl.c | 124 ++ src/vpp-api/java/jvpp-acl/jvpp_acl.h | 45 + src/vpp-api/java/jvpp-common/jvpp_common.c | 65 + src/vpp-api/java/jvpp-common/jvpp_common.h | 67 + .../io/fd/vpp/jvpp/core/test/CallbackApiTest.java | 96 + .../test/CallbackJVppFacadeNotificationTest.java | 87 + .../vpp/jvpp/core/test/CallbackJVppFacadeTest.java | 103 ++ .../core/test/CallbackNotificationApiTest.java | 94 + .../io/fd/vpp/jvpp/core/test/ControlPingTest.java | 68 + .../vpp/jvpp/core/test/CreateSubInterfaceTest.java | 120 ++ .../jvpp/core/test/FutureApiNotificationTest.java | 55 + .../io/fd/vpp/jvpp/core/test/FutureApiTest.java | 123 ++ .../io/fd/vpp/jvpp/core/test/L2AclTest.java | 195 ++ .../fd/vpp/jvpp/core/test/LispAdjacencyTest.java | 124 ++ .../fd/vpp/jvpp/core/test/NotificationUtils.java | 53 + .../jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt | 17 + src/vpp-api/java/jvpp-core/jvpp_core.c | 117 ++ .../jvpp/ioamexport/test/IoamExportApiTest.java | 56 + .../io/fd/vpp/jvpp/ioamexport/test/Readme.txt | 1 + .../java/jvpp-ioamexport/jvpp_ioam_export.c | 124 ++ .../java/jvpp-ioamexport/jvpp_ioam_export.h | 45 + .../fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java | 75 + .../io/fd/vpp/jvpp/ioampot/test/Readme.txt | 1 + src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.c | 124 ++ src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.h | 45 + .../vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java | 77 + .../io/fd/vpp/jvpp/ioamtrace/test/Readme.txt | 1 + src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.c | 124 ++ src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.h | 45 + .../java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java | 56 + .../jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java | 76 + .../io/fd/vpp/jvpp/JVppRegistryImpl.java | 147 ++ .../io/fd/vpp/jvpp/NativeLibraryLoader.java | 73 + .../io/fd/vpp/jvpp/VppBaseCallException.java | 60 + .../io/fd/vpp/jvpp/VppCallbackException.java | 47 + .../io/fd/vpp/jvpp/VppConnection.java | 45 + .../io/fd/vpp/jvpp/VppInvocationException.java | 33 + .../io/fd/vpp/jvpp/VppJNIConnection.java | 137 ++ .../fd/vpp/jvpp/callback/ControlPingCallback.java | 29 + .../io/fd/vpp/jvpp/callback/JVppCallback.java | 29 + .../jvpp/callback/JVppNotificationCallback.java | 24 + .../io/fd/vpp/jvpp/dto/ControlPing.java | 34 + .../io/fd/vpp/jvpp/dto/ControlPingReply.java | 58 + .../jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java | 24 + .../io/fd/vpp/jvpp/dto/JVppNotification.java | 23 + .../io/fd/vpp/jvpp/dto/JVppReply.java | 24 + .../io/fd/vpp/jvpp/dto/JVppReplyDump.java | 25 + .../io/fd/vpp/jvpp/dto/JVppRequest.java | 34 + .../vpp/jvpp/future/AbstractFutureJVppInvoker.java | 141 ++ .../io/fd/vpp/jvpp/future/FutureJVppInvoker.java | 49 + .../jvpp/notification/NotificationRegistry.java | 25 + .../notification/NotificationRegistryProvider.java | 28 + .../io/fd/vpp/jvpp/test/ConnectionTest.java | 44 + src/vpp-api/java/jvpp-registry/jvpp_registry.c | 352 ++++ .../io/fd/vpp/jvpp/snat/test/CallbackApiTest.java | 68 + .../jvpp-snat/io/fd/vpp/jvpp/snat/test/Readme.txt | 1 + src/vpp-api/java/jvpp-snat/jvpp_snat.c | 124 ++ src/vpp-api/java/jvpp-snat/jvpp_snat.h | 45 + src/vpp-api/java/jvpp/gen/jvpp_gen.py | 185 ++ src/vpp-api/java/jvpp/gen/jvppgen/__init__.py | 0 src/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py | 105 ++ src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py | 308 +++ src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py | 295 +++ src/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py | 343 ++++ .../jvpp/gen/jvppgen/jvpp_callback_facade_gen.py | 324 ++++ .../jvpp/gen/jvppgen/jvpp_future_facade_gen.py | 331 ++++ src/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py | 219 +++ .../java/jvpp/gen/jvppgen/notification_gen.py | 199 ++ src/vpp-api/java/jvpp/gen/jvppgen/types_gen.py | 227 +++ src/vpp-api/java/jvpp/gen/jvppgen/util.py | 220 +++ src/vpp-api/lua/README.md | 50 + src/vpp-api/lua/bench.lua | 70 + src/vpp-api/lua/examples/cli/README.md | 5 + src/vpp-api/lua/examples/cli/lua-cli.lua | 747 ++++++++ src/vpp-api/lua/examples/example-acl-plugin.lua | 110 ++ src/vpp-api/lua/examples/example-classifier.lua | 51 + src/vpp-api/lua/examples/example-cli.lua | 44 + src/vpp-api/lua/examples/lute/README.md | 66 + src/vpp-api/lua/examples/lute/lute.lua | 777 ++++++++ .../lua/examples/lute/script-inout-acl-noacl.lute | 329 ++++ .../lua/examples/lute/script-inout-acl-old.lute | 329 ++++ .../lua/examples/lute/script-inout-acl.lute | 329 ++++ src/vpp-api/lua/examples/lute/script.lute | 7 + src/vpp-api/lua/examples/lute/sessions-acl.lute | 308 +++ src/vpp-api/lua/vpp-lapi.lua | 989 ++++++++++ src/vpp.am | 2 +- src/vppapigen.am | 6 +- vpp-api/Makefile.am | 4 - vpp-api/configure.ac | 12 - vpp-api/java/Makefile.am | 116 -- vpp-api/java/Readme.txt | 236 --- vpp-api/java/configure.ac | 24 - vpp-api/java/jvpp-common/jvpp_common.c | 65 - vpp-api/java/jvpp-common/jvpp_common.h | 67 - .../io/fd/vpp/jvpp/core/test/CallbackApiTest.java | 96 - .../test/CallbackJVppFacadeNotificationTest.java | 87 - .../vpp/jvpp/core/test/CallbackJVppFacadeTest.java | 103 -- .../core/test/CallbackNotificationApiTest.java | 94 - .../io/fd/vpp/jvpp/core/test/ControlPingTest.java | 68 - .../vpp/jvpp/core/test/CreateSubInterfaceTest.java | 120 -- .../jvpp/core/test/FutureApiNotificationTest.java | 55 - .../io/fd/vpp/jvpp/core/test/FutureApiTest.java | 123 -- .../io/fd/vpp/jvpp/core/test/L2AclTest.java | 195 -- .../fd/vpp/jvpp/core/test/LispAdjacencyTest.java | 124 -- .../fd/vpp/jvpp/core/test/NotificationUtils.java | 53 - .../jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt | 17 - vpp-api/java/jvpp-core/jvpp_core.c | 117 -- .../java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java | 56 - .../jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java | 76 - .../io/fd/vpp/jvpp/JVppRegistryImpl.java | 147 -- .../io/fd/vpp/jvpp/NativeLibraryLoader.java | 73 - .../io/fd/vpp/jvpp/VppBaseCallException.java | 60 - .../io/fd/vpp/jvpp/VppCallbackException.java | 47 - .../io/fd/vpp/jvpp/VppConnection.java | 45 - .../io/fd/vpp/jvpp/VppInvocationException.java | 33 - .../io/fd/vpp/jvpp/VppJNIConnection.java | 137 -- .../fd/vpp/jvpp/callback/ControlPingCallback.java | 29 - .../io/fd/vpp/jvpp/callback/JVppCallback.java | 29 - .../jvpp/callback/JVppNotificationCallback.java | 24 - .../io/fd/vpp/jvpp/dto/ControlPing.java | 34 - .../io/fd/vpp/jvpp/dto/ControlPingReply.java | 58 - .../jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java | 24 - .../io/fd/vpp/jvpp/dto/JVppNotification.java | 23 - .../io/fd/vpp/jvpp/dto/JVppReply.java | 24 - .../io/fd/vpp/jvpp/dto/JVppReplyDump.java | 25 - .../io/fd/vpp/jvpp/dto/JVppRequest.java | 34 - .../vpp/jvpp/future/AbstractFutureJVppInvoker.java | 141 -- .../io/fd/vpp/jvpp/future/FutureJVppInvoker.java | 49 - .../jvpp/notification/NotificationRegistry.java | 25 - .../notification/NotificationRegistryProvider.java | 28 - .../io/fd/vpp/jvpp/test/ConnectionTest.java | 44 - vpp-api/java/jvpp-registry/jvpp_registry.c | 352 ---- vpp-api/java/jvpp/gen/jvpp_gen.py | 171 -- vpp-api/java/jvpp/gen/jvppgen/__init__.py | 0 vpp-api/java/jvpp/gen/jvppgen/callback_gen.py | 105 -- vpp-api/java/jvpp/gen/jvppgen/dto_gen.py | 308 --- vpp-api/java/jvpp/gen/jvppgen/jni_gen.py | 295 --- vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py | 343 ---- .../jvpp/gen/jvppgen/jvpp_callback_facade_gen.py | 324 ---- .../jvpp/gen/jvppgen/jvpp_future_facade_gen.py | 331 ---- vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py | 219 --- vpp-api/java/jvpp/gen/jvppgen/notification_gen.py | 199 -- vpp-api/java/jvpp/gen/jvppgen/types_gen.py | 227 --- vpp-api/java/jvpp/gen/jvppgen/util.py | 220 --- vpp-api/java/m4/ax_check_java_home.m4 | 80 - vpp-api/java/m4/ax_check_java_plugin.m4 | 101 - vpp-api/java/m4/ax_java_check_class.m4 | 85 - vpp-api/java/m4/ax_java_options.m4 | 48 - vpp-api/java/m4/ax_libgcj_jar.m4 | 83 - vpp-api/java/m4/ax_prog_jar.m4 | 49 - vpp-api/java/m4/ax_prog_java.m4 | 115 -- vpp-api/java/m4/ax_prog_java_cc.m4 | 104 -- vpp-api/java/m4/ax_prog_java_works.m4 | 134 -- vpp-api/java/m4/ax_prog_javac.m4 | 79 - vpp-api/java/m4/ax_prog_javac_works.m4 | 72 - vpp-api/java/m4/ax_prog_javadoc.m4 | 50 - vpp-api/java/m4/ax_prog_javah.m4 | 64 - vpp-api/java/m4/ax_try_compile_java.m4 | 55 - vpp-api/java/m4/ax_try_run_java.m4 | 56 - vpp-api/lua/README.md | 50 - vpp-api/lua/bench.lua | 70 - vpp-api/lua/examples/cli/README.md | 5 - vpp-api/lua/examples/cli/lua-cli.lua | 747 -------- vpp-api/lua/examples/example-acl-plugin.lua | 110 -- vpp-api/lua/examples/example-classifier.lua | 51 - vpp-api/lua/examples/example-cli.lua | 44 - vpp-api/lua/examples/lute/README.md | 66 - vpp-api/lua/examples/lute/lute.lua | 777 -------- .../lua/examples/lute/script-inout-acl-noacl.lute | 329 ---- .../lua/examples/lute/script-inout-acl-old.lute | 329 ---- vpp-api/lua/examples/lute/script-inout-acl.lute | 329 ---- vpp-api/lua/examples/lute/script.lute | 7 - vpp-api/lua/examples/lute/sessions-acl.lute | 308 --- vpp-api/lua/vpp-lapi.lua | 989 ---------- 444 files changed, 38673 insertions(+), 40252 deletions(-) delete mode 100644 build-data/packages/plugins.mk delete mode 100644 build-data/packages/vnet.mk delete mode 100644 build-data/packages/vpp-api.mk delete mode 100755 build-root/scripts/find-python-api-contents create mode 100755 build-root/scripts/find-vpp-api-java-contents create mode 100755 build-root/scripts/find-vpp-api-lua-contents create mode 100755 build-root/scripts/find-vpp-api-python-contents delete mode 100644 plugins/acl-plugin/Makefile.am delete mode 100644 plugins/acl-plugin/acl/acl.api delete mode 100644 plugins/acl-plugin/acl/acl.c delete mode 100644 plugins/acl-plugin/acl/acl.h delete mode 100644 plugins/acl-plugin/acl/acl_all_api_h.h delete mode 100644 plugins/acl-plugin/acl/acl_msg_enum.h delete mode 100644 plugins/acl-plugin/acl/acl_test.c delete mode 100644 plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java delete mode 100644 plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestData.java delete mode 100644 plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestRequests.java delete mode 100644 plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/FutureApiTest.java delete mode 100644 plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/Readme.txt delete mode 100644 plugins/acl-plugin/acl/jvpp_acl.c delete mode 100644 plugins/acl-plugin/acl/jvpp_acl.h delete mode 100644 plugins/acl-plugin/acl/l2sess.c delete mode 100644 plugins/acl-plugin/acl/l2sess.h delete mode 100644 plugins/acl-plugin/acl/l2sess_node.c delete mode 100644 plugins/acl-plugin/acl/node_in.c delete mode 100644 plugins/acl-plugin/acl/node_in.h delete mode 100644 plugins/acl-plugin/acl/node_out.c delete mode 100644 plugins/acl-plugin/acl/node_out.h delete mode 100644 plugins/acl-plugin/configure.ac delete mode 100755 plugins/acl-plugin/test/run-python delete mode 100755 plugins/acl-plugin/test/run-scapy delete mode 100644 plugins/acl-plugin/test/test_acl_plugin.py delete mode 100644 plugins/ioam-plugin/Makefile.am delete mode 100644 plugins/ioam-plugin/configure.ac delete mode 100644 plugins/ioam-plugin/ioam/dir.dox delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.c delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.h delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.c delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.h delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno_analyse.c delete mode 100644 plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c delete mode 100644 plugins/ioam-plugin/ioam/export-common/ioam_export.h delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c delete mode 100644 plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_node.c delete mode 100644 plugins/ioam-plugin/ioam/export/ioam_export.api delete mode 100644 plugins/ioam-plugin/ioam/export/ioam_export.c delete mode 100644 plugins/ioam-plugin/ioam/export/ioam_export_all_api_h.h delete mode 100644 plugins/ioam-plugin/ioam/export/ioam_export_msg_enum.h delete mode 100644 plugins/ioam-plugin/ioam/export/ioam_export_test.c delete mode 100644 plugins/ioam-plugin/ioam/export/ioam_export_thread.c delete mode 100644 plugins/ioam-plugin/ioam/export/jvpp_ioam_export.c delete mode 100644 plugins/ioam-plugin/ioam/export/jvpp_ioam_export.h delete mode 100644 plugins/ioam-plugin/ioam/export/node.c delete mode 100644 plugins/ioam-plugin/ioam/ioam_plugin_doc.md delete mode 100644 plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java delete mode 100644 plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/Readme.txt delete mode 100644 plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java delete mode 100644 plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/Readme.txt delete mode 100644 plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java delete mode 100644 plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.c delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.h delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/math64.h delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot.api delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot_all_api_h.h delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot_api.c delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot_msg_enum.h delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot_test.c delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot_util.c delete mode 100644 plugins/ioam-plugin/ioam/lib-pot/pot_util.h delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.c delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.h delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace.api delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace_api.c delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace_test.c delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace_util.c delete mode 100644 plugins/ioam-plugin/ioam/lib-trace/trace_util.h delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_decap.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_encap.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_pop.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_transit.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe.api delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_api.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h delete mode 100644 plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_test.c delete mode 100644 plugins/lb-plugin/Makefile.am delete mode 100644 plugins/lb-plugin/configure.ac delete mode 100644 plugins/lb-plugin/lb/api.c delete mode 100644 plugins/lb-plugin/lb/cli.c delete mode 100644 plugins/lb-plugin/lb/lb.api delete mode 100644 plugins/lb-plugin/lb/lb.c delete mode 100644 plugins/lb-plugin/lb/lb.h delete mode 100644 plugins/lb-plugin/lb/lb_test.c delete mode 100644 plugins/lb-plugin/lb/lbhash.h delete mode 100644 plugins/lb-plugin/lb/node.c delete mode 100644 plugins/lb-plugin/lb/refcount.c delete mode 100644 plugins/lb-plugin/lb/refcount.h delete mode 100644 plugins/lb-plugin/lb/util.c delete mode 100644 plugins/lb-plugin/lb/util.h delete mode 100644 plugins/lb-plugin/lb_plugin_doc.md delete mode 100644 plugins/sample-plugin/Makefile.am delete mode 100644 plugins/sample-plugin/configure.ac delete mode 100644 plugins/sample-plugin/sample/node.c delete mode 100644 plugins/sample-plugin/sample/sample.api delete mode 100644 plugins/sample-plugin/sample/sample.c delete mode 100644 plugins/sample-plugin/sample/sample.h delete mode 100644 plugins/sample-plugin/sample/sample_all_api_h.h delete mode 100644 plugins/sample-plugin/sample/sample_msg_enum.h delete mode 100644 plugins/sample-plugin/sample/sample_test.c delete mode 100644 plugins/snat-plugin/Makefile.am delete mode 100644 plugins/snat-plugin/configure.ac delete mode 100644 plugins/snat-plugin/snat/in2out.c delete mode 100644 plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java delete mode 100644 plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/Readme.txt delete mode 100644 plugins/snat-plugin/snat/jvpp_snat.c delete mode 100644 plugins/snat-plugin/snat/jvpp_snat.h delete mode 100644 plugins/snat-plugin/snat/out2in.c delete mode 100644 plugins/snat-plugin/snat/snat.api delete mode 100644 plugins/snat-plugin/snat/snat.c delete mode 100644 plugins/snat-plugin/snat/snat.h delete mode 100644 plugins/snat-plugin/snat/snat_all_api_h.h delete mode 100644 plugins/snat-plugin/snat/snat_msg_enum.h delete mode 100644 plugins/snat-plugin/snat/snat_test.c create mode 100644 src/examples/sample-plugin/Makefile.am create mode 100644 src/examples/sample-plugin/configure.ac create mode 100644 src/examples/sample-plugin/sample/node.c create mode 100644 src/examples/sample-plugin/sample/sample.api create mode 100644 src/examples/sample-plugin/sample/sample.c create mode 100644 src/examples/sample-plugin/sample/sample.h create mode 100644 src/examples/sample-plugin/sample/sample_all_api_h.h create mode 100644 src/examples/sample-plugin/sample/sample_msg_enum.h create mode 100644 src/examples/sample-plugin/sample/sample_test.c create mode 100644 src/m4/ax_vpp_find_jdk8.m4 create mode 100644 src/plugins/acl.am create mode 100644 src/plugins/acl/acl.api create mode 100644 src/plugins/acl/acl.c create mode 100644 src/plugins/acl/acl.h create mode 100644 src/plugins/acl/acl_all_api_h.h create mode 100644 src/plugins/acl/acl_msg_enum.h create mode 100644 src/plugins/acl/acl_test.c create mode 100644 src/plugins/acl/l2sess.c create mode 100644 src/plugins/acl/l2sess.h create mode 100644 src/plugins/acl/l2sess_node.c create mode 100644 src/plugins/acl/node_in.c create mode 100644 src/plugins/acl/node_in.h create mode 100644 src/plugins/acl/node_out.c create mode 100644 src/plugins/acl/node_out.h create mode 100755 src/plugins/acl/test/run-python create mode 100755 src/plugins/acl/test/run-scapy create mode 100644 src/plugins/acl/test/test_acl_plugin.py create mode 100644 src/plugins/ioam.am create mode 100644 src/plugins/ioam/dir.dox create mode 100644 src/plugins/ioam/encap/ip6_ioam_e2e.c create mode 100644 src/plugins/ioam/encap/ip6_ioam_e2e.h create mode 100644 src/plugins/ioam/encap/ip6_ioam_pot.c create mode 100644 src/plugins/ioam/encap/ip6_ioam_seqno.c create mode 100644 src/plugins/ioam/encap/ip6_ioam_seqno.h create mode 100644 src/plugins/ioam/encap/ip6_ioam_seqno_analyse.c create mode 100644 src/plugins/ioam/encap/ip6_ioam_trace.c create mode 100644 src/plugins/ioam/export-common/ioam_export.h create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c create mode 100644 src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_node.c create mode 100644 src/plugins/ioam/export/ioam_export.api create mode 100644 src/plugins/ioam/export/ioam_export.c create mode 100644 src/plugins/ioam/export/ioam_export_all_api_h.h create mode 100644 src/plugins/ioam/export/ioam_export_msg_enum.h create mode 100644 src/plugins/ioam/export/ioam_export_test.c create mode 100644 src/plugins/ioam/export/ioam_export_thread.c create mode 100644 src/plugins/ioam/export/node.c create mode 100644 src/plugins/ioam/ioam_plugin_doc.md create mode 100644 src/plugins/ioam/lib-pot/math64.h create mode 100644 src/plugins/ioam/lib-pot/pot.api create mode 100644 src/plugins/ioam/lib-pot/pot_all_api_h.h create mode 100644 src/plugins/ioam/lib-pot/pot_api.c create mode 100644 src/plugins/ioam/lib-pot/pot_msg_enum.h create mode 100644 src/plugins/ioam/lib-pot/pot_test.c create mode 100644 src/plugins/ioam/lib-pot/pot_util.c create mode 100644 src/plugins/ioam/lib-pot/pot_util.h create mode 100644 src/plugins/ioam/lib-trace/trace.api create mode 100644 src/plugins/ioam/lib-trace/trace_all_api_h.h create mode 100644 src/plugins/ioam/lib-trace/trace_api.c create mode 100644 src/plugins/ioam/lib-trace/trace_msg_enum.h create mode 100644 src/plugins/ioam/lib-trace/trace_test.c create mode 100644 src/plugins/ioam/lib-trace/trace_util.c create mode 100644 src/plugins/ioam/lib-trace/trace_util.h create mode 100644 src/plugins/ioam/lib-vxlan-gpe/ioam_decap.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/ioam_encap.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/ioam_transit.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_api.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h create mode 100644 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_test.c create mode 100644 src/plugins/lb.am create mode 100644 src/plugins/lb/api.c create mode 100644 src/plugins/lb/cli.c create mode 100644 src/plugins/lb/lb.api create mode 100644 src/plugins/lb/lb.c create mode 100644 src/plugins/lb/lb.h create mode 100644 src/plugins/lb/lb_plugin_doc.md create mode 100644 src/plugins/lb/lb_test.c create mode 100644 src/plugins/lb/lbhash.h create mode 100644 src/plugins/lb/node.c create mode 100644 src/plugins/lb/refcount.c create mode 100644 src/plugins/lb/refcount.h create mode 100644 src/plugins/lb/util.c create mode 100644 src/plugins/lb/util.h create mode 100644 src/plugins/snat.am create mode 100644 src/plugins/snat/in2out.c create mode 100644 src/plugins/snat/out2in.c create mode 100644 src/plugins/snat/snat.api create mode 100644 src/plugins/snat/snat.c create mode 100644 src/plugins/snat/snat.h create mode 100644 src/plugins/snat/snat_all_api_h.h create mode 100644 src/plugins/snat/snat_msg_enum.h create mode 100644 src/plugins/snat/snat_test.c delete mode 100644 src/tools/g2/configure.ac delete mode 100644 src/tools/perftool/configure.ac delete mode 100644 src/tools/vppapigen/configure.ac create mode 100644 src/vpp-api/java/Makefile.am create mode 100644 src/vpp-api/java/Readme.txt create mode 100644 src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java create mode 100644 src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestData.java create mode 100644 src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestRequests.java create mode 100644 src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/FutureApiTest.java create mode 100644 src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/Readme.txt create mode 100644 src/vpp-api/java/jvpp-acl/jvpp_acl.c create mode 100644 src/vpp-api/java/jvpp-acl/jvpp_acl.h create mode 100644 src/vpp-api/java/jvpp-common/jvpp_common.c create mode 100644 src/vpp-api/java/jvpp-common/jvpp_common.h create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java create mode 100644 src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt create mode 100644 src/vpp-api/java/jvpp-core/jvpp_core.c create mode 100644 src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java create mode 100644 src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/Readme.txt create mode 100644 src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.c create mode 100644 src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.h create mode 100644 src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java create mode 100644 src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/Readme.txt create mode 100644 src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.c create mode 100644 src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.h create mode 100644 src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java create mode 100644 src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt create mode 100644 src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.c create mode 100644 src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.h create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java create mode 100644 src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java create mode 100644 src/vpp-api/java/jvpp-registry/jvpp_registry.c create mode 100644 src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java create mode 100644 src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/Readme.txt create mode 100644 src/vpp-api/java/jvpp-snat/jvpp_snat.c create mode 100644 src/vpp-api/java/jvpp-snat/jvpp_snat.h create mode 100755 src/vpp-api/java/jvpp/gen/jvpp_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/__init__.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/types_gen.py create mode 100644 src/vpp-api/java/jvpp/gen/jvppgen/util.py create mode 100644 src/vpp-api/lua/README.md create mode 100644 src/vpp-api/lua/bench.lua create mode 100644 src/vpp-api/lua/examples/cli/README.md create mode 100644 src/vpp-api/lua/examples/cli/lua-cli.lua create mode 100644 src/vpp-api/lua/examples/example-acl-plugin.lua create mode 100644 src/vpp-api/lua/examples/example-classifier.lua create mode 100644 src/vpp-api/lua/examples/example-cli.lua create mode 100644 src/vpp-api/lua/examples/lute/README.md create mode 100644 src/vpp-api/lua/examples/lute/lute.lua create mode 100644 src/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute create mode 100644 src/vpp-api/lua/examples/lute/script-inout-acl-old.lute create mode 100644 src/vpp-api/lua/examples/lute/script-inout-acl.lute create mode 100644 src/vpp-api/lua/examples/lute/script.lute create mode 100644 src/vpp-api/lua/examples/lute/sessions-acl.lute create mode 100644 src/vpp-api/lua/vpp-lapi.lua delete mode 100644 vpp-api/Makefile.am delete mode 100644 vpp-api/configure.ac delete mode 100644 vpp-api/java/Makefile.am delete mode 100644 vpp-api/java/Readme.txt delete mode 100644 vpp-api/java/configure.ac delete mode 100644 vpp-api/java/jvpp-common/jvpp_common.c delete mode 100644 vpp-api/java/jvpp-common/jvpp_common.h delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java delete mode 100644 vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt delete mode 100644 vpp-api/java/jvpp-core/jvpp_core.c delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java delete mode 100644 vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java delete mode 100644 vpp-api/java/jvpp-registry/jvpp_registry.c delete mode 100755 vpp-api/java/jvpp/gen/jvpp_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/__init__.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/callback_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/dto_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/jni_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/notification_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/types_gen.py delete mode 100644 vpp-api/java/jvpp/gen/jvppgen/util.py delete mode 100644 vpp-api/java/m4/ax_check_java_home.m4 delete mode 100644 vpp-api/java/m4/ax_check_java_plugin.m4 delete mode 100644 vpp-api/java/m4/ax_java_check_class.m4 delete mode 100644 vpp-api/java/m4/ax_java_options.m4 delete mode 100644 vpp-api/java/m4/ax_libgcj_jar.m4 delete mode 100644 vpp-api/java/m4/ax_prog_jar.m4 delete mode 100644 vpp-api/java/m4/ax_prog_java.m4 delete mode 100644 vpp-api/java/m4/ax_prog_java_cc.m4 delete mode 100644 vpp-api/java/m4/ax_prog_java_works.m4 delete mode 100644 vpp-api/java/m4/ax_prog_javac.m4 delete mode 100644 vpp-api/java/m4/ax_prog_javac_works.m4 delete mode 100644 vpp-api/java/m4/ax_prog_javadoc.m4 delete mode 100644 vpp-api/java/m4/ax_prog_javah.m4 delete mode 100644 vpp-api/java/m4/ax_try_compile_java.m4 delete mode 100644 vpp-api/java/m4/ax_try_run_java.m4 delete mode 100644 vpp-api/lua/README.md delete mode 100644 vpp-api/lua/bench.lua delete mode 100644 vpp-api/lua/examples/cli/README.md delete mode 100644 vpp-api/lua/examples/cli/lua-cli.lua delete mode 100644 vpp-api/lua/examples/example-acl-plugin.lua delete mode 100644 vpp-api/lua/examples/example-classifier.lua delete mode 100644 vpp-api/lua/examples/example-cli.lua delete mode 100644 vpp-api/lua/examples/lute/README.md delete mode 100644 vpp-api/lua/examples/lute/lute.lua delete mode 100644 vpp-api/lua/examples/lute/script-inout-acl-noacl.lute delete mode 100644 vpp-api/lua/examples/lute/script-inout-acl-old.lute delete mode 100644 vpp-api/lua/examples/lute/script-inout-acl.lute delete mode 100644 vpp-api/lua/examples/lute/script.lute delete mode 100644 vpp-api/lua/examples/lute/sessions-acl.lute delete mode 100644 vpp-api/lua/vpp-lapi.lua (limited to 'src/plugins/Makefile.am') diff --git a/.gitignore b/.gitignore index c12eedfb..61c407ff 100644 --- a/.gitignore +++ b/.gitignore @@ -80,11 +80,11 @@ GTAGS *.pyc # Python api generator -/vpp-api/python/build -/vpp-api/python/dist -/vpp-api/python/vpp_papi.egg-info -/vpp-api/python/vpp_papi/memclnt.py -/vpp-api/python/vpp_papi/vpe.py +/src/vpp-api/python/build +/src/vpp-api/python/dist +/src/vpp-api/python/vpp_papi.egg-info +/src/vpp-api/python/vpp_papi/memclnt.py +/src/vpp-api/python/vpp_papi/vpe.py # Build files in the test directory /test/*.ok diff --git a/Makefile b/Makefile index eeb9837a..9187404c 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ endif .PHONY: help bootstrap wipe wipe-release build build-release rebuild rebuild-release .PHONY: run run-release debug debug-release build-vat run-vat pkg-deb pkg-rpm -.PHONY: ctags cscope plugins plugins-release build-vpp-api +.PHONY: ctags cscope .PHONY: test test-debug retest retest-debug test-doc test-wipe-doc test-help test-wipe .PHONY: test-cov test-wipe-cov @@ -69,8 +69,6 @@ help: @echo " wipe-release - wipe all products of release build " @echo " build - build debug binaries" @echo " build-release - build release binaries" - @echo " plugins - build debug plugin binaries" - @echo " plugins-release - build release plugin binaries" @echo " rebuild - wipe and build debug binares" @echo " rebuild-release - wipe and build release binares" @echo " run - run debug binary" @@ -83,8 +81,6 @@ help: @echo " retest - run functional tests" @echo " retest-debug - run functional tests (debug build)" @echo " test-help - show help on test framework" - @echo " build-vat - build vpp-api-test tool" - @echo " build-vpp-api - build vpp-api" @echo " run-vat - run vpp-api-test tool" @echo " pkg-deb - build DEB packages" @echo " pkg-rpm - build RPM packages" @@ -206,19 +202,10 @@ wipe-release: $(BR)/.bootstrap.ok rebuild-release: wipe-release build-release -plugins: $(BR)/.bootstrap.ok - $(call make,$(PLATFORM)_debug,plugins-install) - -plugins-release: $(BR)/.bootstrap.ok - $(call make,$(PLATFORM),plugins-install) - -build-vpp-api: $(BR)/.bootstrap.ok - $(call make,$(PLATFORM)_debug,vpp-api-install) - VPP_PYTHON_PREFIX=$(BR)/python define test - $(if $(filter-out $(3),retest),make -C $(BR) PLATFORM=$(1) TAG=$(2) vpp-api-install plugins-install vpp-install,) + $(if $(filter-out $(3),retest),make -C $(BR) PLATFORM=$(1) TAG=$(2) vpp-install,) make -C test \ BR=$(BR) \ VPP_TEST_BUILD_DIR=$(BR)/build-$(2)-native \ @@ -265,12 +252,12 @@ define run @echo "WARNING: STARTUP_CONF not defined or file doesn't exist." @echo " Running with minimal startup config: $(MINIMAL_STARTUP_CONF)\n" @cd $(STARTUP_DIR) && \ - sudo $(2) $(1)/vpp/bin/vpp $(MINIMAL_STARTUP_CONF) plugin_path $(1)/plugins/lib64/vpp_plugins + sudo $(2) $(1)/vpp/bin/vpp $(MINIMAL_STARTUP_CONF) plugin_path $(1)/vpp/lib64/vpp_plugins endef else define run @cd $(STARTUP_DIR) && \ - sudo $(2) $(1)/vpp/bin/vpp $(shell cat $(STARTUP_CONF) | sed -e 's/#.*//') plugin_path $(1)/plugins/lib64/vpp_plugins + sudo $(2) $(1)/vpp/bin/vpp $(shell cat $(STARTUP_CONF) | sed -e 's/#.*//') plugin_path $(1)/vpp/lib64/vpp_plugins endef endif diff --git a/build-data/packages/plugins.mk b/build-data/packages/plugins.mk deleted file mode 100644 index b4d67a28..00000000 --- a/build-data/packages/plugins.mk +++ /dev/null @@ -1,34 +0,0 @@ -plugins_configure_depend = \ - vpp-api-install \ - vpp-install - -plugins_CPPFLAGS = $(call installed_includes_fn, \ - vpp \ - vpp-api) - -plugins_LDFLAGS = $(call installed_libs_fn, \ - vpp) - -ifeq ($($(PLATFORM)_enable_tests),yes) -plugins_configure_args += --enable-tests -endif - -# Platform dependent configure flags -plugins_configure_args += $(plugins_configure_args_$(PLATFORM)) - -# include & link with openssl only if needed -ifneq ($($(PLATFORM)_uses_openssl),no) -plugins_CPPFLAGS += $(call installed_includes_fn, openssl) -plugins_LDFLAGS += $(call installed_libs_fn, openssl) -endif - -ifneq ($($(PLATFORM)_uses_dpdk),no) -ifeq ($($(PLATFORM)_uses_external_dpdk),yes) -plugins_CPPFLAGS += -I$($(PLATFORM)_dpdk_inc_dir) -plugins_LDFLAGS += -L$($(PLATFORM)_dpdk_lib_dir) -else -plugins_configure_depend += dpdk-install -plugins_CPPFLAGS += $(call installed_includes_fn, dpdk) -plugins_LDFLAGS += $(call installed_libs_fn, dpdk) -endif -endif diff --git a/build-data/packages/vnet.mk b/build-data/packages/vnet.mk deleted file mode 100644 index 57c444e8..00000000 --- a/build-data/packages/vnet.mk +++ /dev/null @@ -1,47 +0,0 @@ -vnet_configure_depend = \ - vppinfra-install \ - svm-install \ - vlib-api-install \ - vlib-install - -vnet_CPPFLAGS = $(call installed_includes_fn, \ - vppinfra \ - svm \ - vlib \ - vlib-api) - -vnet_LDFLAGS = $(call installed_libs_fn, \ - vppinfra \ - svm \ - vlib \ - vlib-api) - -ifeq ($($(PLATFORM)_enable_tests),yes) -vnet_configure_args += --enable-tests -endif - -# Platform dependent configure flags -vnet_configure_args += $(vnet_configure_args_$(PLATFORM)) - -# include & link with openssl only if needed -ifneq ($($(PLATFORM)_uses_openssl),no) -vnet_CPPFLAGS += $(call installed_includes_fn, openssl) -vnet_LDFLAGS += $(call installed_libs_fn, openssl) -endif - -ifneq ($($(PLATFORM)_uses_dpdk),no) -ifeq ($($(PLATFORM)_uses_external_dpdk),yes) -vnet_CPPFLAGS += -I$($(PLATFORM)_dpdk_inc_dir) -vnet_LDFLAGS += -L$($(PLATFORM)_dpdk_lib_dir) -else -vnet_configure_depend += dpdk-install -vnet_CPPFLAGS += $(call installed_includes_fn, dpdk) -vnet_LDFLAGS += $(call installed_libs_fn, dpdk) -endif -ifeq ($($(PLATFORM)_uses_dpdk_cryptodev),yes) -vnet_configure_args += --with-dpdk-crypto -endif -ifeq ($($(PLATFORM)_uses_dpdk_mlx5_pmd),yes) -vnet_configure_args += --with-dpdk-mlx5-pmd -endif -endif diff --git a/build-data/packages/vpp-api.mk b/build-data/packages/vpp-api.mk deleted file mode 100644 index d9e8d72e..00000000 --- a/build-data/packages/vpp-api.mk +++ /dev/null @@ -1,9 +0,0 @@ -vpp-api_configure_depend = \ - vpp-install - -vpp-api_CPPFLAGS = $(call installed_includes_fn, \ - vpp) - -vpp-api_LDFLAGS = - -vpp-api_CPPFLAGS += -I/usr/lib/jvm/java-8-openjdk-amd64/include diff --git a/build-data/platforms.mk b/build-data/platforms.mk index 88dd3ed8..6f21b6dc 100644 --- a/build-data/platforms.mk +++ b/build-data/platforms.mk @@ -25,7 +25,7 @@ install-deb: $(patsubst %,%-find-source,$(ROOT_PACKAGES)) \ : generate file manifests ; \ find $(INSTALL_PREFIX)$(ARCH)/*/bin -type f -print \ - | sed -e 's:.*:../& /usr/bin:' \ + | sed -e 's:.*:../& /usr/bin:' | grep -v vppapigen \ > deb/debian/vpp.install ; \ \ : core api definitions ; \ @@ -51,9 +51,17 @@ install-deb: $(patsubst %,%-find-source,$(ROOT_PACKAGES)) ./scripts/find-plugins-contents $(INSTALL_PREFIX)$(ARCH) \ deb/debian/vpp-plugins.install ; \ \ - : python-api package ; \ - ./scripts/find-python-api-contents $(INSTALL_PREFIX)$(ARCH) \ - deb/debian/vpp-python-api.install ; \ + : vpp-api-lua package ; \ + ./scripts/find-vpp-api-lua-contents $(INSTALL_PREFIX)$(ARCH) \ + deb/debian/vpp-api-lua.install ; \ + \ + : vpp-api-java package ; \ + ./scripts/find-vpp-api-java-contents $(INSTALL_PREFIX)$(ARCH) \ + deb/debian/vpp-api-java.install ; \ + \ + : vpp-api-python package ; \ + ./scripts/find-vpp-api-python-contents $(INSTALL_PREFIX)$(ARCH) \ + deb/debian/vpp-api-python.install ; \ \ : dpdk headers ; \ ./scripts/find-dpdk-contents $(INSTALL_PREFIX)$(ARCH) \ @@ -70,9 +78,9 @@ install-deb: $(patsubst %,%-find-source,$(ROOT_PACKAGES)) : dev package needs a couple of additions ; \ echo ../build-tool-native/tools/vppapigen /usr/bin \ >> deb/debian/vpp-dev.install ; \ - echo ../../vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin \ + echo ../../src/vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin \ >> deb/debian/vpp-dev.install ; \ - for i in $$(ls ../vpp-api/java/jvpp/gen/jvppgen/*.py); do \ + for i in $$(ls ../src/vpp-api/java/jvpp/gen/jvppgen/*.py); do \ echo ../$${i} /usr/lib/python2.7/dist-packages/jvppgen \ >> deb/debian/vpp-dev.install; \ done; \ diff --git a/build-data/platforms/vpp.mk b/build-data/platforms/vpp.mk index 513a4db4..c9a214ac 100644 --- a/build-data/platforms/vpp.mk +++ b/build-data/platforms/vpp.mk @@ -29,15 +29,13 @@ vpp_uses_dpdk = yes # Uncoment to enable building unit tests # vpp_enable_tests = yes -vpp_root_packages = vpp vpp-api gmod plugins +vpp_root_packages = vpp gmod vpp_configure_args_vpp = --with-dpdk # Set these parameters carefully. The vlib_buffer_t is 128 bytes, i.e. vlib_configure_args_vpp = --with-pre-data=128 -plugins_configure_args_vpp = --with-dpdk - # DPDK configuration parameters # vpp_uses_dpdk_cryptodev = yes # vpp_uses_dpdk_mlx5_pmd = yes diff --git a/build-data/platforms/vpp_lite.mk b/build-data/platforms/vpp_lite.mk index 55805d10..a556b487 100644 --- a/build-data/platforms/vpp_lite.mk +++ b/build-data/platforms/vpp_lite.mk @@ -27,7 +27,7 @@ vpp_lite_uses_dpdk = no # Uncoment to enable building unit tests #vpp_lite_enable_tests = yes -vpp_lite_root_packages = vpp vpp-api gmod +vpp_lite_root_packages = vpp gmod vlib_configure_args_vpp_lite = --with-pre-data=128 diff --git a/build-root/deb/debian/.gitignore b/build-root/deb/debian/.gitignore index 75d8fbbc..c5e915a8 100644 --- a/build-root/deb/debian/.gitignore +++ b/build-root/deb/debian/.gitignore @@ -11,4 +11,6 @@ vpp-dpdk-dev/ vpp-dpdk-dkms/ vpp-dbg/ vppctl/ -vpp-python-api/ +vpp-api-lua/ +vpp-api-java/ +vpp-api-python/ diff --git a/build-root/deb/debian/control b/build-root/deb/debian/control index 643774e3..de48c903 100644 --- a/build-root/deb/debian/control +++ b/build-root/deb/debian/control @@ -60,7 +60,21 @@ Depends: ${shlibs:Depends}, ${misc:Depends} Description: DPDK 2.1 igb_uio_driver This package contains Linux kernel modules distributed with DPDK. -Package: vpp-python-api +Package: vpp-api-lua +Architecture: any +Depends: ${python:Depends}, ${misc:Depends}, vpp (= ${source:Version}) +Description: VPP LUA API bindings + This package contains VPP lua api bindings + . + +Package: vpp-api-java +Architecture: any +Depends: ${python:Depends}, ${misc:Depends}, vpp (= ${source:Version}) +Description: VPP Java API bindings + This package contains VPP java api bindings + . + +Package: vpp-api-python Architecture: any Depends: ${python:Depends}, ${misc:Depends}, vpp (= ${source:Version}) Description: VPP Python API bindings diff --git a/build-root/packages/tools.mk b/build-root/packages/tools.mk index 506e024b..d64a7959 100644 --- a/build-root/packages/tools.mk +++ b/build-root/packages/tools.mk @@ -1,3 +1,3 @@ tools_source = src -tools_configure_args = --disable-vlib +tools_configure_args = --disable-vlib --disable-svm --disable-japi diff --git a/build-root/rpm/vpp.spec b/build-root/rpm/vpp.spec index 654424c0..149ac51c 100644 --- a/build-root/rpm/vpp.spec +++ b/build-root/rpm/vpp.spec @@ -71,12 +71,28 @@ Requires: vpp = %{_version}-%{_release} %description plugins This package contains VPP plugins -%package python-api +%package api-lua +Summary: VPP api lua bindings +Group: Development/Libraries +Requires: vpp = %{_version}-%{_release}, vpp-lib = %{_version}-%{_release} + +%description api-lua +This package contains the lua bindings for the vpp api + +%package api-java +Summary: VPP api java bindings +Group: Development/Libraries +Requires: vpp = %{_version}-%{_release}, vpp-lib = %{_version}-%{_release} + +%description api-java +This package contains the java bindings for the vpp api + +%package api-python Summary: VPP api python bindings Group: Development/Libraries Requires: vpp = %{_version}-%{_release}, vpp-lib = %{_version}-%{_release}, python-setuptools -%description python-api +%description api-python This package contains the python bindings for the vpp api %prep @@ -131,6 +147,22 @@ do install -p -m 644 $file %{buildroot}/usr/share/vpp/api done +# Lua bindings +mkdir -p -m755 %{buildroot}/usr/share/doc/vpp/examples/lua/examples/cli +mkdir -p -m755 %{buildroot}/usr/share/doc/vpp/examples/lua/examples/lute +for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/../../src/vpp-api/lua && git ls-files .) +do + install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/../../src/vpp-api/lua/$file \ + %{buildroot}/usr/share/doc/vpp/examples/lua/$file +done + +# Java bindings +mkdir -p -m755 %{buildroot}/usr/share/java +for file in $(find %{_mu_build_dir}/%{_vpp_install_dir}/vpp/share/java -type f -name '*.jar' -print ) +do + install -p -m 644 $file %{buildroot}/usr/share/java +done + # Python bindings mkdir -p -m755 %{buildroot}%{python2_sitelib} install -p -m 666 %{_mu_build_dir}/%{_vpp_install_dir}/*/lib/python2.7/site-packages/vpp_papi-*.egg %{buildroot}%{python2_sitelib} @@ -151,16 +183,16 @@ do done mkdir -p -m755 %{buildroot}%{python2_sitelib}/jvppgen -install -p -m755 %{_mu_build_dir}/../vpp-api/java/jvpp/gen/jvpp_gen.py %{buildroot}/usr/bin -for i in $(ls %{_mu_build_dir}/../vpp-api/java/jvpp/gen/jvppgen/*.py); do +install -p -m755 %{_mu_build_dir}/../src/vpp-api/java/jvpp/gen/jvpp_gen.py %{buildroot}/usr/bin +for i in $(ls %{_mu_build_dir}/../src/vpp-api/java/jvpp/gen/jvppgen/*.py); do install -p -m666 ${i} %{buildroot}%{python2_sitelib}/jvppgen done; # sample plugin mkdir -p -m755 %{buildroot}/usr/share/doc/vpp/examples/sample-plugin/sample -for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/../../plugins/sample-plugin && git ls-files .) +for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/../../src/examples/sample-plugin && git ls-files .) do - install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/../../plugins/sample-plugin/$file \ + install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/../../src/examples/sample-plugin/$file \ %{buildroot}/usr/share/doc/vpp/examples/sample-plugin/$file done @@ -170,24 +202,12 @@ done # mkdir -p -m755 %{buildroot}/usr/lib/vpp_plugins mkdir -p -m755 %{buildroot}/usr/lib/vpp_api_test_plugins -for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/plugins/lib64/vpp_plugins && find -type f -print) -do - install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/plugins/lib64/vpp_plugins/$file \ - %{buildroot}/usr/lib/vpp_plugins/$file -done - for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/vpp/lib64/vpp_plugins && find -type f -print) do install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/vpp/lib64/vpp_plugins/$file \ %{buildroot}/usr/lib/vpp_plugins/$file done -for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/plugins/lib64/vpp_api_test_plugins && find -type f -print) -do - install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/plugins/lib64/vpp_api_test_plugins/$file \ - %{buildroot}/usr/lib/vpp_api_test_plugins/$file -done - for file in $(cd %{_mu_build_dir}/%{_vpp_install_dir}/vpp/lib64/vpp_api_test_plugins && find -type f -print) do install -p -m 644 %{_mu_build_dir}/%{_vpp_install_dir}/vpp/lib64/vpp_api_test_plugins/$file \ @@ -208,13 +228,13 @@ done sysctl --system %systemd_post vpp.service -%post python-api +%post api-python easy_install -z %{python2_sitelib}/vpp_papi-*.egg %preun %systemd_preun vpp.service -%preun python-api +%preun api-python easy_install -mxNq vpp_papi %postun @@ -255,7 +275,15 @@ fi %{_libdir}/* /usr/share/vpp/api/* -%files python-api +%files api-lua +%defattr(644,root,root) +/usr/share/doc/vpp/examples/lua + +%files api-java +%defattr(644,root,root) +/usr/share/java/* + +%files api-python %defattr(644,root,root) %{python2_sitelib}/vpp_papi-*.egg diff --git a/build-root/scripts/find-api-core-contents b/build-root/scripts/find-api-core-contents index f1f96f1f..c1af69db 100755 --- a/build-root/scripts/find-api-core-contents +++ b/build-root/scripts/find-api-core-contents @@ -1,6 +1,6 @@ #!/bin/bash -for i in $(find ${1}/vpp -name *.api.json -type f -print); do +for i in $(find ${1}/vpp/share/vpp/api/core -name *.api.json -type f -print); do echo ../${i} /usr/share/vpp/api/ >> ${2} done for i in $(find ${1}/vlib-api -name *.api.json -type f -print); do diff --git a/build-root/scripts/find-dev-contents b/build-root/scripts/find-dev-contents index 2dc6cc4d..d4f7b63f 100755 --- a/build-root/scripts/find-dev-contents +++ b/build-root/scripts/find-dev-contents @@ -16,16 +16,16 @@ do done # sample plugin -paths=`(cd ..; find plugins/sample-plugin -type f -print | grep -v autom4te)` +paths=`(cd ..; find src/examples/sample-plugin -type f -print | grep -v autom4te)` for path in $paths do - relpath=`echo $path | sed -e 's:.*plugins/::'` + relpath=`echo $path | sed -e 's:.*src/examples/::'` dir=`dirname $relpath` if [ $dir = "sample-plugin" ] ; then - echo ../../$path /usr/share/doc/vpp/examples/plugins/sample-plugin >> $2 + echo ../../$path /usr/share/doc/vpp/examples/sample-plugin/ >> $2 else echo ../../$path \ - /usr/share/doc/vpp/examples/plugins/$dir >> $2 + /usr/share/doc/vpp/examples/$dir >> $2 fi done diff --git a/build-root/scripts/find-plugins-contents b/build-root/scripts/find-plugins-contents index 4108f790..ae2a4271 100755 --- a/build-root/scripts/find-plugins-contents +++ b/build-root/scripts/find-plugins-contents @@ -2,11 +2,11 @@ rm -f $2 -for i in ${1}/{plugins,vpp}/lib64/vpp_plugins/*.so; do +for i in ${1}/vpp/lib64/vpp_plugins/*.so; do echo ../${i} /usr/lib/vpp_plugins >> ${2} done -for i in ${1}/{plugins,vpp}/lib64/vpp_api_test_plugins/*.so; do +for i in ${1}/vpp/lib64/vpp_api_test_plugins/*.so; do echo ../${i} /usr/lib/vpp_api_test_plugins >> ${2} done diff --git a/build-root/scripts/find-python-api-contents b/build-root/scripts/find-python-api-contents deleted file mode 100755 index 24e8532c..00000000 --- a/build-root/scripts/find-python-api-contents +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -rm -f $2 - -for i in $(find ${1}/{vpp,vpp-api}/lib/python2.7/site-packages/ -type f -print); do - echo ../${i} /usr/lib/python2.7/site-packages/vpp_papi >> ${2} -done - diff --git a/build-root/scripts/find-vpp-api-java-contents b/build-root/scripts/find-vpp-api-java-contents new file mode 100755 index 00000000..5f1bf197 --- /dev/null +++ b/build-root/scripts/find-vpp-api-java-contents @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -f $2 + +for i in $(find ${1}/vpp/share/java/ -type f -print); do + echo ../${i} /usr/share/java >> ${2} +done + diff --git a/build-root/scripts/find-vpp-api-lua-contents b/build-root/scripts/find-vpp-api-lua-contents new file mode 100755 index 00000000..f1173db8 --- /dev/null +++ b/build-root/scripts/find-vpp-api-lua-contents @@ -0,0 +1,6 @@ +#!/bin/bash + +#i for now put everything into examples directory + +echo ../../src/vpp-api/lua /usr/share/vpp/examples > ${2} + diff --git a/build-root/scripts/find-vpp-api-python-contents b/build-root/scripts/find-vpp-api-python-contents new file mode 100755 index 00000000..819c9122 --- /dev/null +++ b/build-root/scripts/find-vpp-api-python-contents @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -f $2 + +for i in $(find ${1}/vpp/lib/python2.7/site-packages/ -type f -print); do + echo ../${i} /usr/lib/python2.7/site-packages/vpp_papi >> ${2} +done + diff --git a/doxygen/Makefile b/doxygen/Makefile index ffce3c13..face5b44 100644 --- a/doxygen/Makefile +++ b/doxygen/Makefile @@ -52,8 +52,7 @@ DOXY_SRC_DIRECTORIES = \ $(DOXY_SRC)/vlibsocket \ $(DOXY_SRC)/vnet \ $(DOXY_SRC)/vpp \ - $(DOXY_SRC)/vpp-api \ - vpp-api + $(DOXY_SRC)/vpp-api # Input directories and files DOXY_INPUT ?= \ @@ -74,8 +73,8 @@ DOXY_INPUT := $(subst $(WS_ROOT)/,,$(DOXY_INPUT)) DOXY_EXCLUDE ?= \ $(DOXY_SRC)/vlib/vlib/buffer.c \ $(DOXY_SRC)/vlib/example \ - plugins/sample-plugin \ - vpp-api/lua + $(DOXY_SRC)/vpp-api/lua \ + plugins/sample-plugin # Generate a regexp for filenames to exclude DOXY_EXCLUDE_REGEXP = ($(subst .,\.,$(shell echo '$(strip $(DOXY_EXCLUDE))' | sed -e 's/ /|/g'))) diff --git a/plugins/Makefile.am b/plugins/Makefile.am index a101e47f..20676fc8 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -28,22 +28,7 @@ if ENABLE_sample_PLUGIN SUBDIRS += sample-plugin endif -if ENABLE_ioam_PLUGIN -SUBDIRS += ioam-plugin -endif - if ENABLE_vcgn_PLUGIN SUBDIRS += vcgn-plugin endif -if ENABLE_snat_PLUGIN -SUBDIRS += snat-plugin -endif - -if ENABLE_lb_PLUGIN -SUBDIRS += lb-plugin -endif - -if ENABLE_acl_PLUGIN -SUBDIRS += acl-plugin -endif diff --git a/plugins/acl-plugin/Makefile.am b/plugins/acl-plugin/Makefile.am deleted file mode 100644 index 798a32bb..00000000 --- a/plugins/acl-plugin/Makefile.am +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AUTOMAKE_OPTIONS = foreign subdir-objects - -AM_CFLAGS = -Wall -AM_LDFLAGS = -module -shared -avoid-version - -vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins -vpppluginsdir = ${libdir}/vpp_plugins - -vppapitestplugins_LTLIBRARIES = acl_test_plugin.la -vppplugins_LTLIBRARIES = acl_plugin.la - -acl_plugin_la_SOURCES = \ - acl/acl.c \ - acl/node_in.c \ - acl/node_out.c \ - acl/l2sess.c \ - acl/l2sess_node.c \ - acl/l2sess.h \ - acl/acl_plugin.api.h - -BUILT_SOURCES = acl/acl.api.h acl/acl.api.json - -SUFFIXES = .api.h .api .api.json - -# -# ACL API -# -%.api.h: %.api - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --output $@ --show-name $@ - -%.api.json: %.api - @echo " JSON APIGEN " $@ ; \ - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --json $@ - -apidir = $(prefix)/acl -api_DATA = acl/acl.api.json - -noinst_HEADERS = \ - acl/acl_all_api_h.h \ - acl/acl_msg_enum.h \ - acl/acl.api.h - -acl_test_plugin_la_SOURCES = acl/acl_test.c acl/acl_plugin.api.h - -# Remove *.la files -install-data-hook: - @(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) - @(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) - -# -# Java code generation -# -jvpp_registry_root = ../../vpp-api/java -jvpp_registry_version = 17.04 -jacl_jarfile = jvpp-acl-$(PACKAGE_VERSION).jar -jvpp_package_dir = io/fd/vpp/jvpp/acl -jvpp_root = acl/jvpp -jvpp_target_dir = target -jvpp_target = $(jvpp_root)/$(jvpp_target_dir) - -lib_LTLIBRARIES = libjvpp_acl.la -libjvpp_acl_la_SOURCES = acl/acl.api.h acl/jvpp_acl.c acl/jvpp/io_fd_vpp_jvpp_acl_JVppAclImpl.h -libjvpp_acl_la_LIBADD = -lvlibmemoryclient -lvlibapi -lvppinfra \ - -lpthread -lm -lrt -L$(jvpp_registry_root)/.libs -ljvpp_common -libjvpp_acl_la_LDFLAGS = -module -libjvpp_acl_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -I../ -I$(srcdir)/../ - -BUILT_SOURCES += $(jvpp_root)/io_fd_vpp_jvpp_acl_JVppAclImpl.h - -$(jvpp_root)/io_fd_vpp_jvpp_acl_JVppAclImpl.h: acl/acl.api.json - dir=`pwd`; \ - mkdir -p $(jvpp_target); \ - mkdir -p $(jvpp_root)/$(jvpp_package_dir); \ - cd $(jvpp_root)/$(jvpp_package_dir); \ - mkdir -p types dto future callfacade callback notification test; \ - @srcdir@/$(jvpp_registry_root)/jvpp/gen/jvpp_gen.py -i $${dir}/acl/acl.api.json --plugin_name acl; \ - cd -; \ - mv -f $(jvpp_root)/$(jvpp_package_dir)/jvpp_acl_gen.h $(jvpp_root)/jvpp_acl_gen.h; \ - cp $(srcdir)/$(jvpp_root)/$(jvpp_package_dir)/test/*.java $(jvpp_root)/$(jvpp_package_dir)/test/; \ - cd $(jvpp_root); \ - $(JAVAC) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d $(jvpp_target_dir) $(jvpp_package_dir)/*.java \ - $(jvpp_package_dir)/types/*.java \ - $(jvpp_package_dir)/dto/*.java \ - $(jvpp_package_dir)/callback/*.java \ - $(jvpp_package_dir)/notification/*.java \ - $(jvpp_package_dir)/future/*.java \ - $(jvpp_package_dir)/callfacade/*.java \ - $(jvpp_package_dir)/test/*.java \ - || (echo "acl jvpp compilation failed: $$?"; exit 1); \ - $(JAVAH) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d . io.fd.vpp.jvpp.acl.JVppAclImpl ; - -$(jacl_jarfile): libjvpp_acl.la - cp .libs/libjvpp_acl.so.0.0.0 $(jvpp_target); \ - cd $(jvpp_target); \ - $(JAR) cfv $(JARFLAGS) ../../../$@ libjvpp_acl.so.0.0.0 $(jvpp_package_dir)/* ; cd ..; - -all-local: $(jacl_jarfile) diff --git a/plugins/acl-plugin/acl/acl.api b/plugins/acl-plugin/acl/acl.api deleted file mode 100644 index 58a5a171..00000000 --- a/plugins/acl-plugin/acl/acl.api +++ /dev/null @@ -1,444 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** \file - This file defines the vpp control-plane API messages - used to control the ACL plugin -*/ - - -/** \brief Get the plugin version - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ - -define acl_plugin_get_version -{ - u32 client_index; - u32 context; -}; - -/** \brief Reply to get the plugin version - @param context - returned sender context, to match reply w/ request - @param major - Incremented every time a known breaking behavior change is introduced - @param minor - Incremented with small changes, may be used to avoid buggy versions -*/ - -define acl_plugin_get_version_reply -{ - u32 context; - u32 major; - u32 minor; -}; - -/** \brief Access List Rule entry - @param is_permit - deny (0), permit (1), or permit+reflect(2) action on this rule. - @param is_ipv6 - IP addresses in this rule are IPv6 (1) or IPv4 (0) - @param src_ip_addr - Source prefix value - @param src_ip_prefix_len - Source prefix length - @param dst_ip_addr - Destination prefix value - @param dst_ip_prefix_len - Destination prefix length - @param proto - L4 protocol (http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml) - @param srcport_or_icmptype_first - beginning of source port or ICMP4/6 type range - @param srcport_or_icmptype_last - end of source port or ICMP4/6 type range - @param dstport_or_icmpcode_first - beginning of destination port or ICMP4/6 code range - @param dstport_or_icmpcode_last - end of destination port or ICMP4/6 code range - @param tcp_flags_mask - if proto==6, match masked TCP flags with this value - @param tcp_flags_value - if proto==6, mask to AND the TCP flags in the packet with -*/ - -typeonly manual_print manual_endian define acl_rule -{ - u8 is_permit; - u8 is_ipv6; - u8 src_ip_addr[16]; - u8 src_ip_prefix_len; - u8 dst_ip_addr[16]; - u8 dst_ip_prefix_len; -/* - * L4 protocol. IANA number. 1 = ICMP, 58 = ICMPv6, 6 = TCP, 17 = UDP. - * 0 => ignore L4 and ignore the ports/tcpflags when matching. - */ - u8 proto; -/* - * If the L4 protocol is TCP or UDP, the below - * hold ranges of ports, else if the L4 is ICMP/ICMPv6 - * they hold ranges of ICMP(v6) types/codes. - * - * Ranges are inclusive, i.e. to match "any" TCP/UDP port, - * use first=0,last=65535. For ICMP(v6), - * use first=0,last=255. - */ - u16 srcport_or_icmptype_first; - u16 srcport_or_icmptype_last; - u16 dstport_or_icmpcode_first; - u16 dstport_or_icmpcode_last; -/* - * for proto = 6, this matches if the - * TCP flags in the packet, ANDed with tcp_flags_mask, - * is equal to tcp_flags_value. - */ - u8 tcp_flags_mask; - u8 tcp_flags_value; -}; - -/** \brief MACIP Access List Rule entry - @param is_permit - deny (0), permit (1) action on this rule. - @param is_ipv6 - IP addresses in this rule are IPv6 (1) or IPv4 (0) - @param src_mac - match masked source MAC address against this value - @param src_mac_mask - AND source MAC address with this value before matching - @param src_ip_addr - Source prefix value - @param src_ip_prefix_len - Source prefix length -*/ - -typeonly manual_print manual_endian define macip_acl_rule -{ - u8 is_permit; - u8 is_ipv6; -/* - * The source mac of the packet ANDed with src_mac_mask. - * The source ip[46] address in the packet is matched - * against src_ip_addr, with src_ip_prefix_len set to 0. - * - * For better performance, minimize the number of - * (src_mac_mask, src_ip_prefix_len) combinations - * in a MACIP ACL. - */ - u8 src_mac[6]; - u8 src_mac_mask[6]; - u8 src_ip_addr[16]; - u8 src_ip_prefix_len; -}; - -/** \brief Replace an existing ACL in-place or create a new ACL - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param acl_index - an existing ACL entry (0..0xfffffffe) to replace, or 0xffffffff to make new ACL - @param tag - a string value stored along with the ACL, for descriptive purposes - @param count - number of ACL rules - @r - Rules for this access-list -*/ - -manual_print manual_endian define acl_add_replace -{ - u32 client_index; - u32 context; - u32 acl_index; /* ~0 to add, existing ACL# to replace */ - u8 tag[64]; /* What gets in here gets out in the corresponding tag field when dumping the ACLs. */ - u32 count; - vl_api_acl_rule_t r[count]; -}; - -/** \brief Reply to add/replace ACL - @param context - returned sender context, to match reply w/ request - @param acl_index - index of the updated or newly created ACL - @param retval 0 - no error -*/ - -define acl_add_replace_reply -{ - u32 context; - u32 acl_index; - i32 retval; -}; - -/** \brief Delete an ACL - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param acl_index - ACL index to delete -*/ - -define acl_del -{ - u32 client_index; - u32 context; - u32 acl_index; -}; - -/** \brief Reply to delete the ACL - @param context - returned sender context, to match reply w/ request - @param retval 0 - no error -*/ - -define acl_del_reply -{ - u32 context; - i32 retval; -}; - -/* acl_interface_add_del(_reply) to be deprecated in lieu of acl_interface_set_acl_list */ -/** \brief Use acl_interface_set_acl_list instead - Append/remove an ACL index to/from the list of ACLs checked for an interface - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param is_add - add or delete the ACL index from the list - @param is_input - check the ACL on input (1) or output (0) - @param sw_if_index - the interface to alter the list of ACLs on - @param acl_index - index of ACL for the operation -*/ - -define acl_interface_add_del -{ - u32 client_index; - u32 context; - u8 is_add; -/* - * is_input = 0 => ACL applied on interface egress - * is_input = 1 => ACL applied on interface ingress - */ - u8 is_input; - u32 sw_if_index; - u32 acl_index; -}; - -/** \brief Reply to alter the ACL list - @param context - returned sender context, to match reply w/ request - @param retval 0 - no error -*/ - -define acl_interface_add_del_reply -{ - u32 context; - i32 retval; -}; - -/** \brief Set the vector of input/output ACLs checked for an interface - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param sw_if_index - the interface to alter the list of ACLs on - @param count - total number of ACL indices in the vector - @param n_input - this many first elements correspond to input ACLs, the rest - output - @param acls - vector of ACL indices -*/ - -manual_endian define acl_interface_set_acl_list -{ - u32 client_index; - u32 context; - u32 sw_if_index; - u8 count; - u8 n_input; /* First n_input ACLs are set as a list of input ACLs, the rest are applied as output */ - u32 acls[count]; -}; - -/** \brief Reply to set the ACL list on an interface - @param context - returned sender context, to match reply w/ request - @param retval 0 - no error -*/ - -define acl_interface_set_acl_list_reply -{ - u32 context; - i32 retval; -}; - -/** \brief Dump the specific ACL contents or all of the ACLs' contents - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param acl_index - ACL index to dump, ~0 to dump all ACLs -*/ - -define acl_dump -{ - u32 client_index; - u32 context; - u32 acl_index; /* ~0 for all ACLs */ -}; - -/** \brief Details about a single ACL contents - @param context - returned sender context, to match reply w/ request - @param acl_index - ACL index whose contents are being sent in this message - @param tag - Descriptive tag value which was supplied at ACL creation - @param count - Number of rules in this ACL - @param r - Array of rules within this ACL -*/ - -manual_print manual_endian define acl_details -{ - u32 context; - u32 acl_index; - u8 tag[64]; /* Same blob that was supplied to us when creating the ACL, one hopes. */ - u32 count; - vl_api_acl_rule_t r[count]; -}; - -/** \brief Dump the list(s) of ACL applied to specific or all interfaces - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param sw_if_index - interface to dump the ACL list for -*/ - -define acl_interface_list_dump -{ - u32 client_index; - u32 context; - u32 sw_if_index; /* ~0 for all interfaces */ -}; - -/** \brief Details about a single ACL contents - @param context - returned sender context, to match reply w/ request - @param sw_if_index - interface for which the list of ACLs is applied - @param count - total length of acl indices vector - @param n_input - this many of indices in the beginning are input ACLs, the rest - output - @param acls - the vector of ACL indices -*/ - -manual_endian define acl_interface_list_details -{ - u32 context; - u32 sw_if_index; - u8 count; - u8 n_input; - u32 acls[count]; -}; - -/** \brief Add a MACIP ACL - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param tag - descriptive value for this MACIP ACL - @param count - number of rules in this ACL - @param r - vector of MACIP ACL rules -*/ - -manual_print manual_endian define macip_acl_add -{ - u32 client_index; - u32 context; - u8 tag[64]; - u32 count; - vl_api_macip_acl_rule_t r[count]; -}; - -/** \brief Reply to add MACIP ACL - @param context - returned sender context, to match reply w/ request - @param acl_index - index of the newly created ACL - @param retval 0 - no error -*/ - -define macip_acl_add_reply -{ - u32 context; - u32 acl_index; - i32 retval; -}; - -/** \brief Delete a MACIP ACL - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param acl_index - MACIP ACL index to delete -*/ - -define macip_acl_del -{ - u32 client_index; - u32 context; - u32 acl_index; -}; - -/** \brief Reply to delete the MACIP ACL - @param context - returned sender context, to match reply w/ request - @param retval 0 - no error -*/ - -define macip_acl_del_reply -{ - u32 context; - i32 retval; -}; - -/** \brief Add or delete a MACIP ACL to/from interface - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param is_add - add (1) or delete (0) ACL from being used on an interface - @param sw_if_index - interface to apply the action to - @param acl_index - MACIP ACL index -*/ - -define macip_acl_interface_add_del -{ - u32 client_index; - u32 context; - u8 is_add; - /* macip ACLs are always input */ - u32 sw_if_index; - u32 acl_index; -}; - -/** \brief Reply to apply/unapply the MACIP ACL - @param context - returned sender context, to match reply w/ request - @param retval 0 - no error -*/ - -define macip_acl_interface_add_del_reply -{ - u32 context; - i32 retval; -}; - -/** \brief Dump one or all defined MACIP ACLs - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param acl_index - MACIP ACL index or ~0 to dump all ACLs -*/ - -define macip_acl_dump -{ - u32 client_index; - u32 context; - u32 acl_index; /* ~0 for all ACLs */ -}; - -/** \brief Details about one MACIP ACL - @param context - returned sender context, to match reply w/ request - @param acl_index - index of this MACIP ACL - @param tag - descriptive tag which was supplied during the creation - @param count - length of the vector of MACIP ACL rules - @param r - rules comprising this ACL -*/ - -manual_print manual_endian define macip_acl_details -{ - u32 context; - u32 acl_index; - u8 tag[64]; - u32 count; - vl_api_macip_acl_rule_t r[count]; -}; - -/** \brief Get the vector of MACIP ACL IDs applied to the interfaces - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ - -define macip_acl_interface_get -{ - u32 client_index; - u32 context; -}; - -/** \brief Reply with the vector of MACIP ACLs by sw_if_index - @param context - returned sender context, to match reply w/ request - @param count - total number of elements in the vector - @param acls - the vector of active MACACL indices per sw_if_index -*/ - -define macip_acl_interface_get_reply -{ - u32 context; - u32 count; - u32 acls[count]; -}; - diff --git a/plugins/acl-plugin/acl/acl.c b/plugins/acl-plugin/acl/acl.c deleted file mode 100644 index 8ff5a6b7..00000000 --- a/plugins/acl-plugin/acl/acl.c +++ /dev/null @@ -1,1901 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#include "node_in.h" -#include "node_out.h" - -acl_main_t acl_main; - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -#define REPLY_MACRO2(t, body) \ -do { \ - unix_shared_memory_queue_t * q; \ - rv = vl_msg_api_pd_handler (mp, rv); \ - q = vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+am->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - do {body;} while (0); \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -#define REPLY_MACRO3(t, n, body) \ -do { \ - unix_shared_memory_queue_t * q; \ - rv = vl_msg_api_pd_handler (mp, rv); \ - q = vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp) + n); \ - rmp->_vl_msg_id = ntohs((t)+am->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - do {body;} while (0); \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - - -/* List of message types that this plugin understands */ - -#define foreach_acl_plugin_api_msg \ -_(ACL_PLUGIN_GET_VERSION, acl_plugin_get_version) \ -_(ACL_ADD_REPLACE, acl_add_replace) \ -_(ACL_DEL, acl_del) \ -_(ACL_INTERFACE_ADD_DEL, acl_interface_add_del) \ -_(ACL_INTERFACE_SET_ACL_LIST, acl_interface_set_acl_list) \ -_(ACL_DUMP, acl_dump) \ -_(ACL_INTERFACE_LIST_DUMP, acl_interface_list_dump) \ -_(MACIP_ACL_ADD, macip_acl_add) \ -_(MACIP_ACL_DEL, macip_acl_del) \ -_(MACIP_ACL_INTERFACE_ADD_DEL, macip_acl_interface_add_del) \ -_(MACIP_ACL_DUMP, macip_acl_dump) \ -_(MACIP_ACL_INTERFACE_GET, macip_acl_interface_get) - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - acl_main_t *am = &acl_main; - clib_error_t *error = 0; - - am->vlib_main = vm; - am->vnet_main = h->vnet_main; - am->ethernet_main = h->ethernet_main; - - l2sess_vlib_plugin_register(vm, h, from_early_init); - - return error; -} - - -static void -vl_api_acl_plugin_get_version_t_handler (vl_api_acl_plugin_get_version_t * mp) -{ - acl_main_t *am = &acl_main; - vl_api_acl_plugin_get_version_reply_t *rmp; - int msg_size = sizeof (*rmp); - unix_shared_memory_queue_t *q; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - { - return; - } - - rmp = vl_msg_api_alloc (msg_size); - memset (rmp, 0, msg_size); - rmp->_vl_msg_id = - ntohs (VL_API_ACL_PLUGIN_GET_VERSION_REPLY + am->msg_id_base); - rmp->context = mp->context; - rmp->major = htonl (ACL_PLUGIN_VERSION_MAJOR); - rmp->minor = htonl (ACL_PLUGIN_VERSION_MINOR); - - vl_msg_api_send_shmem (q, (u8 *) & rmp); -} - - -static int -acl_add_list (u32 count, vl_api_acl_rule_t rules[], - u32 * acl_list_index, u8 * tag) -{ - acl_main_t *am = &acl_main; - acl_list_t *a; - acl_rule_t *r; - acl_rule_t *acl_new_rules; - int i; - - if (*acl_list_index != ~0) - { - /* They supplied some number, let's see if this ACL exists */ - if (pool_is_free_index (am->acls, *acl_list_index)) - { - /* tried to replace a non-existent ACL, no point doing anything */ - return -1; - } - } - - /* Create and populate the rules */ - acl_new_rules = clib_mem_alloc_aligned (sizeof (acl_rule_t) * count, - CLIB_CACHE_LINE_BYTES); - if (!acl_new_rules) - { - /* Could not allocate rules. New or existing ACL - bail out regardless */ - return -1; - } - - for (i = 0; i < count; i++) - { - r = &acl_new_rules[i]; - r->is_permit = rules[i].is_permit; - r->is_ipv6 = rules[i].is_ipv6; - if (r->is_ipv6) - { - memcpy (&r->src, rules[i].src_ip_addr, sizeof (r->src)); - memcpy (&r->dst, rules[i].dst_ip_addr, sizeof (r->dst)); - } - else - { - memcpy (&r->src.ip4, rules[i].src_ip_addr, sizeof (r->src.ip4)); - memcpy (&r->dst.ip4, rules[i].dst_ip_addr, sizeof (r->dst.ip4)); - } - r->src_prefixlen = rules[i].src_ip_prefix_len; - r->dst_prefixlen = rules[i].dst_ip_prefix_len; - r->proto = rules[i].proto; - r->src_port_or_type_first = rules[i].srcport_or_icmptype_first; - r->src_port_or_type_last = rules[i].srcport_or_icmptype_last; - r->dst_port_or_code_first = rules[i].dstport_or_icmpcode_first; - r->dst_port_or_code_last = rules[i].dstport_or_icmpcode_last; - r->tcp_flags_value = rules[i].tcp_flags_value; - r->tcp_flags_mask = rules[i].tcp_flags_mask; - } - - if (~0 == *acl_list_index) - { - /* Get ACL index */ - pool_get_aligned (am->acls, a, CLIB_CACHE_LINE_BYTES); - memset (a, 0, sizeof (*a)); - /* Will return the newly allocated ACL index */ - *acl_list_index = a - am->acls; - } - else - { - a = am->acls + *acl_list_index; - /* Get rid of the old rules */ - clib_mem_free (a->rules); - } - a->rules = acl_new_rules; - a->count = count; - memcpy (a->tag, tag, sizeof (a->tag)); - - return 0; -} - -static int -acl_del_list (u32 acl_list_index) -{ - acl_main_t *am = &acl_main; - acl_list_t *a; - int i, ii; - if (pool_is_free_index (am->acls, acl_list_index)) - { - return -1; - } - - /* delete any references to the ACL */ - for (i = 0; i < vec_len (am->output_acl_vec_by_sw_if_index); i++) - { - for (ii = 0; ii < vec_len (am->output_acl_vec_by_sw_if_index[i]); - /* see body */ ) - { - if (acl_list_index == am->output_acl_vec_by_sw_if_index[i][ii]) - { - vec_del1 (am->output_acl_vec_by_sw_if_index[i], ii); - } - else - { - ii++; - } - } - } - for (i = 0; i < vec_len (am->input_acl_vec_by_sw_if_index); i++) - { - for (ii = 0; ii < vec_len (am->input_acl_vec_by_sw_if_index[i]); - /* see body */ ) - { - if (acl_list_index == am->input_acl_vec_by_sw_if_index[i][ii]) - { - vec_del1 (am->input_acl_vec_by_sw_if_index[i], ii); - } - else - { - ii++; - } - } - } - - /* now we can delete the ACL itself */ - a = &am->acls[acl_list_index]; - if (a->rules) - { - clib_mem_free (a->rules); - } - pool_put (am->acls, a); - return 0; -} - -/* Some aids in ASCII graphing the content */ -#define XX "\377" -#define __ "\000" -#define _(x) -#define v - -u8 ip4_5tuple_mask[] = -_(" dmac smac etype ") -_(ether) __ __ __ __ __ __ v __ __ __ __ __ __ v __ __ v - _(" v ihl totlen ") - _(0x0000) - __ __ __ __ - _(" ident fl+fo ") - _(0x0004) - __ __ __ __ - _(" ttl pr checksum ") - _(0x0008) - __ XX __ __ - _(" src address ") - _(0x000C) - XX XX XX XX - _(" dst address ") - _(0x0010) - XX XX XX XX - _("L4 T/U sport dport ") - _(tcpudp) - XX XX XX XX - _(padpad) - __ __ __ __ - _(padpad) - __ __ __ __ - _(padeth) - __ __; - - u8 ip6_5tuple_mask[] = - _(" dmac smac etype ") - _(ether) __ __ __ __ __ __ v __ __ __ __ __ __ v __ __ v - _(" v tc + flow ") - _(0x0000) __ __ __ __ - _(" plen nh hl ") - _(0x0004) __ __ XX __ - _(" src address ") - _(0x0008) XX XX XX XX - _(0x000C) XX XX XX XX - _(0x0010) XX XX XX XX - _(0x0014) XX XX XX XX - _(" dst address ") - _(0x0018) XX XX XX XX - _(0x001C) XX XX XX XX - _(0x0020) XX XX XX XX - _(0x0024) XX XX XX XX - _("L4T/U sport dport ") - _(tcpudp) XX XX XX XX _(padpad) __ __ __ __ _(padeth) __ __; - -#undef XX -#undef __ -#undef _ -#undef v - - static int count_skip (u8 * p, u32 size) -{ - u64 *p64 = (u64 *) p; - /* Be tolerant to null pointer */ - if (0 == p) - return 0; - - while ((0ULL == *p64) && ((u8 *) p64 - p) < size) - { - p64++; - } - return (p64 - (u64 *) p) / 2; -} - -static int -acl_classify_add_del_table_big (vnet_classify_main_t * cm, u8 * mask, - u32 mask_len, u32 next_table_index, - u32 miss_next_index, u32 * table_index, - int is_add) -{ - u32 nbuckets = 65536; - u32 memory_size = 2 << 30; - u32 skip = count_skip (mask, mask_len); - u32 match = (mask_len / 16) - skip; - u8 *skip_mask_ptr = mask + 16 * skip; - u32 current_data_flag = 0; - int current_data_offset = 0; - - if (0 == match) - match = 1; - - return vnet_classify_add_del_table (cm, skip_mask_ptr, nbuckets, - memory_size, skip, match, - next_table_index, miss_next_index, - table_index, current_data_flag, - current_data_offset, is_add, - 1 /* delete_chain */); -} - -static int -acl_classify_add_del_table_small (vnet_classify_main_t * cm, u8 * mask, - u32 mask_len, u32 next_table_index, - u32 miss_next_index, u32 * table_index, - int is_add) -{ - u32 nbuckets = 32; - u32 memory_size = 2 << 20; - u32 skip = count_skip (mask, mask_len); - u32 match = (mask_len / 16) - skip; - u8 *skip_mask_ptr = mask + 16 * skip; - u32 current_data_flag = 0; - int current_data_offset = 0; - - if (0 == match) - match = 1; - - return vnet_classify_add_del_table (cm, skip_mask_ptr, nbuckets, - memory_size, skip, match, - next_table_index, miss_next_index, - table_index, current_data_flag, - current_data_offset, is_add, - 1 /* delete_chain */); -} - - -static int -acl_unhook_l2_input_classify (acl_main_t * am, u32 sw_if_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - u32 ip4_table_index = ~0; - u32 ip6_table_index = ~0; - - vec_validate_init_empty (am->acl_ip4_input_classify_table_by_sw_if_index, - sw_if_index, ~0); - vec_validate_init_empty (am->acl_ip6_input_classify_table_by_sw_if_index, - sw_if_index, ~0); - - vnet_l2_input_classify_enable_disable (sw_if_index, 0); - - if (am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index] != ~0) - { - ip4_table_index = - am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index]; - am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index] = ~0; - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip4_table_index, 0); - } - if (am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index] != ~0) - { - ip6_table_index = - am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index]; - am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index] = ~0; - acl_classify_add_del_table_big (cm, ip6_5tuple_mask, - sizeof (ip6_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip6_table_index, 0); - } - - return 0; -} - -static int -acl_unhook_l2_output_classify (acl_main_t * am, u32 sw_if_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - u32 ip4_table_index = ~0; - u32 ip6_table_index = ~0; - - vec_validate_init_empty (am->acl_ip4_output_classify_table_by_sw_if_index, - sw_if_index, ~0); - vec_validate_init_empty (am->acl_ip6_output_classify_table_by_sw_if_index, - sw_if_index, ~0); - - vnet_l2_output_classify_enable_disable (sw_if_index, 0); - - if (am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index] != ~0) - { - ip4_table_index = - am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index]; - am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index] = ~0; - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip4_table_index, 0); - } - if (am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index] != ~0) - { - ip6_table_index = - am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index]; - am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index] = ~0; - acl_classify_add_del_table_big (cm, ip6_5tuple_mask, - sizeof (ip6_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip6_table_index, 0); - } - - return 0; -} - -static int -acl_hook_l2_input_classify (acl_main_t * am, u32 sw_if_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - u32 ip4_table_index = ~0; - u32 ip6_table_index = ~0; - int rv; - - /* in case there were previous tables attached */ - acl_unhook_l2_input_classify (am, sw_if_index); - rv = - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip4_table_index, 1); - if (rv) - return rv; - rv = - acl_classify_add_del_table_big (cm, ip6_5tuple_mask, - sizeof (ip6_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip6_table_index, 1); - if (rv) - { - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip4_table_index, 0); - return rv; - } - rv = - vnet_l2_input_classify_set_tables (sw_if_index, ip4_table_index, - ip6_table_index, ~0); - clib_warning - ("ACL enabling on interface sw_if_index %d, setting tables to the following: ip4: %d ip6: %d\n", - sw_if_index, ip4_table_index, ip6_table_index); - if (rv) - { - acl_classify_add_del_table_big (cm, ip6_5tuple_mask, - sizeof (ip6_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip6_table_index, 0); - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_input_classify_next_acl, - &ip4_table_index, 0); - return rv; - } - - am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index] = - ip4_table_index; - am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index] = - ip6_table_index; - - vnet_l2_input_classify_enable_disable (sw_if_index, 1); - return rv; -} - -static int -acl_hook_l2_output_classify (acl_main_t * am, u32 sw_if_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - u32 ip4_table_index = ~0; - u32 ip6_table_index = ~0; - int rv; - - /* in case there were previous tables attached */ - acl_unhook_l2_output_classify (am, sw_if_index); - rv = - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip4_table_index, 1); - if (rv) - return rv; - rv = - acl_classify_add_del_table_big (cm, ip6_5tuple_mask, - sizeof (ip6_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip6_table_index, 1); - if (rv) - { - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip4_table_index, 0); - return rv; - } - rv = - vnet_l2_output_classify_set_tables (sw_if_index, ip4_table_index, - ip6_table_index, ~0); - clib_warning - ("ACL enabling on interface sw_if_index %d, setting tables to the following: ip4: %d ip6: %d\n", - sw_if_index, ip4_table_index, ip6_table_index); - if (rv) - { - acl_classify_add_del_table_big (cm, ip6_5tuple_mask, - sizeof (ip6_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip6_table_index, 0); - acl_classify_add_del_table_big (cm, ip4_5tuple_mask, - sizeof (ip4_5tuple_mask) - 1, ~0, - am->l2_output_classify_next_acl, - &ip4_table_index, 0); - return rv; - } - - am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index] = - ip4_table_index; - am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index] = - ip6_table_index; - - vnet_l2_output_classify_enable_disable (sw_if_index, 1); - return rv; -} - - -int -acl_interface_in_enable_disable (acl_main_t * am, u32 sw_if_index, - int enable_disable) -{ - int rv; - - /* Utterly wrong? */ - if (pool_is_free_index (am->vnet_main->interface_main.sw_interfaces, - sw_if_index)) - return VNET_API_ERROR_INVALID_SW_IF_INDEX; - - if (enable_disable) - { - rv = acl_hook_l2_input_classify (am, sw_if_index); - } - else - { - rv = acl_unhook_l2_input_classify (am, sw_if_index); - } - - return rv; -} - -int -acl_interface_out_enable_disable (acl_main_t * am, u32 sw_if_index, - int enable_disable) -{ - int rv; - - /* Utterly wrong? */ - if (pool_is_free_index (am->vnet_main->interface_main.sw_interfaces, - sw_if_index)) - return VNET_API_ERROR_INVALID_SW_IF_INDEX; - - if (enable_disable) - { - rv = acl_hook_l2_output_classify (am, sw_if_index); - } - else - { - rv = acl_unhook_l2_output_classify (am, sw_if_index); - } - - return rv; -} - - -static int -acl_interface_add_inout_acl (u32 sw_if_index, u8 is_input, u32 acl_list_index) -{ - acl_main_t *am = &acl_main; - if (is_input) - { - vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); - vec_add (am->input_acl_vec_by_sw_if_index[sw_if_index], &acl_list_index, - 1); - acl_interface_in_enable_disable (am, sw_if_index, 1); - } - else - { - vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); - vec_add (am->output_acl_vec_by_sw_if_index[sw_if_index], - &acl_list_index, 1); - acl_interface_out_enable_disable (am, sw_if_index, 1); - } - return 0; -} - -static int -acl_interface_del_inout_acl (u32 sw_if_index, u8 is_input, u32 acl_list_index) -{ - acl_main_t *am = &acl_main; - int i; - int rv = -1; - if (is_input) - { - vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); - for (i = 0; i < vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]); - i++) - { - if (acl_list_index == - am->input_acl_vec_by_sw_if_index[sw_if_index][i]) - { - vec_del1 (am->input_acl_vec_by_sw_if_index[sw_if_index], i); - rv = 0; - break; - } - } - if (0 == vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index])) - { - acl_interface_in_enable_disable (am, sw_if_index, 0); - } - } - else - { - vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); - for (i = 0; - i < vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]); i++) - { - if (acl_list_index == - am->output_acl_vec_by_sw_if_index[sw_if_index][i]) - { - vec_del1 (am->output_acl_vec_by_sw_if_index[sw_if_index], i); - rv = 0; - break; - } - } - if (0 == vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index])) - { - acl_interface_out_enable_disable (am, sw_if_index, 0); - } - } - return rv; -} - -static void -acl_interface_reset_inout_acls (u32 sw_if_index, u8 is_input) -{ - acl_main_t *am = &acl_main; - if (is_input) - { - acl_interface_in_enable_disable (am, sw_if_index, 0); - vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); - vec_reset_length (am->input_acl_vec_by_sw_if_index[sw_if_index]); - } - else - { - acl_interface_out_enable_disable (am, sw_if_index, 0); - vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); - vec_reset_length (am->output_acl_vec_by_sw_if_index[sw_if_index]); - } -} - -static int -acl_interface_add_del_inout_acl (u32 sw_if_index, u8 is_add, u8 is_input, - u32 acl_list_index) -{ - int rv = -1; - if (is_add) - { - rv = - acl_interface_add_inout_acl (sw_if_index, is_input, acl_list_index); - } - else - { - rv = - acl_interface_del_inout_acl (sw_if_index, is_input, acl_list_index); - } - return rv; -} - - -static void * -get_ptr_to_offset (vlib_buffer_t * b0, int offset) -{ - u8 *p = vlib_buffer_get_current (b0) + offset; - return p; -} - -static u8 -acl_get_l4_proto (vlib_buffer_t * b0, int node_is_ip6) -{ - u8 proto; - int proto_offset; - if (node_is_ip6) - { - proto_offset = 20; - } - else - { - proto_offset = 23; - } - proto = *((u8 *) vlib_buffer_get_current (b0) + proto_offset); - return proto; -} - -static int -acl_match_addr (ip46_address_t * addr1, ip46_address_t * addr2, int prefixlen, - int is_ip6) -{ - if (prefixlen == 0) - { - /* match any always succeeds */ - return 1; - } - if (is_ip6) - { - if (memcmp (addr1, addr2, prefixlen / 8)) - { - /* If the starting full bytes do not match, no point in bittwidling the thumbs further */ - return 0; - } - if (prefixlen % 8) - { - u8 b1 = *((u8 *) addr1 + 1 + prefixlen / 8); - u8 b2 = *((u8 *) addr2 + 1 + prefixlen / 8); - u8 mask0 = (0xff - ((1 << (8 - (prefixlen % 8))) - 1)); - return (b1 & mask0) == b2; - } - else - { - /* The prefix fits into integer number of bytes, so nothing left to do */ - return 1; - } - } - else - { - uint32_t a1 = ntohl (addr1->ip4.as_u32); - uint32_t a2 = ntohl (addr2->ip4.as_u32); - uint32_t mask0 = 0xffffffff - ((1 << (32 - prefixlen)) - 1); - return (a1 & mask0) == a2; - } -} - -static int -acl_match_port (u16 port, u16 port_first, u16 port_last, int is_ip6) -{ - return ((port >= port_first) && (port <= port_last)); -} - -static int -acl_packet_match (acl_main_t * am, u32 acl_index, vlib_buffer_t * b0, - u8 * r_action, int *r_is_ip6, u32 * r_acl_match_p, - u32 * r_rule_match_p, u32 * trace_bitmap) -{ - ethernet_header_t *h0; - u16 type0; - - ip46_address_t src, dst; - int is_ip6; - int is_ip4; - u8 proto; - u16 src_port; - u16 dst_port; - u8 tcp_flags = 0; - int i; - acl_list_t *a; - acl_rule_t *r; - - h0 = vlib_buffer_get_current (b0); - type0 = clib_net_to_host_u16 (h0->type); - is_ip4 = (type0 == ETHERNET_TYPE_IP4); - is_ip6 = (type0 == ETHERNET_TYPE_IP6); - - if (!(is_ip4 || is_ip6)) - { - return 0; - } - /* The bunch of hardcoded offsets here is intentional to get rid of them - ASAP, when getting to a faster matching code */ - if (is_ip4) - { - clib_memcpy (&src.ip4, get_ptr_to_offset (b0, 26), 4); - clib_memcpy (&dst.ip4, get_ptr_to_offset (b0, 30), 4); - proto = acl_get_l4_proto (b0, 0); - if (1 == proto) - { - *trace_bitmap |= 0x00000001; - /* type */ - src_port = *(u8 *) get_ptr_to_offset (b0, 34); - /* code */ - dst_port = *(u8 *) get_ptr_to_offset (b0, 35); - } - else - { - /* assume TCP/UDP */ - src_port = (*(u16 *) get_ptr_to_offset (b0, 34)); - dst_port = (*(u16 *) get_ptr_to_offset (b0, 36)); - /* UDP gets ability to check on an oddball data byte as a bonus */ - tcp_flags = *(u8 *) get_ptr_to_offset (b0, 14 + 20 + 13); - } - } - else /* is_ipv6 implicitly */ - { - clib_memcpy (&src, get_ptr_to_offset (b0, 22), 16); - clib_memcpy (&dst, get_ptr_to_offset (b0, 38), 16); - proto = acl_get_l4_proto (b0, 1); - if (58 == proto) - { - *trace_bitmap |= 0x00000002; - /* type */ - src_port = *(u8 *) get_ptr_to_offset (b0, 54); - /* code */ - dst_port = *(u8 *) get_ptr_to_offset (b0, 55); - } - else - { - /* assume TCP/UDP */ - src_port = (*(u16 *) get_ptr_to_offset (b0, 54)); - dst_port = (*(u16 *) get_ptr_to_offset (b0, 56)); - tcp_flags = *(u8 *) get_ptr_to_offset (b0, 14 + 40 + 13); - } - } - if (pool_is_free_index (am->acls, acl_index)) - { - if (r_acl_match_p) - *r_acl_match_p = acl_index; - if (r_rule_match_p) - *r_rule_match_p = -1; - /* the ACL does not exist but is used for policy. Block traffic. */ - return 0; - } - a = am->acls + acl_index; - for (i = 0; i < a->count; i++) - { - r = a->rules + i; - if (is_ip6 != r->is_ipv6) - { - continue; - } - if (!acl_match_addr (&dst, &r->dst, r->dst_prefixlen, is_ip6)) - continue; - if (!acl_match_addr (&src, &r->src, r->src_prefixlen, is_ip6)) - continue; - if (r->proto) - { - if (proto != r->proto) - continue; - if (!acl_match_port - (src_port, r->src_port_or_type_first, r->src_port_or_type_last, - is_ip6)) - continue; - if (!acl_match_port - (dst_port, r->dst_port_or_code_first, r->dst_port_or_code_last, - is_ip6)) - continue; - /* No need for check of proto == TCP, since in other rules both fields should be zero, so this match will succeed */ - if ((tcp_flags & r->tcp_flags_mask) != r->tcp_flags_value) - continue; - } - /* everything matches! */ - *r_action = r->is_permit; - *r_is_ip6 = is_ip6; - if (r_acl_match_p) - *r_acl_match_p = acl_index; - if (r_rule_match_p) - *r_rule_match_p = i; - return 1; - } - return 0; -} - -void -input_acl_packet_match (u32 sw_if_index, vlib_buffer_t * b0, u32 * nextp, - u32 * acl_match_p, u32 * rule_match_p, - u32 * trace_bitmap) -{ - acl_main_t *am = &acl_main; - uint8_t action = 0; - int is_ip6 = 0; - int i; - vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); - for (i = 0; i < vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]); - i++) - { - if (acl_packet_match - (am, am->input_acl_vec_by_sw_if_index[sw_if_index][i], b0, &action, - &is_ip6, acl_match_p, rule_match_p, trace_bitmap)) - { - if (is_ip6) - { - *nextp = am->acl_in_ip6_match_next[action]; - } - else - { - *nextp = am->acl_in_ip4_match_next[action]; - } - return; - } - } - if (vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]) > 0) - { - /* If there are ACLs and none matched, deny by default */ - *nextp = 0; - } - -} - -void -output_acl_packet_match (u32 sw_if_index, vlib_buffer_t * b0, u32 * nextp, - u32 * acl_match_p, u32 * rule_match_p, - u32 * trace_bitmap) -{ - acl_main_t *am = &acl_main; - uint8_t action = 0; - int is_ip6 = 0; - int i; - vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); - for (i = 0; i < vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]); - i++) - { - if (acl_packet_match - (am, am->output_acl_vec_by_sw_if_index[sw_if_index][i], b0, &action, - &is_ip6, acl_match_p, rule_match_p, trace_bitmap)) - { - if (is_ip6) - { - *nextp = am->acl_out_ip6_match_next[action]; - } - else - { - *nextp = am->acl_out_ip4_match_next[action]; - } - return; - } - } - if (vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]) > 0) - { - /* If there are ACLs and none matched, deny by default */ - *nextp = 0; - } -} - -typedef struct -{ - u8 is_ipv6; - u8 mac_mask[6]; - u8 prefix_len; - u32 count; - u32 table_index; -} macip_match_type_t; - -static u32 -macip_find_match_type (macip_match_type_t * mv, u8 * mac_mask, u8 prefix_len, - u8 is_ipv6) -{ - u32 i; - if (mv) - { - for (i = 0; i < vec_len (mv); i++) - { - if ((mv[i].prefix_len == prefix_len) && (mv[i].is_ipv6 == is_ipv6) - && (0 == memcmp (mv[i].mac_mask, mac_mask, 6))) - { - return i; - } - } - } - return ~0; -} - - -/* Get metric used to sort match types. - The more specific and the more often seen - the bigger the metric */ -static int -match_type_metric (macip_match_type_t * m) -{ - /* FIXME: count the ones in the MAC mask as well, check how well this heuristic works in real life */ - return m->prefix_len + m->is_ipv6 + 10 * m->count; -} - -static int -match_type_compare (macip_match_type_t * m1, macip_match_type_t * m2) -{ - /* Ascending sort based on the metric values */ - return match_type_metric (m1) - match_type_metric (m2); -} - -/* Get the offset of L3 source within ethernet packet */ -static int -get_l3_src_offset(int is6) -{ - if(is6) - return (sizeof(ethernet_header_t) + offsetof(ip6_header_t, src_address)); - else - return (sizeof(ethernet_header_t) + offsetof(ip4_header_t, src_address)); -} - -static int -macip_create_classify_tables (acl_main_t * am, u32 macip_acl_index) -{ - macip_match_type_t *mvec = NULL; - macip_match_type_t *mt; - macip_acl_list_t *a = &am->macip_acls[macip_acl_index]; - int i; - u32 match_type_index; - u32 last_table; - u8 mask[5 * 16]; - vnet_classify_main_t *cm = &vnet_classify_main; - - /* Count the number of different types of rules */ - for (i = 0; i < a->count; i++) - { - if (~0 == - (match_type_index = - macip_find_match_type (mvec, a->rules[i].src_mac_mask, - a->rules[i].src_prefixlen, - a->rules[i].is_ipv6))) - { - match_type_index = vec_len (mvec); - vec_validate (mvec, match_type_index); - memcpy (mvec[match_type_index].mac_mask, - a->rules[match_type_index].src_mac_mask, 6); - mvec[match_type_index].prefix_len = a->rules[i].src_prefixlen; - mvec[match_type_index].is_ipv6 = a->rules[i].is_ipv6; - mvec[match_type_index].table_index = ~0; - } - mvec[match_type_index].count++; - } - /* Put the most frequently used tables last in the list so we can create classifier tables in reverse order */ - vec_sort_with_function (mvec, match_type_compare); - /* Create the classifier tables */ - last_table = ~0; - vec_foreach (mt, mvec) - { - int mask_len; - int is6 = mt->is_ipv6; - int l3_src_offs = get_l3_src_offset(is6); - memset (mask, 0, sizeof (mask)); - memcpy (&mask[6], mt->mac_mask, 6); - for (i = 0; i < (mt->prefix_len / 8); i++) - { - mask[l3_src_offs + i] = 0xff; - } - if (mt->prefix_len % 8) - { - mask[l3_src_offs + (mt->prefix_len / 8)] = - 0xff - ((1 << (8 - mt->prefix_len % 8)) - 1); - } - /* - * Round-up the number of bytes needed to store the prefix, - * and round up the number of vectors too - */ - mask_len = ((l3_src_offs + ((mt->prefix_len+7) / 8) + - (sizeof (u32x4)-1))/sizeof(u32x4)) * sizeof (u32x4); - acl_classify_add_del_table_small (cm, mask, mask_len, last_table, - (~0 == last_table) ? 0 : ~0, &mt->table_index, - 1); - last_table = mt->table_index; - } - a->ip4_table_index = ~0; - a->ip6_table_index = ~0; - a->l2_table_index = last_table; - - /* Populate the classifier tables with rules from the MACIP ACL */ - for (i = 0; i < a->count; i++) - { - u32 action = 0; - u32 metadata = 0; - int is6 = a->rules[i].is_ipv6; - int l3_src_offs = get_l3_src_offset(is6); - memset (mask, 0, sizeof (mask)); - memcpy (&mask[6], a->rules[i].src_mac, 6); - if (is6) - { - memcpy (&mask[l3_src_offs], &a->rules[i].src_ip_addr.ip6, 16); - } - else - { - memcpy (&mask[l3_src_offs], &a->rules[i].src_ip_addr.ip4, 4); - } - match_type_index = - macip_find_match_type (mvec, a->rules[i].src_mac_mask, - a->rules[i].src_prefixlen, - a->rules[i].is_ipv6); - /* add session to table mvec[match_type_index].table_index; */ - vnet_classify_add_del_session (cm, mvec[match_type_index].table_index, - mask, a->rules[i].is_permit ? ~0 : 0, i, - 0, action, metadata, 1); - } - return 0; -} - -static void -macip_destroy_classify_tables (acl_main_t * am, u32 macip_acl_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - macip_acl_list_t *a = &am->macip_acls[macip_acl_index]; - - if (a->ip4_table_index != ~0) - { - acl_classify_add_del_table_small (cm, 0, ~0, ~0, ~0, &a->ip4_table_index, 0); - a->ip4_table_index = ~0; - } - if (a->ip6_table_index != ~0) - { - acl_classify_add_del_table_small (cm, 0, ~0, ~0, ~0, &a->ip6_table_index, 0); - a->ip6_table_index = ~0; - } - if (a->l2_table_index != ~0) - { - acl_classify_add_del_table_small (cm, 0, ~0, ~0, ~0, &a->l2_table_index, 0); - a->l2_table_index = ~0; - } -} - -static int -macip_acl_add_list (u32 count, vl_api_macip_acl_rule_t rules[], - u32 * acl_list_index, u8 * tag) -{ - acl_main_t *am = &acl_main; - macip_acl_list_t *a; - macip_acl_rule_t *r; - macip_acl_rule_t *acl_new_rules; - int i; - - /* Create and populate the rules */ - acl_new_rules = clib_mem_alloc_aligned (sizeof (macip_acl_rule_t) * count, - CLIB_CACHE_LINE_BYTES); - if (!acl_new_rules) - { - /* Could not allocate rules. New or existing ACL - bail out regardless */ - return -1; - } - - for (i = 0; i < count; i++) - { - r = &acl_new_rules[i]; - r->is_permit = rules[i].is_permit; - r->is_ipv6 = rules[i].is_ipv6; - memcpy (&r->src_mac, rules[i].src_mac, 6); - memcpy (&r->src_mac_mask, rules[i].src_mac_mask, 6); - if(rules[i].is_ipv6) - memcpy (&r->src_ip_addr.ip6, rules[i].src_ip_addr, 16); - else - memcpy (&r->src_ip_addr.ip4, rules[i].src_ip_addr, 4); - r->src_prefixlen = rules[i].src_ip_prefix_len; - } - - /* Get ACL index */ - pool_get_aligned (am->macip_acls, a, CLIB_CACHE_LINE_BYTES); - memset (a, 0, sizeof (*a)); - /* Will return the newly allocated ACL index */ - *acl_list_index = a - am->macip_acls; - - a->rules = acl_new_rules; - a->count = count; - memcpy (a->tag, tag, sizeof (a->tag)); - - /* Create and populate the classifer tables */ - macip_create_classify_tables (am, *acl_list_index); - - return 0; -} - - -/* No check for validity of sw_if_index - the callers were supposed to validate */ - -static int -macip_acl_interface_del_acl (acl_main_t * am, u32 sw_if_index) -{ - int rv; - u32 macip_acl_index; - macip_acl_list_t *a; - vec_validate_init_empty (am->macip_acl_by_sw_if_index, sw_if_index, ~0); - macip_acl_index = am->macip_acl_by_sw_if_index[sw_if_index]; - /* No point in deleting MACIP ACL which is not applied */ - if (~0 == macip_acl_index) - return -1; - a = &am->macip_acls[macip_acl_index]; - /* remove the classifier tables off the interface L2 ACL */ - rv = - vnet_set_input_acl_intfc (am->vlib_main, sw_if_index, a->ip4_table_index, - a->ip6_table_index, a->l2_table_index, 0); - /* Unset the MACIP ACL index */ - am->macip_acl_by_sw_if_index[sw_if_index] = ~0; - return rv; -} - -/* No check for validity of sw_if_index - the callers were supposed to validate */ - -static int -macip_acl_interface_add_acl (acl_main_t * am, u32 sw_if_index, - u32 macip_acl_index) -{ - macip_acl_list_t *a; - int rv; - if (pool_is_free_index (am->macip_acls, macip_acl_index)) - { - return -1; - } - a = &am->macip_acls[macip_acl_index]; - vec_validate_init_empty (am->macip_acl_by_sw_if_index, sw_if_index, ~0); - /* If there already a MACIP ACL applied, unapply it */ - if (~0 != am->macip_acl_by_sw_if_index[sw_if_index]) - macip_acl_interface_del_acl(am, sw_if_index); - am->macip_acl_by_sw_if_index[sw_if_index] = macip_acl_index; - /* Apply the classifier tables for L2 ACLs */ - rv = - vnet_set_input_acl_intfc (am->vlib_main, sw_if_index, a->ip4_table_index, - a->ip6_table_index, a->l2_table_index, 1); - return rv; -} - -static int -macip_acl_del_list (u32 acl_list_index) -{ - acl_main_t *am = &acl_main; - macip_acl_list_t *a; - int i; - if (pool_is_free_index (am->macip_acls, acl_list_index)) - { - return -1; - } - - /* delete any references to the ACL */ - for (i = 0; i < vec_len (am->macip_acl_by_sw_if_index); i++) - { - if (am->macip_acl_by_sw_if_index[i] == acl_list_index) - { - macip_acl_interface_del_acl (am, i); - } - } - - /* Now that classifier tables are detached, clean them up */ - macip_destroy_classify_tables (am, acl_list_index); - - /* now we can delete the ACL itself */ - a = &am->macip_acls[acl_list_index]; - if (a->rules) - { - clib_mem_free (a->rules); - } - pool_put (am->macip_acls, a); - return 0; -} - - -static int -macip_acl_interface_add_del_acl (u32 sw_if_index, u8 is_add, - u32 acl_list_index) -{ - acl_main_t *am = &acl_main; - int rv = -1; - if (is_add) - { - rv = macip_acl_interface_add_acl (am, sw_if_index, acl_list_index); - } - else - { - rv = macip_acl_interface_del_acl (am, sw_if_index); - } - return rv; -} - -/* API message handler */ -static void -vl_api_acl_add_replace_t_handler (vl_api_acl_add_replace_t * mp) -{ - vl_api_acl_add_replace_reply_t *rmp; - acl_main_t *am = &acl_main; - int rv; - u32 acl_list_index = ntohl (mp->acl_index); - - rv = acl_add_list (ntohl (mp->count), mp->r, &acl_list_index, mp->tag); - - /* *INDENT-OFF* */ - REPLY_MACRO2(VL_API_ACL_ADD_REPLACE_REPLY, - ({ - rmp->acl_index = htonl(acl_list_index); - })); - /* *INDENT-ON* */ -} - -static void -vl_api_acl_del_t_handler (vl_api_acl_del_t * mp) -{ - acl_main_t *sm = &acl_main; - vl_api_acl_del_reply_t *rmp; - int rv; - - rv = acl_del_list (ntohl (mp->acl_index)); - - REPLY_MACRO (VL_API_ACL_DEL_REPLY); -} - -static void -vl_api_acl_interface_add_del_t_handler (vl_api_acl_interface_add_del_t * mp) -{ - acl_main_t *sm = &acl_main; - vnet_interface_main_t *im = &sm->vnet_main->interface_main; - u32 sw_if_index = ntohl (mp->sw_if_index); - vl_api_acl_interface_add_del_reply_t *rmp; - int rv = -1; - - if (pool_is_free_index(im->sw_interfaces, sw_if_index)) - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; - else - rv = - acl_interface_add_del_inout_acl (sw_if_index, mp->is_add, - mp->is_input, ntohl (mp->acl_index)); - - REPLY_MACRO (VL_API_ACL_INTERFACE_ADD_DEL_REPLY); -} - -static void -vl_api_acl_interface_set_acl_list_t_handler - (vl_api_acl_interface_set_acl_list_t * mp) -{ - acl_main_t *sm = &acl_main; - vl_api_acl_interface_set_acl_list_reply_t *rmp; - int rv = 0; - int i; - vnet_interface_main_t *im = &sm->vnet_main->interface_main; - u32 sw_if_index = ntohl (mp->sw_if_index); - - if (pool_is_free_index(im->sw_interfaces, sw_if_index)) - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; - else - { - acl_interface_reset_inout_acls (sw_if_index, 0); - acl_interface_reset_inout_acls (sw_if_index, 1); - - for (i = 0; i < mp->count; i++) - { - acl_interface_add_del_inout_acl (sw_if_index, 1, (i < mp->n_input), - ntohl (mp->acls[i])); - } - } - - REPLY_MACRO (VL_API_ACL_INTERFACE_SET_ACL_LIST_REPLY); -} - -static void -copy_acl_rule_to_api_rule (vl_api_acl_rule_t * api_rule, acl_rule_t * r) -{ - api_rule->is_permit = r->is_permit; - api_rule->is_ipv6 = r->is_ipv6; - if(r->is_ipv6) - { - memcpy (api_rule->src_ip_addr, &r->src, sizeof (r->src)); - memcpy (api_rule->dst_ip_addr, &r->dst, sizeof (r->dst)); - } - else - { - memcpy (api_rule->src_ip_addr, &r->src.ip4, sizeof (r->src.ip4)); - memcpy (api_rule->dst_ip_addr, &r->dst.ip4, sizeof (r->dst.ip4)); - } - api_rule->src_ip_prefix_len = r->src_prefixlen; - api_rule->dst_ip_prefix_len = r->dst_prefixlen; - api_rule->proto = r->proto; - api_rule->srcport_or_icmptype_first = r->src_port_or_type_first; - api_rule->srcport_or_icmptype_last = r->src_port_or_type_last; - api_rule->dstport_or_icmpcode_first = r->dst_port_or_code_first; - api_rule->dstport_or_icmpcode_last = r->dst_port_or_code_last; - api_rule->tcp_flags_mask = r->tcp_flags_mask; - api_rule->tcp_flags_value = r->tcp_flags_value; -} - -static void -send_acl_details (acl_main_t * am, unix_shared_memory_queue_t * q, - acl_list_t * acl, u32 context) -{ - vl_api_acl_details_t *mp; - vl_api_acl_rule_t *rules; - int i; - int msg_size = sizeof (*mp) + sizeof (mp->r[0]) * acl->count; - - mp = vl_msg_api_alloc (msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = ntohs (VL_API_ACL_DETAILS + am->msg_id_base); - - /* fill in the message */ - mp->context = context; - mp->count = htonl (acl->count); - mp->acl_index = htonl (acl - am->acls); - memcpy (mp->tag, acl->tag, sizeof (mp->tag)); - // clib_memcpy (mp->r, acl->rules, acl->count * sizeof(acl->rules[0])); - rules = mp->r; - for (i = 0; i < acl->count; i++) - { - copy_acl_rule_to_api_rule (&rules[i], &acl->rules[i]); - } - - clib_warning("Sending acl details for ACL index %d", ntohl(mp->acl_index)); - vl_msg_api_send_shmem (q, (u8 *) & mp); -} - - -static void -vl_api_acl_dump_t_handler (vl_api_acl_dump_t * mp) -{ - acl_main_t *am = &acl_main; - u32 acl_index; - acl_list_t *acl; - - int rv = -1; - unix_shared_memory_queue_t *q; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - { - return; - } - - if (mp->acl_index == ~0) - { - /* *INDENT-OFF* */ - /* Just dump all ACLs */ - pool_foreach (acl, am->acls, - ({ - send_acl_details(am, q, acl, mp->context); - })); - /* *INDENT-ON* */ - } - else - { - acl_index = ntohl (mp->acl_index); - if (!pool_is_free_index (am->acls, acl_index)) - { - acl = &am->acls[acl_index]; - send_acl_details (am, q, acl, mp->context); - } - } - - if (rv == -1) - { - /* FIXME API: should we signal an error here at all ? */ - return; - } -} - -static void -send_acl_interface_list_details (acl_main_t * am, - unix_shared_memory_queue_t * q, - u32 sw_if_index, u32 context) -{ - vl_api_acl_interface_list_details_t *mp; - int msg_size; - int n_input; - int n_output; - int count; - int i = 0; - - vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); - vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); - - n_input = vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]); - n_output = vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]); - count = n_input + n_output; - - msg_size = sizeof (*mp); - msg_size += sizeof (mp->acls[0]) * count; - - mp = vl_msg_api_alloc (msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = - ntohs (VL_API_ACL_INTERFACE_LIST_DETAILS + am->msg_id_base); - - /* fill in the message */ - mp->context = context; - mp->sw_if_index = htonl (sw_if_index); - mp->count = count; - mp->n_input = n_input; - for (i = 0; i < n_input; i++) - { - mp->acls[i] = htonl (am->input_acl_vec_by_sw_if_index[sw_if_index][i]); - } - for (i = 0; i < n_output; i++) - { - mp->acls[n_input + i] = - htonl (am->output_acl_vec_by_sw_if_index[sw_if_index][i]); - } - - vl_msg_api_send_shmem (q, (u8 *) & mp); -} - -static void -vl_api_acl_interface_list_dump_t_handler (vl_api_acl_interface_list_dump_t * - mp) -{ - acl_main_t *am = &acl_main; - vnet_sw_interface_t *swif; - vnet_interface_main_t *im = &am->vnet_main->interface_main; - - u32 sw_if_index; - unix_shared_memory_queue_t *q; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - { - return; - } - - if (mp->sw_if_index == ~0) - { - /* *INDENT-OFF* */ - pool_foreach (swif, im->sw_interfaces, - ({ - send_acl_interface_list_details(am, q, swif->sw_if_index, mp->context); - })); - /* *INDENT-ON* */ - } - else - { - sw_if_index = ntohl (mp->sw_if_index); - if (!pool_is_free_index(im->sw_interfaces, sw_if_index)) - send_acl_interface_list_details (am, q, sw_if_index, mp->context); - } -} - -/* MACIP ACL API handlers */ - -static void -vl_api_macip_acl_add_t_handler (vl_api_macip_acl_add_t * mp) -{ - vl_api_macip_acl_add_reply_t *rmp; - acl_main_t *am = &acl_main; - int rv; - u32 acl_list_index = ~0; - - rv = - macip_acl_add_list (ntohl (mp->count), mp->r, &acl_list_index, mp->tag); - - /* *INDENT-OFF* */ - REPLY_MACRO2(VL_API_MACIP_ACL_ADD_REPLY, - ({ - rmp->acl_index = htonl(acl_list_index); - })); - /* *INDENT-ON* */ -} - -static void -vl_api_macip_acl_del_t_handler (vl_api_macip_acl_del_t * mp) -{ - acl_main_t *sm = &acl_main; - vl_api_macip_acl_del_reply_t *rmp; - int rv; - - rv = macip_acl_del_list (ntohl (mp->acl_index)); - - REPLY_MACRO (VL_API_MACIP_ACL_DEL_REPLY); -} - -static void -vl_api_macip_acl_interface_add_del_t_handler - (vl_api_macip_acl_interface_add_del_t * mp) -{ - acl_main_t *sm = &acl_main; - vl_api_macip_acl_interface_add_del_reply_t *rmp; - int rv = -1; - vnet_interface_main_t *im = &sm->vnet_main->interface_main; - u32 sw_if_index = ntohl (mp->sw_if_index); - - if (pool_is_free_index(im->sw_interfaces, sw_if_index)) - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; - else - rv = - macip_acl_interface_add_del_acl (ntohl (mp->sw_if_index), mp->is_add, - ntohl (mp->acl_index)); - - REPLY_MACRO (VL_API_MACIP_ACL_INTERFACE_ADD_DEL_REPLY); -} - -static void -send_macip_acl_details (acl_main_t * am, unix_shared_memory_queue_t * q, - macip_acl_list_t * acl, u32 context) -{ - vl_api_macip_acl_details_t *mp; - vl_api_macip_acl_rule_t *rules; - macip_acl_rule_t *r; - int i; - int msg_size = sizeof (*mp) + (acl ? sizeof (mp->r[0]) * acl->count : 0); - - mp = vl_msg_api_alloc (msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = ntohs (VL_API_MACIP_ACL_DETAILS + am->msg_id_base); - - /* fill in the message */ - mp->context = context; - if (acl) - { - memcpy (mp->tag, acl->tag, sizeof (mp->tag)); - mp->count = htonl (acl->count); - mp->acl_index = htonl (acl - am->macip_acls); - rules = mp->r; - for (i = 0; i < acl->count; i++) - { - r = &acl->rules[i]; - rules[i].is_permit = r->is_permit; - rules[i].is_ipv6 = r->is_ipv6; - memcpy (rules[i].src_mac, &r->src_mac, sizeof (r->src_mac)); - memcpy (rules[i].src_mac_mask, &r->src_mac_mask, - sizeof (r->src_mac_mask)); - if (r->is_ipv6) - memcpy (rules[i].src_ip_addr, &r->src_ip_addr.ip6, - sizeof (r->src_ip_addr.ip6)); - else - memcpy (rules[i].src_ip_addr, &r->src_ip_addr.ip4, - sizeof (r->src_ip_addr.ip4)); - rules[i].src_ip_prefix_len = r->src_prefixlen; - } - } - else - { - /* No martini, no party - no ACL applied to this interface. */ - mp->acl_index = ~0; - mp->count = 0; - } - - vl_msg_api_send_shmem (q, (u8 *) & mp); -} - - -static void -vl_api_macip_acl_dump_t_handler (vl_api_macip_acl_dump_t * mp) -{ - acl_main_t *am = &acl_main; - macip_acl_list_t *acl; - - unix_shared_memory_queue_t *q; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - { - return; - } - - if (mp->acl_index == ~0) - { - /* Just dump all ACLs for now, with sw_if_index = ~0 */ - pool_foreach (acl, am->macip_acls, ( - { - send_macip_acl_details (am, q, acl, - mp-> - context);} - )); - /* *INDENT-ON* */ - } - else - { - u32 acl_index = ntohl (mp->acl_index); - if (!pool_is_free_index (am->macip_acls, acl_index)) - { - acl = &am->macip_acls[acl_index]; - send_macip_acl_details (am, q, acl, mp->context); - } - } -} - -static void -vl_api_macip_acl_interface_get_t_handler (vl_api_macip_acl_interface_get_t * - mp) -{ - acl_main_t *am = &acl_main; - vl_api_macip_acl_interface_get_reply_t *rmp; - u32 count = vec_len (am->macip_acl_by_sw_if_index); - int msg_size = sizeof (*rmp) + sizeof (rmp->acls[0]) * count; - unix_shared_memory_queue_t *q; - int i; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - { - return; - } - - rmp = vl_msg_api_alloc (msg_size); - memset (rmp, 0, msg_size); - rmp->_vl_msg_id = - ntohs (VL_API_MACIP_ACL_INTERFACE_GET_REPLY + am->msg_id_base); - rmp->context = mp->context; - rmp->count = htonl (count); - for (i = 0; i < count; i++) - { - rmp->acls[i] = htonl (am->macip_acl_by_sw_if_index[i]); - } - - vl_msg_api_send_shmem (q, (u8 *) & rmp); -} - - - -/* Set up the API message handling tables */ -static clib_error_t * -acl_plugin_api_hookup (vlib_main_t * vm) -{ - acl_main_t *sm = &acl_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_acl_plugin_api_msg; -#undef _ - - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (acl_main_t * sm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); - foreach_vl_msg_name_crc_acl; -#undef _ -} - -u32 -register_match_action_nexts (u32 next_in_ip4, u32 next_in_ip6, - u32 next_out_ip4, u32 next_out_ip6) -{ - acl_main_t *am = &acl_main; - u32 act = am->n_match_actions; - if (am->n_match_actions == 255) - { - return ~0; - } - am->n_match_actions++; - am->acl_in_ip4_match_next[act] = next_in_ip4; - am->acl_in_ip6_match_next[act] = next_in_ip6; - am->acl_out_ip4_match_next[act] = next_out_ip4; - am->acl_out_ip6_match_next[act] = next_out_ip6; - return act; -} - -void -acl_setup_nodes (void) -{ - vlib_main_t *vm = vlib_get_main (); - acl_main_t *am = &acl_main; - vlib_node_t *n; - - n = vlib_get_node_by_name (vm, (u8 *) "l2-input-classify"); - am->l2_input_classify_next_acl = - vlib_node_add_next_with_slot (vm, n->index, acl_in_node.index, ~0); - n = vlib_get_node_by_name (vm, (u8 *) "l2-output-classify"); - am->l2_output_classify_next_acl = - vlib_node_add_next_with_slot (vm, n->index, acl_out_node.index, ~0); - - feat_bitmap_init_next_nodes (vm, acl_in_node.index, L2INPUT_N_FEAT, - l2input_get_feat_names (), - am->acl_in_node_input_next_node_index); - - memset (&am->acl_in_ip4_match_next[0], 0, - sizeof (am->acl_in_ip4_match_next)); - memset (&am->acl_in_ip6_match_next[0], 0, - sizeof (am->acl_in_ip6_match_next)); - memset (&am->acl_out_ip4_match_next[0], 0, - sizeof (am->acl_out_ip4_match_next)); - memset (&am->acl_out_ip6_match_next[0], 0, - sizeof (am->acl_out_ip6_match_next)); - am->n_match_actions = 0; - - register_match_action_nexts (0, 0, 0, 0); /* drop */ - register_match_action_nexts (~0, ~0, ~0, ~0); /* permit */ - register_match_action_nexts (ACL_IN_L2S_INPUT_IP4_ADD, ACL_IN_L2S_INPUT_IP6_ADD, ACL_OUT_L2S_OUTPUT_IP4_ADD, ACL_OUT_L2S_OUTPUT_IP6_ADD); /* permit + create session */ -} - - - -static clib_error_t * -acl_init (vlib_main_t * vm) -{ - acl_main_t *am = &acl_main; - clib_error_t *error = 0; - memset (am, 0, sizeof (*am)); - am->vlib_main = vm; - am->vnet_main = vnet_get_main (); - - u8 *name = format (0, "acl_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - am->msg_id_base = vl_msg_api_get_msg_ids ((char *) name, - VL_MSG_FIRST_AVAILABLE); - - error = acl_plugin_api_hookup (vm); - acl_setup_nodes (); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (am, &api_main); - - vec_free (name); - - return error; -} - -VLIB_INIT_FUNCTION (acl_init); diff --git a/plugins/acl-plugin/acl/acl.h b/plugins/acl-plugin/acl/acl.h deleted file mode 100644 index afc9b289..00000000 --- a/plugins/acl-plugin/acl/acl.h +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_acl_h -#define included_acl_h - -#include -#include -#include -#include - - -#include -#include -#include - -#define ACL_PLUGIN_VERSION_MAJOR 1 -#define ACL_PLUGIN_VERSION_MINOR 1 - -extern vlib_node_registration_t acl_in_node; -extern vlib_node_registration_t acl_out_node; - -void input_acl_packet_match(u32 sw_if_index, vlib_buffer_t * b0, u32 *nextp, u32 *acl_match_p, u32 *rule_match_p, u32 *trace_bitmap); -void output_acl_packet_match(u32 sw_if_index, vlib_buffer_t * b0, u32 *nextp, u32 *acl_match_p, u32 *rule_match_p, u32 *trace_bitmap); - -enum address_e { IP4, IP6 }; -typedef struct -{ - enum address_e type; - union { - ip6_address_t ip6; - ip4_address_t ip4; - } addr; -} address_t; - -/* - * ACL rules - */ -typedef struct -{ - u8 is_permit; - u8 is_ipv6; - ip46_address_t src; - u8 src_prefixlen; - ip46_address_t dst; - u8 dst_prefixlen; - u8 proto; - u16 src_port_or_type_first; - u16 src_port_or_type_last; - u16 dst_port_or_code_first; - u16 dst_port_or_code_last; - u8 tcp_flags_value; - u8 tcp_flags_mask; -} acl_rule_t; - -typedef struct -{ - u8 is_permit; - u8 is_ipv6; - u8 src_mac[6]; - u8 src_mac_mask[6]; - ip46_address_t src_ip_addr; - u8 src_prefixlen; -} macip_acl_rule_t; - -/* - * ACL - */ -typedef struct -{ - u8 tag[64]; - u32 count; - acl_rule_t *rules; -} acl_list_t; - -typedef struct -{ - u8 tag[64]; - u32 count; - macip_acl_rule_t *rules; - /* References to the classifier tables that will enforce the rules */ - u32 ip4_table_index; - u32 ip6_table_index; - u32 l2_table_index; -} macip_acl_list_t; - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - - acl_list_t *acls; /* Pool of ACLs */ - macip_acl_list_t *macip_acls; /* Pool of MAC-IP ACLs */ - - /* ACLs associated with interfaces */ - u32 **input_acl_vec_by_sw_if_index; - u32 **output_acl_vec_by_sw_if_index; - - /* - * Classify tables used to grab the packets for the ACL check, - * and serving as the 5-tuple session tables at the same time - */ - u32 *acl_ip4_input_classify_table_by_sw_if_index; - u32 *acl_ip6_input_classify_table_by_sw_if_index; - u32 *acl_ip4_output_classify_table_by_sw_if_index; - u32 *acl_ip6_output_classify_table_by_sw_if_index; - - /* MACIP (input) ACLs associated with the interfaces */ - u32 *macip_acl_by_sw_if_index; - - /* next indices for our nodes in the l2-classify tables */ - u32 l2_input_classify_next_acl; - u32 l2_output_classify_next_acl; - - /* next node indices for feature bitmap */ - u32 acl_in_node_input_next_node_index[32]; - /* the respective thing for the output feature */ - l2_output_next_nodes_st acl_out_output_next_nodes; - - /* ACL match actions (must be coherent across in/out ACLs to next indices (can differ) */ - - u32 acl_in_ip4_match_next[256]; - u32 acl_in_ip6_match_next[256]; - u32 acl_out_ip4_match_next[256]; - u32 acl_out_ip6_match_next[256]; - u32 n_match_actions; - - - /* convenience */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; - ethernet_main_t * ethernet_main; -} acl_main_t; - -extern acl_main_t acl_main; - - -#endif diff --git a/plugins/acl-plugin/acl/acl_all_api_h.h b/plugins/acl-plugin/acl/acl_all_api_h.h deleted file mode 100644 index 96eca56d..00000000 --- a/plugins/acl-plugin/acl/acl_all_api_h.h +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include - -#ifdef vl_printfun - -#ifdef LP64 -#define _uword_fmt "%lld" -#define _uword_cast (long long) -#else -#define _uword_fmt "%ld" -#define _uword_cast long -#endif - -static inline void * -vl_api_acl_rule_t_print (vl_api_acl_rule_t * a, void *handle) -{ - vl_print (handle, "vl_api_acl_rule_t:\n"); - vl_print (handle, "is_permit: %u\n", (unsigned) a->is_permit); - vl_print (handle, "is_ipv6: %u\n", (unsigned) a->is_ipv6); - { - int _i; - for (_i = 0; _i < 16; _i++) - { - vl_print (handle, "src_ip_addr[%d]: %u\n", _i, a->src_ip_addr[_i]); - } - } - vl_print (handle, "src_ip_prefix_len: %u\n", - (unsigned) a->src_ip_prefix_len); - { - int _i; - for (_i = 0; _i < 16; _i++) - { - vl_print (handle, "dst_ip_addr[%d]: %u\n", _i, a->dst_ip_addr[_i]); - } - } - vl_print (handle, "dst_ip_prefix_len: %u\n", - (unsigned) a->dst_ip_prefix_len); - vl_print (handle, "proto: %u\n", (unsigned) a->proto); - vl_print (handle, "srcport_or_icmptype_first: %u\n", - (unsigned) a->srcport_or_icmptype_first); - vl_print (handle, "srcport_or_icmptype_last: %u\n", - (unsigned) a->srcport_or_icmptype_last); - vl_print (handle, "dstport_or_icmpcode_first: %u\n", - (unsigned) a->dstport_or_icmpcode_first); - vl_print (handle, "dstport_or_icmpcode_last: %u\n", - (unsigned) a->dstport_or_icmpcode_last); - vl_print (handle, "tcp_flags_mask: %u\n", (unsigned) a->tcp_flags_mask); - vl_print (handle, "tcp_flags_value: %u\n", (unsigned) a->tcp_flags_value); - return handle; -} - -static inline void * -vl_api_acl_add_replace_t_print (vl_api_acl_add_replace_t * a, void *handle) -{ - int i; - vl_print (handle, "vl_api_acl_add_replace_t:\n"); - vl_print (handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); - vl_print (handle, "client_index: %u\n", (unsigned) a->client_index); - vl_print (handle, "context: %u\n", (unsigned) a->context); - vl_print (handle, "acl_index: %u\n", (unsigned) a->acl_index); - vl_print (handle, "count: %u\n", (unsigned) a->count); - vl_print (handle, "r ----- \n"); - for (i = 0; i < a->count; i++) - { - vl_print (handle, " r[%d]:\n", i); - vl_api_acl_rule_t_print (&a->r[i], handle); - } - vl_print (handle, "r ----- END \n"); - return handle; -} - - -static inline void *vl_api_acl_details_t_print (vl_api_acl_details_t *a,void *handle) -{ - vl_print(handle, "vl_api_acl_details_t:\n"); - vl_print(handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); - vl_print(handle, "context: %u\n", (unsigned) a->context); - vl_print(handle, "acl_index: %u\n", (unsigned) a->acl_index); - { - int _i; - for (_i = 0; _i < 64; _i++) { - vl_print(handle, "tag[%d]: %u\n", _i, a->tag[_i]); - } - } - vl_print(handle, "count: %u\n", (unsigned) a->count); - vl_print(handle, "r ----- \n"); - // FIXME vl_api_acl_rule_t_print(&a->r, handle); - vl_print(handle, "r ----- END \n"); - return handle; -} - -static inline void * -vl_api_macip_acl_rule_t_print (vl_api_macip_acl_rule_t * a, void *handle) -{ - vl_print (handle, "vl_api_macip_acl_rule_t:\n"); - vl_print (handle, "is_permit: %u\n", (unsigned) a->is_permit); - vl_print (handle, "is_ipv6: %u\n", (unsigned) a->is_ipv6); - { - int _i; - for (_i = 0; _i < 6; _i++) - { - vl_print (handle, "src_mac[%d]: %u\n", _i, a->src_mac[_i]); - } - } - { - int _i; - for (_i = 0; _i < 6; _i++) - { - vl_print (handle, "src_mac_mask[%d]: %u\n", _i, a->src_mac_mask[_i]); - } - } - { - int _i; - for (_i = 0; _i < 16; _i++) - { - vl_print (handle, "src_ip_addr[%d]: %u\n", _i, a->src_ip_addr[_i]); - } - } - vl_print (handle, "src_ip_prefix_len: %u\n", - (unsigned) a->src_ip_prefix_len); - return handle; -} - -static inline void * -vl_api_macip_acl_add_t_print (vl_api_macip_acl_add_t * a, void *handle) -{ - int i; - vl_print (handle, "vl_api_macip_acl_add_t:\n"); - vl_print (handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); - vl_print (handle, "client_index: %u\n", (unsigned) a->client_index); - vl_print (handle, "context: %u\n", (unsigned) a->context); - vl_print (handle, "count: %u\n", (unsigned) a->count); - vl_print (handle, "r ----- \n"); - for (i = 0; i < a->count; i++) - { - vl_print (handle, " r[%d]:\n", i); - vl_api_macip_acl_rule_t_print (&a->r[i], handle); - } - vl_print (handle, "r ----- END \n"); - return handle; -} - -static inline void *vl_api_macip_acl_details_t_print (vl_api_macip_acl_details_t *a,void *handle) -{ - int i; - vl_print(handle, "vl_api_macip_acl_details_t:\n"); - vl_print(handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); - vl_print(handle, "context: %u\n", (unsigned) a->context); - vl_print(handle, "acl_index: %u\n", (unsigned) a->acl_index); - { - int _i; - for (_i = 0; _i < 64; _i++) { - vl_print(handle, "tag[%d]: %u\n", _i, a->tag[_i]); - } - } - vl_print(handle, "count: %u\n", (unsigned) a->count); - vl_print(handle, "r ----- \n"); - for (i = 0; i < a->count; i++) - { - vl_print (handle, " r[%d]:\n", i); - vl_api_macip_acl_rule_t_print (&a->r[i], handle); - } - vl_print(handle, "r ----- END \n"); - return handle; -} - -#endif /* vl_printfun */ - - -#ifdef vl_endianfun - -#undef clib_net_to_host_uword -#ifdef LP64 -#define clib_net_to_host_uword clib_net_to_host_u64 -#else -#define clib_net_to_host_uword clib_net_to_host_u32 -#endif - -/* - * Manual endian/print functions created by copypasting the automatically - * generated ones with small required adjustments. Appears the codegen - * can't make code to print the contents of custom-type array. - */ - -static inline void -vl_api_acl_rule_t_endian (vl_api_acl_rule_t * a) -{ - /* a->is_permit = a->is_permit (no-op) */ - /* a->is_ipv6 = a->is_ipv6 (no-op) */ - /* a->src_ip_addr[0..15] = a->src_ip_addr[0..15] (no-op) */ - /* a->src_ip_prefix_len = a->src_ip_prefix_len (no-op) */ - /* a->dst_ip_addr[0..15] = a->dst_ip_addr[0..15] (no-op) */ - /* a->dst_ip_prefix_len = a->dst_ip_prefix_len (no-op) */ - /* a->proto = a->proto (no-op) */ - a->srcport_or_icmptype_first = - clib_net_to_host_u16 (a->srcport_or_icmptype_first); - a->srcport_or_icmptype_last = - clib_net_to_host_u16 (a->srcport_or_icmptype_last); - a->dstport_or_icmpcode_first = - clib_net_to_host_u16 (a->dstport_or_icmpcode_first); - a->dstport_or_icmpcode_last = - clib_net_to_host_u16 (a->dstport_or_icmpcode_last); - /* a->tcp_flags_mask = a->tcp_flags_mask (no-op) */ - /* a->tcp_flags_value = a->tcp_flags_value (no-op) */ -} - -static inline void -vl_api_acl_add_replace_t_endian (vl_api_acl_add_replace_t * a) -{ - int i; - a->_vl_msg_id = clib_net_to_host_u16 (a->_vl_msg_id); - a->client_index = clib_net_to_host_u32 (a->client_index); - a->context = clib_net_to_host_u32 (a->context); - a->acl_index = clib_net_to_host_u32 (a->acl_index); - a->count = clib_net_to_host_u32 (a->count); - for (i = 0; i < a->count; i++) - { - vl_api_acl_rule_t_endian (&a->r[i]); - } -} - -static inline void vl_api_acl_details_t_endian (vl_api_acl_details_t *a) -{ - int i; - a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); - a->context = clib_net_to_host_u32(a->context); - a->acl_index = clib_net_to_host_u32(a->acl_index); - /* a->tag[0..63] = a->tag[0..63] (no-op) */ - a->count = clib_net_to_host_u32(a->count); - for (i = 0; i < a->count; i++) - { - vl_api_acl_rule_t_endian (&a->r[i]); - } -} - -static inline void vl_api_acl_interface_list_details_t_endian (vl_api_acl_interface_list_details_t *a) -{ - int i; - a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); - a->context = clib_net_to_host_u32(a->context); - a->sw_if_index = clib_net_to_host_u32(a->sw_if_index); - /* a->count = a->count (no-op) */ - /* a->n_input = a->n_input (no-op) */ - for(i=0; icount; i++) { - a->acls[i] = clib_net_to_host_u32(a->acls[i]); - } -} - -static inline void vl_api_acl_interface_set_acl_list_t_endian (vl_api_acl_interface_set_acl_list_t *a) -{ - int i; - a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); - a->client_index = clib_net_to_host_u32(a->client_index); - a->context = clib_net_to_host_u32(a->context); - a->sw_if_index = clib_net_to_host_u32(a->sw_if_index); - /* a->count = a->count (no-op) */ - /* a->n_input = a->n_input (no-op) */ - for(i=0; icount; i++) { - a->acls[i] = clib_net_to_host_u32(a->acls[i]); - } -} - -static inline void -vl_api_macip_acl_rule_t_endian (vl_api_macip_acl_rule_t * a) -{ - /* a->is_permit = a->is_permit (no-op) */ - /* a->is_ipv6 = a->is_ipv6 (no-op) */ - /* a->src_mac[0..5] = a->src_mac[0..5] (no-op) */ - /* a->src_mac_mask[0..5] = a->src_mac_mask[0..5] (no-op) */ - /* a->src_ip_addr[0..15] = a->src_ip_addr[0..15] (no-op) */ - /* a->src_ip_prefix_len = a->src_ip_prefix_len (no-op) */ -} - -static inline void -vl_api_macip_acl_add_t_endian (vl_api_macip_acl_add_t * a) -{ - int i; - a->_vl_msg_id = clib_net_to_host_u16 (a->_vl_msg_id); - a->client_index = clib_net_to_host_u32 (a->client_index); - a->context = clib_net_to_host_u32 (a->context); - a->count = clib_net_to_host_u32 (a->count); - for (i = 0; i < a->count; i++) - { - vl_api_macip_acl_rule_t_endian (&a->r[i]); - } -} - -static inline void vl_api_macip_acl_details_t_endian (vl_api_macip_acl_details_t *a) -{ - int i; - a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); - a->context = clib_net_to_host_u32(a->context); - a->acl_index = clib_net_to_host_u32(a->acl_index); - /* a->tag[0..63] = a->tag[0..63] (no-op) */ - a->count = clib_net_to_host_u32(a->count); - for (i = 0; i < a->count; i++) - { - vl_api_macip_acl_rule_t_endian (&a->r[i]); - } -} - - - - -#endif /* vl_printfun */ - - diff --git a/plugins/acl-plugin/acl/acl_msg_enum.h b/plugins/acl-plugin/acl/acl_msg_enum.h deleted file mode 100644 index 14d8b48c..00000000 --- a/plugins/acl-plugin/acl/acl_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_acl_msg_enum_h -#define included_acl_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif diff --git a/plugins/acl-plugin/acl/acl_test.c b/plugins/acl-plugin/acl/acl_test.c deleted file mode 100644 index a0e413e1..00000000 --- a/plugins/acl-plugin/acl/acl_test.c +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * acl_test.c - test harness plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include -#include -#include - -uword unformat_sw_if_index (unformat_input_t * input, va_list * args); - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define message structures */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} acl_test_main_t; - -acl_test_main_t acl_test_main; - -#define foreach_standard_reply_retval_handler \ -_(acl_del_reply) \ -_(acl_interface_add_del_reply) \ -_(macip_acl_interface_add_del_reply) \ -_(acl_interface_set_acl_list_reply) \ -_(macip_acl_del_reply) - -#define foreach_reply_retval_aclindex_handler \ -_(acl_add_replace_reply) \ -_(macip_acl_add_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = acl_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = acl_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - clib_warning("ACL index: %d", ntohl(mp->acl_index)); \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_reply_retval_aclindex_handler; -#undef _ - -/* These two ought to be in a library somewhere but they aren't */ -static uword -my_unformat_mac_address (unformat_input_t * input, va_list * args) -{ - u8 *a = va_arg (*args, u8 *); - return unformat (input, "%x:%x:%x:%x:%x:%x", &a[0], &a[1], &a[2], &a[3], - &a[4], &a[5]); -} - -static u8 * -my_format_mac_address (u8 * s, va_list * args) -{ - u8 *a = va_arg (*args, u8 *); - return format (s, "%02x:%02x:%02x:%02x:%02x:%02x", - a[0], a[1], a[2], a[3], a[4], a[5]); -} - - - -static void vl_api_acl_plugin_get_version_reply_t_handler - (vl_api_acl_plugin_get_version_reply_t * mp) - { - vat_main_t * vam = acl_test_main.vat_main; - clib_warning("ACL plugin version: %d.%d", ntohl(mp->major), ntohl(mp->minor)); - vam->result_ready = 1; - } - -static void vl_api_acl_interface_list_details_t_handler - (vl_api_acl_interface_list_details_t * mp) - { - int i; - vat_main_t * vam = acl_test_main.vat_main; - u8 *out = 0; - vl_api_acl_interface_list_details_t_endian(mp); - out = format(out, "sw_if_index: %d, count: %d, n_input: %d\n", mp->sw_if_index, mp->count, mp->n_input); - out = format(out, " input "); - for(i=0; icount; i++) { - out = format(out, "%d ", mp->acls[i]); - if (i == mp->n_input-1) - out = format(out, "\n output "); - } - out = format(out, "\n"); - clib_warning("%s", out); - vec_free(out); - vam->result_ready = 1; - } - - -static inline u8 * -vl_api_acl_rule_t_pretty_format (u8 *out, vl_api_acl_rule_t * a) -{ - int af = a->is_ipv6 ? AF_INET6 : AF_INET; - u8 src[INET6_ADDRSTRLEN]; - u8 dst[INET6_ADDRSTRLEN]; - inet_ntop(af, a->src_ip_addr, (void *)src, sizeof(src)); - inet_ntop(af, a->dst_ip_addr, (void *)dst, sizeof(dst)); - - out = format(out, "%s action %d src %s/%d dst %s/%d proto %d sport %d-%d dport %d-%d tcpflags %d %d", - a->is_ipv6 ? "ipv6" : "ipv4", a->is_permit, - src, a->src_ip_prefix_len, - dst, a->dst_ip_prefix_len, - a->proto, - a->srcport_or_icmptype_first, a->srcport_or_icmptype_last, - a->dstport_or_icmpcode_first, a->dstport_or_icmpcode_last, - a->tcp_flags_mask, a->tcp_flags_value); - return(out); -} - - - -static void vl_api_acl_details_t_handler - (vl_api_acl_details_t * mp) - { - int i; - vat_main_t * vam = acl_test_main.vat_main; - vl_api_acl_details_t_endian(mp); - u8 *out = 0; - out = format(0, "acl_index: %d, count: %d\n tag {%s}\n", mp->acl_index, mp->count, mp->tag); - for(i=0; icount; i++) { - out = format(out, " "); - out = vl_api_acl_rule_t_pretty_format(out, &mp->r[i]); - out = format(out, "%s\n", icount-1 ? "," : ""); - } - clib_warning("%s", out); - vec_free(out); - vam->result_ready = 1; - } - -static inline u8 * -vl_api_macip_acl_rule_t_pretty_format (u8 *out, vl_api_macip_acl_rule_t * a) -{ - int af = a->is_ipv6 ? AF_INET6 : AF_INET; - u8 src[INET6_ADDRSTRLEN]; - inet_ntop(af, a->src_ip_addr, (void *)src, sizeof(src)); - - out = format(out, "%s action %d ip %s/%d mac %U mask %U", - a->is_ipv6 ? "ipv6" : "ipv4", a->is_permit, - src, a->src_ip_prefix_len, - my_format_mac_address, a->src_mac, - my_format_mac_address, a->src_mac_mask); - return(out); -} - - -static void vl_api_macip_acl_details_t_handler - (vl_api_macip_acl_details_t * mp) - { - int i; - vat_main_t * vam = acl_test_main.vat_main; - vl_api_macip_acl_details_t_endian(mp); - u8 *out = format(0,"MACIP acl_index: %d, count: %d\n tag {%s}\n", mp->acl_index, mp->count, mp->tag); - for(i=0; icount; i++) { - out = format(out, " "); - out = vl_api_macip_acl_rule_t_pretty_format(out, &mp->r[i]); - out = format(out, "%s\n", icount-1 ? "," : ""); - } - clib_warning("%s", out); - vec_free(out); - vam->result_ready = 1; - } - -static void vl_api_macip_acl_interface_get_reply_t_handler - (vl_api_macip_acl_interface_get_reply_t * mp) - { - int i; - vat_main_t * vam = acl_test_main.vat_main; - u8 *out = format(0, "sw_if_index with MACIP ACL count: %d\n", ntohl(mp->count)); - for(i=0; icount); i++) { - out = format(out, " macip_acl_interface_add_del sw_if_index %d add acl %d\n", i, ntohl(mp->acls[i])); - } - out = format(out, "\n"); - clib_warning("%s", out); - vec_free(out); - vam->result_ready = 1; - } - - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(ACL_ADD_REPLACE_REPLY, acl_add_replace_reply) \ -_(ACL_DEL_REPLY, acl_del_reply) \ -_(ACL_INTERFACE_ADD_DEL_REPLY, acl_interface_add_del_reply) \ -_(ACL_INTERFACE_SET_ACL_LIST_REPLY, acl_interface_set_acl_list_reply) \ -_(ACL_INTERFACE_LIST_DETAILS, acl_interface_list_details) \ -_(ACL_DETAILS, acl_details) \ -_(MACIP_ACL_ADD_REPLY, macip_acl_add_reply) \ -_(MACIP_ACL_DEL_REPLY, macip_acl_del_reply) \ -_(MACIP_ACL_DETAILS, macip_acl_details) \ -_(MACIP_ACL_INTERFACE_ADD_DEL_REPLY, macip_acl_interface_add_del_reply) \ -_(MACIP_ACL_INTERFACE_GET_REPLY, macip_acl_interface_get_reply) \ -_(ACL_PLUGIN_GET_VERSION_REPLY, acl_plugin_get_version_reply) - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - -static int api_acl_plugin_get_version (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - vl_api_acl_plugin_get_version_t * mp; - u32 msg_size = sizeof(*mp); - f64 timeout; - - vam->result_ready = 0; - mp = vl_msg_api_alloc_as_if_client(msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = ntohs (VL_API_ACL_PLUGIN_GET_VERSION + sm->msg_id_base); - mp->client_index = vam->my_client_index; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; - - return 0; -} - -static int api_macip_acl_interface_get (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - vl_api_acl_plugin_get_version_t * mp; - u32 msg_size = sizeof(*mp); - f64 timeout; - - vam->result_ready = 0; - mp = vl_msg_api_alloc_as_if_client(msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = ntohs (VL_API_MACIP_ACL_INTERFACE_GET + sm->msg_id_base); - mp->client_index = vam->my_client_index; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; - - return 0; -} - -#define vec_validate_acl_rules(v, idx) \ - do { \ - if (vec_len(v) < idx+1) { \ - vec_validate(v, idx); \ - v[idx].is_permit = 0x1; \ - v[idx].srcport_or_icmptype_last = 0xffff; \ - v[idx].dstport_or_icmpcode_last = 0xffff; \ - } \ - } while (0) - - -static int api_acl_add_replace (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_acl_add_replace_t * mp; - u32 acl_index = ~0; - u32 msg_size = sizeof (*mp); /* without the rules */ - - vl_api_acl_rule_t *rules = 0; - int rule_idx = 0; - int n_rules = 0; - u32 proto = 0; - u32 port1 = 0; - u32 port2 = 0; - u32 action = 0; - u32 tcpflags, tcpmask; - u32 src_prefix_length = 0, dst_prefix_length = 0; - ip4_address_t src_v4address, dst_v4address; - ip6_address_t src_v6address, dst_v6address; - u8 *tag = 0; - - if (!unformat (i, "%d", &acl_index)) { - /* Just assume -1 */ - } - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "ipv6")) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].is_ipv6 = 1; - } - else if (unformat (i, "ipv4")) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].is_ipv6 = 0; - } - else if (unformat (i, "permit+reflect")) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].is_permit = 2; - } - else if (unformat (i, "permit")) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].is_permit = 1; - } - else if (unformat (i, "action %d", &action)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].is_permit = action; - } - else if (unformat (i, "src %U/%d", - unformat_ip4_address, &src_v4address, &src_prefix_length)) - { - vec_validate_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].src_ip_addr, &src_v4address, 4); - rules[rule_idx].src_ip_prefix_len = src_prefix_length; - rules[rule_idx].is_ipv6 = 0; - } - else if (unformat (i, "src %U/%d", - unformat_ip6_address, &src_v6address, &src_prefix_length)) - { - vec_validate_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].src_ip_addr, &src_v6address, 16); - rules[rule_idx].src_ip_prefix_len = src_prefix_length; - rules[rule_idx].is_ipv6 = 1; - } - else if (unformat (i, "dst %U/%d", - unformat_ip4_address, &dst_v4address, &dst_prefix_length)) - { - vec_validate_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].dst_ip_addr, &dst_v4address, 4); - rules[rule_idx].dst_ip_prefix_len = dst_prefix_length; - rules[rule_idx].is_ipv6 = 0; - } - else if (unformat (i, "dst %U/%d", - unformat_ip6_address, &dst_v6address, &dst_prefix_length)) - { - vec_validate_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].dst_ip_addr, &dst_v6address, 16); - rules[rule_idx].dst_ip_prefix_len = dst_prefix_length; - rules[rule_idx].is_ipv6 = 1; - } - else if (unformat (i, "sport %d-%d", &port1, &port2)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].srcport_or_icmptype_first = htons(port1); - rules[rule_idx].srcport_or_icmptype_last = htons(port2); - } - else if (unformat (i, "sport %d", &port1)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].srcport_or_icmptype_first = htons(port1); - rules[rule_idx].srcport_or_icmptype_last = htons(port1); - } - else if (unformat (i, "dport %d-%d", &port1, &port2)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].dstport_or_icmpcode_first = htons(port1); - rules[rule_idx].dstport_or_icmpcode_last = htons(port2); - } - else if (unformat (i, "dport %d", &port1)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].dstport_or_icmpcode_first = htons(port1); - rules[rule_idx].dstport_or_icmpcode_last = htons(port1); - } - else if (unformat (i, "tcpflags %d %d", &tcpflags, &tcpmask)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].tcp_flags_value = tcpflags; - rules[rule_idx].tcp_flags_mask = tcpmask; - } - else if (unformat (i, "proto %d", &proto)) - { - vec_validate_acl_rules(rules, rule_idx); - rules[rule_idx].proto = proto; - } - else if (unformat (i, "tag %s", &tag)) - { - } - else if (unformat (i, ",")) - { - rule_idx++; - vec_validate_acl_rules(rules, rule_idx); - } - else - break; - } - - /* Construct the API message */ - vam->result_ready = 0; - - if(rules) - n_rules = vec_len(rules); - else - n_rules = 0; - - msg_size += n_rules*sizeof(rules[0]); - - mp = vl_msg_api_alloc_as_if_client(msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = ntohs (VL_API_ACL_ADD_REPLACE + sm->msg_id_base); - mp->client_index = vam->my_client_index; - if (n_rules > 0) - clib_memcpy(mp->r, rules, n_rules*sizeof (vl_api_acl_rule_t)); - if (tag) - { - if (vec_len(tag) >= sizeof(mp->tag)) - { - tag[sizeof(mp->tag)-1] = 0; - _vec_len(tag) = sizeof(mp->tag); - } - clib_memcpy(mp->tag, tag, vec_len(tag)); - vec_free(tag); - } - mp->acl_index = ntohl(acl_index); - mp->count = htonl(n_rules); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_acl_del (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_acl_del_t * mp; - u32 acl_index = ~0; - - if (!unformat (i, "%d", &acl_index)) { - errmsg ("missing acl index\n"); - return -99; - } - - /* Construct the API message */ - M(ACL_DEL, acl_del); - mp->acl_index = ntohl(acl_index); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_macip_acl_del (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_acl_del_t * mp; - u32 acl_index = ~0; - - if (!unformat (i, "%d", &acl_index)) { - errmsg ("missing acl index\n"); - return -99; - } - - /* Construct the API message */ - M(MACIP_ACL_DEL, acl_del); - mp->acl_index = ntohl(acl_index); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_acl_interface_add_del (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_acl_interface_add_del_t * mp; - u32 sw_if_index = ~0; - u32 acl_index = ~0; - u8 is_input = 0; - u8 is_add = 0; - -// acl_interface_add_del | sw_if_index acl_index [out] [del] - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "%d", &acl_index)) - ; - else - break; - } - - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) - ; - else if (unformat (i, "sw_if_index %d", &sw_if_index)) - ; - else if (unformat (i, "add")) - is_add = 1; - else if (unformat (i, "del")) - is_add = 0; - else if (unformat (i, "acl %d", &acl_index)) - ; - else if (unformat (i, "input")) - is_input = 1; - else if (unformat (i, "output")) - is_input = 0; - else - break; - } - - if (sw_if_index == ~0) { - errmsg ("missing interface name / explicit sw_if_index number \n"); - return -99; - } - - if (acl_index == ~0) { - errmsg ("missing ACL index\n"); - return -99; - } - - - - /* Construct the API message */ - M(ACL_INTERFACE_ADD_DEL, acl_interface_add_del); - mp->acl_index = ntohl(acl_index); - mp->sw_if_index = ntohl(sw_if_index); - mp->is_add = is_add; - mp->is_input = is_input; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_macip_acl_interface_add_del (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_macip_acl_interface_add_del_t * mp; - u32 sw_if_index = ~0; - u32 acl_index = ~0; - u8 is_add = 0; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) - ; - else if (unformat (i, "sw_if_index %d", &sw_if_index)) - ; - else if (unformat (i, "add")) - is_add = 1; - else if (unformat (i, "del")) - is_add = 0; - else if (unformat (i, "acl %d", &acl_index)) - ; - else - break; - } - - if (sw_if_index == ~0) { - errmsg ("missing interface name / explicit sw_if_index number \n"); - return -99; - } - - if (acl_index == ~0) { - errmsg ("missing ACL index\n"); - return -99; - } - - - - /* Construct the API message */ - M(MACIP_ACL_INTERFACE_ADD_DEL, macip_acl_interface_add_del); - mp->acl_index = ntohl(acl_index); - mp->sw_if_index = ntohl(sw_if_index); - mp->is_add = is_add; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_acl_interface_set_acl_list (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_acl_interface_set_acl_list_t * mp; - u32 sw_if_index = ~0; - u32 acl_index = ~0; - u32 *inacls = 0; - u32 *outacls = 0; - u8 is_input = 0; - -// acl_interface_set_acl_list | sw_if_index input [acl-idx list] output [acl-idx list] - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) - ; - else if (unformat (i, "sw_if_index %d", &sw_if_index)) - ; - else if (unformat (i, "%d", &acl_index)) - { - if(is_input) - vec_add1(inacls, htonl(acl_index)); - else - vec_add1(outacls, htonl(acl_index)); - } - else if (unformat (i, "acl %d", &acl_index)) - ; - else if (unformat (i, "input")) - is_input = 1; - else if (unformat (i, "output")) - is_input = 0; - else - break; - } - - if (sw_if_index == ~0) { - errmsg ("missing interface name / explicit sw_if_index number \n"); - return -99; - } - - /* Construct the API message */ - M2(ACL_INTERFACE_SET_ACL_LIST, acl_interface_set_acl_list, sizeof(u32) * (vec_len(inacls) + vec_len(outacls))); - mp->sw_if_index = ntohl(sw_if_index); - mp->n_input = vec_len(inacls); - mp->count = vec_len(inacls) + vec_len(outacls); - vec_append(inacls, outacls); - if (vec_len(inacls) > 0) - clib_memcpy(mp->acls, inacls, vec_len(inacls)*sizeof(u32)); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - - -static int api_acl_interface_list_dump (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - u32 sw_if_index = ~0; - vl_api_acl_interface_list_dump_t * mp; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) - ; - else if (unformat (i, "sw_if_index %d", &sw_if_index)) - ; - else - break; - } - - /* Construct the API message */ - M(ACL_INTERFACE_LIST_DUMP, acl_interface_list_dump); - mp->sw_if_index = ntohl (sw_if_index); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_acl_dump (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - u32 acl_index = ~0; - vl_api_acl_dump_t * mp; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%d", &acl_index)) - ; - else - break; - } - - /* Construct the API message */ - M(ACL_DUMP, acl_dump); - mp->acl_index = ntohl (acl_index); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -static int api_macip_acl_dump (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - u32 acl_index = ~0; - vl_api_acl_dump_t * mp; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%d", &acl_index)) - ; - else - break; - } - - /* Construct the API message */ - M(MACIP_ACL_DUMP, macip_acl_dump); - mp->acl_index = ntohl (acl_index); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -#define vec_validate_macip_acl_rules(v, idx) \ - do { \ - if (vec_len(v) < idx+1) { \ - vec_validate(v, idx); \ - v[idx].is_permit = 0x1; \ - } \ - } while (0) - - -static int api_macip_acl_add (vat_main_t * vam) -{ - acl_test_main_t * sm = &acl_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_macip_acl_add_t * mp; - u32 msg_size = sizeof (*mp); /* without the rules */ - - vl_api_macip_acl_rule_t *rules = 0; - int rule_idx = 0; - int n_rules = 0; - u32 src_prefix_length = 0; - u32 action = 0; - ip4_address_t src_v4address; - ip6_address_t src_v6address; - u8 src_mac[6]; - u8 *tag = 0; - u8 mac_mask_all_1[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "ipv6")) - { - vec_validate_macip_acl_rules(rules, rule_idx); - rules[rule_idx].is_ipv6 = 1; - } - else if (unformat (i, "ipv4")) - { - vec_validate_macip_acl_rules(rules, rule_idx); - rules[rule_idx].is_ipv6 = 1; - } - else if (unformat (i, "permit")) - { - vec_validate_macip_acl_rules(rules, rule_idx); - rules[rule_idx].is_permit = 1; - } - else if (unformat (i, "deny")) - { - vec_validate_macip_acl_rules(rules, rule_idx); - rules[rule_idx].is_permit = 0; - } - else if (unformat (i, "action %d", &action)) - { - vec_validate_macip_acl_rules(rules, rule_idx); - rules[rule_idx].is_permit = action; - } - else if (unformat (i, "ip %U/%d", - unformat_ip4_address, &src_v4address, &src_prefix_length)) - { - vec_validate_macip_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].src_ip_addr, &src_v4address, 4); - rules[rule_idx].src_ip_prefix_len = src_prefix_length; - rules[rule_idx].is_ipv6 = 0; - } - else if (unformat (i, "ip %U/%d", - unformat_ip6_address, &src_v6address, &src_prefix_length)) - { - vec_validate_macip_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].src_ip_addr, &src_v6address, 16); - rules[rule_idx].src_ip_prefix_len = src_prefix_length; - rules[rule_idx].is_ipv6 = 1; - } - else if (unformat (i, "mac %U", - my_unformat_mac_address, &src_mac)) - { - vec_validate_macip_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].src_mac, &src_mac, 6); - memcpy (rules[rule_idx].src_mac_mask, &mac_mask_all_1, 6); - } - else if (unformat (i, "mask %U", - my_unformat_mac_address, &src_mac)) - { - vec_validate_macip_acl_rules(rules, rule_idx); - memcpy (rules[rule_idx].src_mac_mask, &src_mac, 6); - } - else if (unformat (i, "tag %s", &tag)) - { - } - else if (unformat (i, ",")) - { - rule_idx++; - vec_validate_macip_acl_rules(rules, rule_idx); - } - else - break; - } - - /* Construct the API message */ - vam->result_ready = 0; - - if(rules) - n_rules = vec_len(rules); - else - n_rules = 0; - - msg_size += n_rules*sizeof(rules[0]); - - mp = vl_msg_api_alloc_as_if_client(msg_size); - memset (mp, 0, msg_size); - mp->_vl_msg_id = ntohs (VL_API_MACIP_ACL_ADD + sm->msg_id_base); - mp->client_index = vam->my_client_index; - if (n_rules > 0) - clib_memcpy(mp->r, rules, n_rules*sizeof (mp->r[0])); - if (tag) - { - if (vec_len(tag) >= sizeof(mp->tag)) - { - tag[sizeof(mp->tag)-1] = 0; - _vec_len(tag) = sizeof(mp->tag); - } - clib_memcpy(mp->tag, tag, vec_len(tag)); - vec_free(tag); - } - - mp->count = htonl(n_rules); - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(acl_plugin_get_version, "") \ -_(acl_add_replace, " [ [src IP/plen] [dst IP/plen] [sport X-Y] [dport X-Y] [proto P] [tcpflags FL MASK], ... , ...") \ -_(acl_del, "") \ -_(acl_dump, "[]") \ -_(acl_interface_add_del, " | sw_if_index [add|del] [input|output] acl ") \ -_(acl_interface_set_acl_list, " | sw_if_index input [acl-idx list] output [acl-idx list]") \ -_(acl_interface_list_dump, "[ | sw_if_index ]") \ -_(macip_acl_add, "...") \ -_(macip_acl_del, "")\ -_(macip_acl_dump, "[]") \ -_(macip_acl_interface_add_del, " | sw_if_index [add|del] acl ") \ -_(macip_acl_interface_get, "") - - - -void vat_api_hookup (vat_main_t *vam) -{ - acl_test_main_t * sm = &acl_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * vat_plugin_register (vat_main_t *vam) -{ - acl_test_main_t * sm = &acl_test_main; - u8 * name; - - sm->vat_main = vam; - - name = format (0, "acl_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~0) - vat_api_hookup (vam); - - vec_free(name); - - return 0; -} diff --git a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java b/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java deleted file mode 100644 index 979edbc4..00000000 --- a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.acl.test; - - -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_ADDRESS_2_AS_ARRAY; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_ADDRESS_AS_ARRAY; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_DST_ICMP_TYPE_END; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_DST_ICMP_TYPE_START; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_MAC; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_MAC_MASK; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_PREFIX; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_PREFIX_2; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_SRC_ICMP_TYPE_END; -import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_SRC_ICMP_TYPE_START; -import static io.fd.vpp.jvpp.acl.test.AclTestData.ICMP_PROTOCOL; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_ADDRESS_2_AS_ARRAY; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_ADDRESS_AS_ARRAY; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_DST_PORT_RANGE_END; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_DST_PORT_RANGE_START; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_MAC; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_MAC_MASK; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_PREFIX; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_PREFIX_2; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_SRC_PORT_RANGE_END; -import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_SRC_PORT_RANGE_START; -import static io.fd.vpp.jvpp.acl.test.AclTestData.UDP_PROTOCOL; - -import io.fd.vpp.jvpp.acl.dto.AclDetails; -import io.fd.vpp.jvpp.acl.dto.AclInterfaceListDetails; -import io.fd.vpp.jvpp.acl.dto.MacipAclDetails; -import io.fd.vpp.jvpp.acl.types.AclRule; -import io.fd.vpp.jvpp.acl.types.MacipAclRule; -import java.util.Arrays; - -class AclExpectedDumpData { - - static void verifyMacIpDump(final MacipAclDetails macipAclDetails) { - // asserting data create by previous call - assertEquals(0, macipAclDetails.aclIndex); - assertEquals(2, macipAclDetails.count); - - final MacipAclRule currentIpv4Rule = macipAclDetails.r[0]; - final MacipAclRule currentIpv6Rule = macipAclDetails.r[1]; - - // Comparing one property at the time to better pointer if something is wrong - //Ipv4 rule - assertEquals(0, currentIpv4Rule.isIpv6); - assertEquals(1, currentIpv4Rule.isPermit); - - // cutting expected ipv4 to 4 bytes,vpp sends it as 16 always - assertArrays(FIRST_RULE_ADDRESS_AS_ARRAY, Arrays.copyOfRange(currentIpv4Rule.srcIpAddr, 0, 4)); - assertEquals(FIRST_RULE_PREFIX, currentIpv4Rule.srcIpPrefixLen); - assertArrays(FIRST_RULE_MAC, currentIpv4Rule.srcMac); - assertArrays(FIRST_RULE_MAC_MASK, currentIpv4Rule.srcMacMask); - - //Ipv6 rule - assertEquals(1, currentIpv6Rule.isIpv6); - assertEquals(0, currentIpv6Rule.isPermit); - assertArrays(SECOND_RULE_ADDRESS_AS_ARRAY, currentIpv6Rule.srcIpAddr); - assertEquals(SECOND_RULE_PREFIX, currentIpv6Rule.srcIpPrefixLen); - assertArrays(SECOND_RULE_MAC, currentIpv6Rule.srcMac); - assertArrays(SECOND_RULE_MAC_MASK, currentIpv6Rule.srcMacMask); - } - - static void verifyAclDump(final AclDetails aclDetails) { - assertEquals(0, aclDetails.aclIndex); - assertEquals(2, aclDetails.count); - - final AclRule currentIpv4Rule = aclDetails.r[0]; - final AclRule currentIpv6Rule = aclDetails.r[1]; - - // Comparing one property at the time to better pointer if something is wrong - //Ipv4 rule - assertEquals(0, currentIpv4Rule.isIpv6); - assertEquals(1, currentIpv4Rule.isPermit); - - // cutting expected ipv4 to 4 bytes,vpp sends it as 16 always - assertArrays(FIRST_RULE_ADDRESS_AS_ARRAY, Arrays.copyOfRange(currentIpv4Rule.srcIpAddr, 0, 4)); - assertEquals(FIRST_RULE_PREFIX, currentIpv4Rule.srcIpPrefixLen); - assertArrays(FIRST_RULE_ADDRESS_2_AS_ARRAY, Arrays.copyOfRange(currentIpv4Rule.dstIpAddr, 0, 4)); - assertEquals(FIRST_RULE_PREFIX_2, currentIpv4Rule.dstIpPrefixLen); - - assertEquals(ICMP_PROTOCOL, currentIpv4Rule.proto); - assertEquals(FIRST_RULE_SRC_ICMP_TYPE_START, currentIpv4Rule.srcportOrIcmptypeFirst); - assertEquals(FIRST_RULE_SRC_ICMP_TYPE_END, currentIpv4Rule.srcportOrIcmptypeLast); - assertEquals(FIRST_RULE_DST_ICMP_TYPE_START, currentIpv4Rule.dstportOrIcmpcodeFirst); - assertEquals(FIRST_RULE_DST_ICMP_TYPE_END, currentIpv4Rule.dstportOrIcmpcodeLast); - - assertArrays(SECOND_RULE_ADDRESS_AS_ARRAY, currentIpv6Rule.srcIpAddr); - assertEquals(SECOND_RULE_PREFIX, currentIpv6Rule.srcIpPrefixLen); - assertArrays(SECOND_RULE_ADDRESS_2_AS_ARRAY, currentIpv6Rule.dstIpAddr); - assertEquals(SECOND_RULE_PREFIX_2, currentIpv6Rule.dstIpPrefixLen); - - assertEquals(UDP_PROTOCOL, currentIpv6Rule.proto); - assertEquals(SECOND_RULE_SRC_PORT_RANGE_START, currentIpv6Rule.srcportOrIcmptypeFirst); - assertEquals(SECOND_RULE_SRC_PORT_RANGE_END, currentIpv6Rule.srcportOrIcmptypeLast); - assertEquals(SECOND_RULE_DST_PORT_RANGE_START, currentIpv6Rule.dstportOrIcmpcodeFirst); - assertEquals(SECOND_RULE_DST_PORT_RANGE_END, currentIpv6Rule.dstportOrIcmpcodeLast); - } - - static void verifyAclInterfaceList(final AclInterfaceListDetails aclInterfaceListDetails) { - assertEquals(1, aclInterfaceListDetails.count); - assertEquals(1, aclInterfaceListDetails.acls[0]); - assertEquals(0, aclInterfaceListDetails.nInput); - assertEquals(0, aclInterfaceListDetails.swIfIndex); - } - - private static void assertArrays(final byte[] expected, final byte[] actual) { - if (!Arrays.equals(expected, actual)) { - throw new IllegalArgumentException( - String.format("Expected[%s]/Actual[%s]", Arrays.toString(expected), Arrays.toString(actual))); - } - } - - private static void assertEquals(final int expected, final int actual) { - if (expected != actual) { - throw new IllegalArgumentException(String.format("Expected[%s]/Actual[%s]", expected, actual)); - } - } -} diff --git a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestData.java b/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestData.java deleted file mode 100644 index 5d228eea..00000000 --- a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestData.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.acl.test; - - -import io.fd.vpp.jvpp.acl.types.AclRule; -import io.fd.vpp.jvpp.acl.types.MacipAclRule; - -class AclTestData { - - static final byte[] FIRST_RULE_ADDRESS_AS_ARRAY = {-64, -88, 2, 1}; - static final byte[] FIRST_RULE_ADDRESS_2_AS_ARRAY = {-64, -88, 2, 3}; - static final byte[] SECOND_RULE_ADDRESS_AS_ARRAY = - {32, 1, 13, -72, 10, 11, 18, -16, 0, 0, 0, 0, 0, 0, 0, 1}; - static final byte[] SECOND_RULE_ADDRESS_2_AS_ARRAY = - {32, 1, 13, -72, 10, 11, 18, -16, 0, 0, 0, 0, 0, 0, 0, 1}; - static final byte[] FIRST_RULE_MAC = {11, 11, 11, 11, 11, 11}; - static final byte[] FIRST_RULE_MAC_MASK = {0, 0, 0, 0, 0, 0}; - static final byte[] SECOND_RULE_MAC = {11, 12, 11, 11, 12, 11}; - static final byte[] SECOND_RULE_MAC_MASK = {(byte) 170, 0, 0, 0, 0, 0}; - static final int FIRST_RULE_PREFIX = 32; - static final int FIRST_RULE_PREFIX_2 = 24; - static final int SECOND_RULE_PREFIX = 64; - static final int SECOND_RULE_PREFIX_2 = 62; - static final int FIRST_RULE_DST_ICMP_TYPE_START = 0; - static final int FIRST_RULE_DST_ICMP_TYPE_END = 8; - static final int FIRST_RULE_SRC_ICMP_TYPE_START = 1; - static final int FIRST_RULE_SRC_ICMP_TYPE_END = 7; - static final int ICMP_PROTOCOL = 1; - static final int SECOND_RULE_DST_PORT_RANGE_START = 2000; - static final int SECOND_RULE_DST_PORT_RANGE_END = 6000; - static final int SECOND_RULE_SRC_PORT_RANGE_START = 400; - static final int SECOND_RULE_SRC_PORT_RANGE_END = 2047; - static final int UDP_PROTOCOL = 17; - - - static MacipAclRule[] createMacipRules() { - MacipAclRule ruleOne = new MacipAclRule(); - ruleOne.isIpv6 = 0; - ruleOne.isPermit = 1; - ruleOne.srcIpAddr = FIRST_RULE_ADDRESS_AS_ARRAY; - ruleOne.srcIpPrefixLen = FIRST_RULE_PREFIX; - ruleOne.srcMac = FIRST_RULE_MAC; - ruleOne.srcMacMask = FIRST_RULE_MAC_MASK;// no mask - - MacipAclRule ruleTwo = new MacipAclRule(); - ruleTwo.isIpv6 = 1; - ruleTwo.isPermit = 0; - ruleTwo.srcIpAddr = SECOND_RULE_ADDRESS_AS_ARRAY; - ruleTwo.srcIpPrefixLen = SECOND_RULE_PREFIX; - ruleTwo.srcMac = SECOND_RULE_MAC; - ruleTwo.srcMacMask = SECOND_RULE_MAC_MASK; - - return new MacipAclRule[]{ruleOne, ruleTwo}; - } - - static AclRule[] createAclRules() { - AclRule ruleOne = new AclRule(); - - ruleOne.isIpv6 = 0; - ruleOne.isPermit = 1; - ruleOne.srcIpAddr = FIRST_RULE_ADDRESS_AS_ARRAY; - ruleOne.srcIpPrefixLen = FIRST_RULE_PREFIX; - ruleOne.dstIpAddr = FIRST_RULE_ADDRESS_2_AS_ARRAY; - ruleOne.dstIpPrefixLen = FIRST_RULE_PREFIX_2; - ruleOne.dstportOrIcmpcodeFirst = FIRST_RULE_DST_ICMP_TYPE_START; - ruleOne.dstportOrIcmpcodeLast = FIRST_RULE_DST_ICMP_TYPE_END; - ruleOne.srcportOrIcmptypeFirst = FIRST_RULE_SRC_ICMP_TYPE_START; - ruleOne.srcportOrIcmptypeLast = FIRST_RULE_SRC_ICMP_TYPE_END; - ruleOne.proto = ICMP_PROTOCOL; //ICMP - - AclRule ruleTwo = new AclRule(); - ruleTwo.isIpv6 = 1; - ruleTwo.isPermit = 0; - ruleTwo.srcIpAddr = SECOND_RULE_ADDRESS_AS_ARRAY; - ruleTwo.srcIpPrefixLen = SECOND_RULE_PREFIX; - ruleTwo.dstIpAddr = SECOND_RULE_ADDRESS_2_AS_ARRAY; - ruleTwo.dstIpPrefixLen = SECOND_RULE_PREFIX_2; - ruleTwo.dstportOrIcmpcodeFirst = SECOND_RULE_DST_PORT_RANGE_START; - ruleTwo.dstportOrIcmpcodeLast = SECOND_RULE_DST_PORT_RANGE_END; - ruleTwo.srcportOrIcmptypeFirst = SECOND_RULE_SRC_PORT_RANGE_START; - ruleTwo.srcportOrIcmptypeLast = SECOND_RULE_SRC_PORT_RANGE_END; - ruleTwo.proto = UDP_PROTOCOL; //UDP - - return new AclRule[]{ruleOne, ruleTwo}; - } -} diff --git a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestRequests.java b/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestRequests.java deleted file mode 100644 index b580ee8c..00000000 --- a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/AclTestRequests.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.acl.test; - -import static io.fd.vpp.jvpp.acl.test.AclTestData.createAclRules; -import static io.fd.vpp.jvpp.acl.test.AclTestData.createMacipRules; - -import io.fd.vpp.jvpp.VppInvocationException; -import io.fd.vpp.jvpp.acl.dto.AclAddReplace; -import io.fd.vpp.jvpp.acl.dto.AclAddReplaceReply; -import io.fd.vpp.jvpp.acl.dto.AclDel; -import io.fd.vpp.jvpp.acl.dto.AclDelReply; -import io.fd.vpp.jvpp.acl.dto.AclDetailsReplyDump; -import io.fd.vpp.jvpp.acl.dto.AclDump; -import io.fd.vpp.jvpp.acl.dto.AclInterfaceListDetailsReplyDump; -import io.fd.vpp.jvpp.acl.dto.AclInterfaceListDump; -import io.fd.vpp.jvpp.acl.dto.AclInterfaceSetAclList; -import io.fd.vpp.jvpp.acl.dto.AclInterfaceSetAclListReply; -import io.fd.vpp.jvpp.acl.dto.MacipAclAdd; -import io.fd.vpp.jvpp.acl.dto.MacipAclAddReply; -import io.fd.vpp.jvpp.acl.dto.MacipAclDel; -import io.fd.vpp.jvpp.acl.dto.MacipAclDelReply; -import io.fd.vpp.jvpp.acl.dto.MacipAclDetailsReplyDump; -import io.fd.vpp.jvpp.acl.dto.MacipAclDump; -import io.fd.vpp.jvpp.acl.future.FutureJVppAclFacade; -import java.util.concurrent.ExecutionException; - -class AclTestRequests { - - static MacipAclDetailsReplyDump sendMacIpDumpRequest(final FutureJVppAclFacade jvpp) - throws ExecutionException, InterruptedException { - System.out.println("Sending MacipAclDump request..."); - MacipAclDetailsReplyDump dump = jvpp.macipAclDump(new MacipAclDump()).toCompletableFuture().get(); - System.out.println("MacipAclDump returned"); - return dump; - } - - static void sendMacIpAddRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { - final MacipAclAdd request = createMacIpAddRequest(); - System.out.printf("Sending MacipAclAdd request %s%n", request.toString()); - final MacipAclAddReply reply = jvpp.macipAclAdd(createMacIpAddRequest()).toCompletableFuture().get(); - System.out.printf("MacipAclAdd send result = %s%n", reply); - } - - static void sendMacIpDelRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { - final MacipAclDel request = new MacipAclDel(); - request.aclIndex = 0; - System.out.printf("Sending MacipAclDel request %s%n", request.toString()); - final MacipAclDelReply reply = jvpp.macipAclDel(request).toCompletableFuture().get(); - System.out.printf("MacipAclDel send result = %s%n", reply); - } - - static void sendAclAddRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { - final AclAddReplace request = createAclAddRequest(); - System.out.printf("Sending AclAddReplace request %s%n", request.toString()); - final AclAddReplaceReply reply = jvpp.aclAddReplace(request).toCompletableFuture().get(); - System.out.printf("AclAddReplace send result = %s%n", reply); - } - - static AclDetailsReplyDump sendAclDumpRequest(final FutureJVppAclFacade jvpp) - throws InterruptedException, VppInvocationException, ExecutionException { - System.out.println("Sending AclDump request..."); - final AclDetailsReplyDump dump = jvpp.aclDump(new AclDump()).toCompletableFuture().get(); - System.out.printf("AclDump send result = %s%n", dump); - return dump; - } - - static void sendAclDelRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { - final AclDel request = new AclDel(); - request.aclIndex = 0; - System.out.printf("Sending AclDel request %s%n", request.toString()); - final AclDelReply reply = jvpp.aclDel(request).toCompletableFuture().get(); - System.out.printf("AclDel send result = %s%n", reply); - } - - static AclInterfaceListDetailsReplyDump sendAclInterfaceListDumpRequest(final FutureJVppAclFacade jvpp) - throws InterruptedException, ExecutionException { - final AclInterfaceListDump request = new AclInterfaceListDump(); - request.swIfIndex = 0; - System.out.printf("Sending AclInterfaceListDump request %s%n", request.toString()); - final AclInterfaceListDetailsReplyDump dump = jvpp.aclInterfaceListDump(request).toCompletableFuture().get(); - System.out.printf("AclInterfaceListDump send result = %s%n", dump); - return dump; - } - - static void sendAclInterfaceSetAclList(final FutureJVppAclFacade jvpp) - throws InterruptedException, ExecutionException { - final AclInterfaceSetAclList request = new AclInterfaceSetAclList(); - request.count = 1; - request.acls = new int[]{1}; - request.swIfIndex = 0; - request.nInput = 0; - System.out.printf("Sending AclInterfaceSetAclList request %s%n", request.toString()); - final AclInterfaceSetAclListReply reply = jvpp.aclInterfaceSetAclList(request).toCompletableFuture().get(); - System.out.printf("AclInterfaceSetAclList send result = %s%n", reply); - } - - static void sendAclInterfaceDeleteList(final FutureJVppAclFacade jvpp) - throws InterruptedException, ExecutionException { - // uses same api but sets list to empty - final AclInterfaceSetAclList request = new AclInterfaceSetAclList(); - request.count = 0; - request.acls = new int[]{}; - request.swIfIndex = 0; - request.nInput = 0; - System.out.printf("Sending AclInterfaceSetAclList(Delete) request %s%n", request.toString()); - final AclInterfaceSetAclListReply reply = jvpp.aclInterfaceSetAclList(request).toCompletableFuture().get(); - System.out.printf("AclInterfaceSetAclList(Delete) send result = %s%n", reply); - } - - private static MacipAclAdd createMacIpAddRequest() { - MacipAclAdd request = new MacipAclAdd(); - - request.count = 2; - request.r = createMacipRules(); - return request; - } - - private static AclAddReplace createAclAddRequest() { - AclAddReplace request = new AclAddReplace(); - - request.aclIndex = -1;// to define new one - request.count = 2; - request.r = createAclRules(); - return request; - } -} diff --git a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/FutureApiTest.java b/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/FutureApiTest.java deleted file mode 100644 index 94490193..00000000 --- a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/FutureApiTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.acl.test; - -import static io.fd.vpp.jvpp.acl.test.AclExpectedDumpData.verifyAclDump; -import static io.fd.vpp.jvpp.acl.test.AclExpectedDumpData.verifyAclInterfaceList; -import static io.fd.vpp.jvpp.acl.test.AclExpectedDumpData.verifyMacIpDump; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclAddRequest; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclDelRequest; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclDumpRequest; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclInterfaceDeleteList; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclInterfaceListDumpRequest; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclInterfaceSetAclList; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendMacIpAddRequest; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendMacIpDelRequest; -import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendMacIpDumpRequest; - -import io.fd.vpp.jvpp.JVppRegistry; -import io.fd.vpp.jvpp.JVppRegistryImpl; -import io.fd.vpp.jvpp.acl.JVppAclImpl; -import io.fd.vpp.jvpp.acl.future.FutureJVppAclFacade; - -public class FutureApiTest { - - public static void main(String[] args) throws Exception { - testCallbackApi(); - } - - private static void testCallbackApi() throws Exception { - System.out.println("Testing Java callback API for acl plugin"); - try (final JVppRegistry registry = new JVppRegistryImpl("macipAclAddTest"); - final FutureJVppAclFacade jvpp = new FutureJVppAclFacade(registry, new JVppAclImpl())) { - - // adds,dump and verifies Mac-Ip acl - sendMacIpAddRequest(jvpp); - verifyMacIpDump(sendMacIpDumpRequest(jvpp).macipAclDetails.get(0)); - - // adds,dumps and verifies Acl acl - sendAclAddRequest(jvpp); - verifyAclDump(sendAclDumpRequest(jvpp).aclDetails.get(0)); - - // adds,dumps and verifies Interface for acl - sendAclInterfaceSetAclList(jvpp); - verifyAclInterfaceList(sendAclInterfaceListDumpRequest(jvpp).aclInterfaceListDetails.get(0)); - - // deletes all created data - sendAclInterfaceDeleteList(jvpp); - sendAclDelRequest(jvpp); - sendMacIpDelRequest(jvpp); - - System.out.println("Disconnecting..."); - } - } -} diff --git a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/Readme.txt b/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/Readme.txt deleted file mode 100644 index f68e7aba..00000000 --- a/plugins/acl-plugin/acl/jvpp/io/fd/vpp/jvpp/acl/test/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/plugins/acl-plugin/jvpp-acl-1.0.jar io.fd.vpp.jvpp.acl.test.FutureApiTest diff --git a/plugins/acl-plugin/acl/jvpp_acl.c b/plugins/acl-plugin/acl/jvpp_acl.c deleted file mode 100644 index 0af53bc9..00000000 --- a/plugins/acl-plugin/acl/jvpp_acl.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#include -#include -#include - -#if VPPJNI_DEBUG == 1 - #define DEBUG_LOG(...) clib_warning(__VA_ARGS__) -#else - #define DEBUG_LOG(...) -#endif - -#include - -#include "acl/jvpp/io_fd_vpp_jvpp_acl_JVppAclImpl.h" -#include "jvpp_acl.h" -#include "acl/jvpp/jvpp_acl_gen.h" - -/* - * Class: io_fd_vpp_jvpp_acl_JVppaclImpl - * Method: init0 - * Signature: (JI)V - */ -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_acl_JVppAclImpl_init0 - (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) { - acl_main_t * plugin_main = &acl_main; - u8 * name; - clib_warning ("Java_io_fd_vpp_jvpp_acl_JVppAclImpl_init0"); - - plugin_main->my_client_index = my_client_index; - plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address; - - name = format (0, "acl_%08x%c", api_version, 0); - plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (plugin_main->msg_id_base == (u16) ~0) { - jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException"); - (*env)->ThrowNew(env, exClass, "acl plugin is not loaded in VPP"); - } else { - plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback); - plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback)); - - #define _(N,n) \ - vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_api_reply_handler; - #undef _ - } -} - -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_acl_JVppAclImpl_close0 -(JNIEnv *env, jclass clazz) { - acl_main_t * plugin_main = &acl_main; - - // cleanup: - (*env)->DeleteGlobalRef(env, plugin_main->callbackClass); - (*env)->DeleteGlobalRef(env, plugin_main->callbackObject); - - plugin_main->callbackClass = NULL; - plugin_main->callbackObject = NULL; -} - -/* Attach thread to JVM and cache class references when initiating JVPP ACL */ -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv* env; - - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return JNI_EVERSION; - } - - if (cache_class_references(env) != 0) { - clib_warning ("Failed to cache class references\n"); - return JNI_ERR; - } - - return JNI_VERSION_1_8; -} - -/* Clean up cached references when disposing JVPP ACL */ -void JNI_OnUnload(JavaVM *vm, void *reserved) { - JNIEnv* env; - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return; - } - delete_class_references(env); -} diff --git a/plugins/acl-plugin/acl/jvpp_acl.h b/plugins/acl-plugin/acl/jvpp_acl.h deleted file mode 100644 index 2b73d672..00000000 --- a/plugins/acl-plugin/acl/jvpp_acl.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_jvpp_acl_h__ -#define __included_jvpp_acl_h__ - -#include -#include -#include -#include -#include -#include - -/* Global state for JVPP-acl */ -typedef struct { - /* Base message index for the acl plugin */ - u16 msg_id_base; - - /* Pointer to shared memory queue */ - unix_shared_memory_queue_t * vl_input_queue; - - /* VPP api client index */ - u32 my_client_index; - - /* Callback object and class references enabling asynchronous Java calls */ - jobject callbackObject; - jclass callbackClass; - -} acl_main_t; - -acl_main_t acl_main __attribute__((aligned (64))); - - -#endif /* __included_jvpp_acl_h__ */ diff --git a/plugins/acl-plugin/acl/l2sess.c b/plugins/acl-plugin/acl/l2sess.c deleted file mode 100644 index cc9bde44..00000000 --- a/plugins/acl-plugin/acl/l2sess.c +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * l2sess.c - simple MAC-swap API / debug CLI handling - *------------------------------------------------------------------ - */ - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -void -l2sess_vlib_plugin_register (vlib_main_t * vm, void* hh, - int from_early_init) -{ - l2sess_main_t *sm = &l2sess_main; - vnet_plugin_handoff_t * h = hh; - memset (sm, 0, sizeof (*sm)); - - sm->vlib_main = vm; - sm->vnet_main = h->vnet_main; - sm->ethernet_main = h->ethernet_main; -} - -void -l2sess_init_next_features_input (vlib_main_t * vm, l2sess_main_t * sm) -{ -#define _(node_name, node_var, is_out, is_ip6, is_track) \ - if (!is_out) feat_bitmap_init_next_nodes(vm, node_var.index, L2INPUT_N_FEAT, l2input_get_feat_names (), sm->node_var ## _input_next_node_index); - foreach_l2sess_node -#undef _ -} - -void -l2sess_add_our_next_nodes (vlib_main_t * vm, l2sess_main_t * sm, - u8 * prev_node_name, int add_output_nodes) -{ - vlib_node_t *n; - n = vlib_get_node_by_name (vm, prev_node_name); -#define _(node_name, node_var, is_out, is_ip6, is_track) \ - if (is_out == add_output_nodes) { \ - u32 idx = vlib_node_add_next_with_slot(vm, n->index, node_var.index, ~0); \ - if (is_track) { \ - sm->next_slot_track_node_by_is_ip6_is_out[is_ip6][is_out] = idx; \ - } \ - } - foreach_l2sess_node -#undef _ -} - -void -l2sess_setup_nodes (void) -{ - vlib_main_t *vm = vlib_get_main (); - l2sess_main_t *sm = &l2sess_main; - - l2sess_init_next_features_input (vm, sm); - - l2sess_add_our_next_nodes (vm, sm, (u8 *) "l2-input-classify", 0); - l2sess_add_our_next_nodes (vm, sm, (u8 *) "l2-output-classify", 1); - -} - -static char * -get_l4_proto_str (int is_ip6, uint8_t l4_proto) -{ - switch (l4_proto) - { - case 6: - return "tcp"; - case 17: - return "udp"; - case 1: - return "icmp"; - case 58: - return "icmp6"; - default: - return ""; - } -} - -static clib_error_t * -l2sess_show_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - l2sess_main_t *sm = &l2sess_main; - clib_time_t *ct = &vm->clib_time; - l2s_session_t *s; - u64 now = clib_cpu_time_now (); - - vlib_cli_output (vm, "Timing wheel info: \n%U", format_timing_wheel, - &sm->timing_wheel, 255); - - pool_foreach (s, sm->sessions, ( - { - f64 ctime = - (now - - s->create_time) * ct->seconds_per_clock; - f64 atime0 = - (now - - s->side[0].active_time) * - ct->seconds_per_clock; - f64 atime1 = - (now - - s->side[1].active_time) * - ct->seconds_per_clock; -/* - f64 ctime = (s->create_time - vm->cpu_time_main_loop_start) * ct->seconds_per_clock; - f64 atime0 = (s->side[0].active_time - vm->cpu_time_main_loop_start) * ct->seconds_per_clock; - f64 atime1 = (s->side[1].active_time - vm->cpu_time_main_loop_start) * ct->seconds_per_clock; -*/ - u8 * out0 = - format (0, - "%5d: create time: %U pkts/bytes/active time: [ %ld %ld %U : %ld %ld %U ]\n", - (s - sm->sessions), - format_time_interval, "h:m:s:u", - ctime, s->side[0].n_packets, - s->side[0].n_bytes, - format_time_interval, "h:m:s:u", - atime0, s->side[1].n_packets, - s->side[1].n_bytes, - format_time_interval, "h:m:s:u", - atime1); u8 * out1 = 0; - if (s->is_ip6) - { - out1 = - format (0, "%s %U :%u <-> %U :%u", - get_l4_proto_str (s->is_ip6, - s->l4_proto), - format_ip6_address, - &s->side[0].addr.ip6, - s->side[0].port, - format_ip6_address, - &s->side[1].addr.ip6, - s->side[1].port);} - else - { - out1 = - format (0, "%s %U :%u <-> %U :%u", - get_l4_proto_str (s->is_ip6, - s->l4_proto), - format_ip4_address, - &s->side[0].addr.ip4, - s->side[0].port, - format_ip4_address, - &s->side[1].addr.ip4, - s->side[1].port);} - vlib_cli_output (vm, "%s %s", out0, - out1); vec_free (out0); - vec_free (out1);} - )); - return 0; -} - -static clib_error_t * -l2sess_show_count_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - l2sess_main_t *sm = &l2sess_main; - - vlib_cli_output (vm, "Timing wheel info: \n%U", format_timing_wheel, - &sm->timing_wheel, 255); - vlib_cli_output (vm, "session pool len: %d, pool elts: %d", - pool_len (sm->sessions), pool_elts (sm->sessions)); - vlib_cli_output (vm, - "attempted to delete sessions which were already free: %d", - sm->counter_attempted_delete_free_session); - return 0; -} - - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (l2sess_show_command, static) = { - .path = "show l2sess", - .short_help = "show l2sess", - .function = l2sess_show_command_fn, -}; - -VLIB_CLI_COMMAND (l2sess_show_count_command, static) = { - .path = "show l2sess count", - .short_help = "show l2sess count", - .function = l2sess_show_count_command_fn, -}; -/* *INDENT-OFF* */ - -static inline u64 -time_sec_to_clock( clib_time_t *ct, f64 sec) -{ - return (u64)(((f64)sec)/ct->seconds_per_clock); -} - -static clib_error_t * l2sess_init (vlib_main_t * vm) -{ - l2sess_main_t * sm = &l2sess_main; - clib_error_t * error = 0; - u64 cpu_time_now = clib_cpu_time_now(); - - - clib_time_t *ct = &vm->clib_time; - sm->udp_session_idle_timeout = time_sec_to_clock(ct, UDP_SESSION_IDLE_TIMEOUT_SEC); - sm->tcp_session_idle_timeout = time_sec_to_clock(ct, TCP_SESSION_IDLE_TIMEOUT_SEC); - sm->tcp_session_transient_timeout = time_sec_to_clock(ct, TCP_SESSION_TRANSIENT_TIMEOUT_SEC); - - /* The min sched time of 10e-1 causes erroneous behavior... */ - sm->timing_wheel.min_sched_time = 10e-2; - sm->timing_wheel.max_sched_time = 3600.0*48.0; - timing_wheel_init (&sm->timing_wheel, cpu_time_now, vm->clib_time.clocks_per_second); - sm->timer_wheel_next_expiring_time = 0; - sm->timer_wheel_tick = time_sec_to_clock(ct, sm->timing_wheel.min_sched_time); - /* Pre-allocate expired nodes. */ - vec_alloc (sm->data_from_advancing_timing_wheel, 32); - - l2sess_setup_nodes(); - l2output_init_output_node_vec (&sm->output_next_nodes.output_node_index_vec); - - return error; -} - -VLIB_INIT_FUNCTION (l2sess_init); - - diff --git a/plugins/acl-plugin/acl/l2sess.h b/plugins/acl-plugin/acl/l2sess.h deleted file mode 100644 index db899917..00000000 --- a/plugins/acl-plugin/acl/l2sess.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_l2sess_h__ -#define __included_l2sess_h__ - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -#define _(node_name, node_var, is_out, is_ip6, is_track) -#undef _ -#define foreach_l2sess_node \ - _("aclp-l2s-input-ip4-add", l2sess_in_ip4_add, 0, 0, 0) \ - _("aclp-l2s-input-ip6-add", l2sess_in_ip6_add, 0, 1, 0) \ - _("aclp-l2s-output-ip4-add", l2sess_out_ip4_add, 1, 0, 0) \ - _("aclp-l2s-output-ip6-add", l2sess_out_ip6_add, 1, 1, 0) \ - _("aclp-l2s-input-ip4-track", l2sess_in_ip4_track, 0, 0, 1) \ - _("aclp-l2s-input-ip6-track", l2sess_in_ip6_track, 0, 1, 1) \ - _("aclp-l2s-output-ip4-track",l2sess_out_ip4_track, 1, 0, 1) \ - _("aclp-l2s-output-ip6-track", l2sess_out_ip6_track, 1, 1, 1) - -#define _(node_name, node_var, is_out, is_ip6, is_track) \ - extern vlib_node_registration_t node_var; -foreach_l2sess_node -#undef _ - -#define TCP_FLAG_FIN 0x01 -#define TCP_FLAG_SYN 0x02 -#define TCP_FLAG_RST 0x04 -#define TCP_FLAG_PUSH 0x08 -#define TCP_FLAG_ACK 0x10 -#define TCP_FLAG_URG 0x20 -#define TCP_FLAG_ECE 0x40 -#define TCP_FLAG_CWR 0x80 -#define TCP_FLAGS_RSTFINACKSYN (TCP_FLAG_RST + TCP_FLAG_FIN + TCP_FLAG_SYN + TCP_FLAG_ACK) -#define TCP_FLAGS_ACKSYN (TCP_FLAG_SYN + TCP_FLAG_ACK) - -typedef struct { - ip46_address_t addr; - u64 active_time; - u64 n_packets; - u64 n_bytes; - u16 port; -} l2s_session_side_t; - -enum { - L2S_SESSION_SIDE_IN = 0, - L2S_SESSION_SIDE_OUT, - L2S_N_SESSION_SIDES -}; - -typedef struct { - u64 create_time; - l2s_session_side_t side[L2S_N_SESSION_SIDES]; - u8 l4_proto; - u8 is_ip6; - u16 tcp_flags_seen; /* u16 because of two sides */ -} l2s_session_t; - -#define PROD -#ifdef PROD -#define UDP_SESSION_IDLE_TIMEOUT_SEC 600 -#define TCP_SESSION_IDLE_TIMEOUT_SEC (3600*24) -#define TCP_SESSION_TRANSIENT_TIMEOUT_SEC 120 -#else -#define UDP_SESSION_IDLE_TIMEOUT_SEC 15 -#define TCP_SESSION_IDLE_TIMEOUT_SEC 15 -#define TCP_SESSION_TRANSIENT_TIMEOUT_SEC 5 -#endif - -typedef struct { - /* - * the next two fields are present for all nodes, but - * only one of them is used per node - depending - * on whether the node is an input or output one. - */ -#define _(node_name, node_var, is_out, is_ip6, is_track) \ - u32 node_var ## _input_next_node_index[32]; \ - l2_output_next_nodes_st node_var ## _next_nodes; -foreach_l2sess_node -#undef _ - l2_output_next_nodes_st output_next_nodes; - - /* Next indices of the tracker nodes */ - u32 next_slot_track_node_by_is_ip6_is_out[2][2]; - - /* - * Pairing of "forward" and "reverse" tables by table index. - * Each relationship has two entries - for one and the other table, - * so it is bidirectional. - */ - - u32 *fwd_to_rev_by_table_index; - - /* - * The vector of per-interface session pools - */ - - l2s_session_t *sessions; - - /* The session timeouts */ - u64 tcp_session_transient_timeout; - u64 tcp_session_idle_timeout; - u64 udp_session_idle_timeout; - - /* Timing wheel to time out the idle sessions */ - timing_wheel_t timing_wheel; - u32 *data_from_advancing_timing_wheel; - u64 timer_wheel_next_expiring_time; - u64 timer_wheel_tick; - - /* convenience */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; - ethernet_main_t * ethernet_main; - - /* Counter(s) */ - u64 counter_attempted_delete_free_session; -} l2sess_main_t; - -l2sess_main_t l2sess_main; - -/* Just exposed for acl.c */ - -void -l2sess_vlib_plugin_register (vlib_main_t * vm, void * hh, - int from_early_init); - - -#endif /* __included_l2sess_h__ */ diff --git a/plugins/acl-plugin/acl/l2sess_node.c b/plugins/acl-plugin/acl/l2sess_node.c deleted file mode 100644 index 520e5929..00000000 --- a/plugins/acl-plugin/acl/l2sess_node.c +++ /dev/null @@ -1,816 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include - - -typedef struct -{ - u32 next_index; - u32 sw_if_index; - u32 trace_flags; - u32 session_tables[2]; - u32 session_nexts[2]; - u8 l4_proto; -} l2sess_trace_t; - -/* packet trace format function */ - -#define _(node_name, node_var, is_out, is_ip6, is_track) \ -static u8 * format_## node_var ##_trace (u8 * s, va_list * args) \ -{ \ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); \ - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); \ - l2sess_trace_t * t = va_arg (*args, l2sess_trace_t *); \ - \ - s = format (s, node_name ": sw_if_index %d, next index %d trace_flags %08x L4 proto %d\n" \ - " tables [ %d, %d ] nexts [ %d, %d ]", \ - t->sw_if_index, t->next_index, t->trace_flags, t->l4_proto, \ - t->session_tables[0], t->session_tables[1], \ - t->session_nexts[0], t->session_nexts[1]); \ - return s; \ -} -foreach_l2sess_node -#undef _ -#define foreach_l2sess_error \ -_(SWAPPED, "Mac swap packets processed") - typedef enum -{ -#define _(sym,str) L2SESS_ERROR_##sym, - foreach_l2sess_error -#undef _ - L2SESS_N_ERROR, -} l2sess_error_t; - -static char *l2sess_error_strings[] = { -#define _(sym,string) string, - foreach_l2sess_error -#undef _ -}; - -typedef enum -{ - L2SESS_NEXT_DROP, - L2SESS_N_NEXT, -} l2sess_next_t; - -u8 -l2sess_get_l4_proto (vlib_buffer_t * b0, int node_is_ip6) -{ - u8 proto; - int proto_offset; - if (node_is_ip6) - { - proto_offset = 20; - } - else - { - proto_offset = 23; - } - proto = *((u8 *) vlib_buffer_get_current (b0) + proto_offset); - return proto; -} - - -u8 -l2sess_get_tcp_flags (vlib_buffer_t * b0, int node_is_ip6) -{ - u8 flags; - int flags_offset; - if (node_is_ip6) - { - flags_offset = 14 + 40 + 13; /* FIXME: no extension headers assumed */ - } - else - { - flags_offset = 14 + 20 + 13; - } - flags = *((u8 *) vlib_buffer_get_current (b0) + flags_offset); - return flags; -} - -static inline int -l4_tcp_or_udp (u8 proto) -{ - return ((proto == 6) || (proto == 17)); -} - -void -l2sess_get_session_tables (l2sess_main_t * sm, u32 sw_if_index, - int node_is_out, int node_is_ip6, u8 l4_proto, - u32 * session_tables) -{ -/* - * Based on the direction, l3 and l4 protocol, fill a u32[2] array: - * [0] is index for the "direct match" path, [1] is for "mirrored match". - * Store the indices of the tables to add the session to in session_tables[] - */ - l2_output_classify_main_t *l2om = &l2_output_classify_main; - l2_input_classify_main_t *l2im = &l2_input_classify_main; - - u32 output_table_index; - u32 input_table_index; - - if (!l4_tcp_or_udp (l4_proto)) - { - return; - } - - if (node_is_ip6) - { - vec_validate_init_empty (l2im-> - classify_table_index_by_sw_if_index - [L2_INPUT_CLASSIFY_TABLE_IP6], sw_if_index, - ~0); - input_table_index = - l2im-> - classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP6] - [sw_if_index]; - vec_validate_init_empty (l2om-> - classify_table_index_by_sw_if_index - [L2_OUTPUT_CLASSIFY_TABLE_IP6], sw_if_index, - ~0); - output_table_index = - l2om-> - classify_table_index_by_sw_if_index[L2_OUTPUT_CLASSIFY_TABLE_IP6] - [sw_if_index]; - } - else - { - vec_validate_init_empty (l2im-> - classify_table_index_by_sw_if_index - [L2_INPUT_CLASSIFY_TABLE_IP4], sw_if_index, - ~0); - input_table_index = - l2im-> - classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP4] - [sw_if_index]; - vec_validate_init_empty (l2om-> - classify_table_index_by_sw_if_index - [L2_OUTPUT_CLASSIFY_TABLE_IP4], sw_if_index, - ~0); - output_table_index = - l2om-> - classify_table_index_by_sw_if_index[L2_OUTPUT_CLASSIFY_TABLE_IP4] - [sw_if_index]; - } - - if (node_is_out) - { - session_tables[0] = output_table_index; - session_tables[1] = input_table_index; - } - else - { - session_tables[0] = input_table_index; - session_tables[1] = output_table_index; - } -} - -void -l2sess_get_session_nexts (l2sess_main_t * sm, u32 sw_if_index, - int node_is_out, int node_is_ip6, u8 l4_proto, - u32 * session_nexts) -{ -/* - * Based on the direction, l3 and l4 protocol, fill a u32[2] array: - * [0] is the index for the "direct match" path, [1] is for "mirrored match". - * Store the match_next_index in session_nexts[] for a new session entry which is being added to session tables. - */ - u32 input_node_index; - u32 output_node_index; - - if (!l4_tcp_or_udp (l4_proto)) - { - return; - } - - input_node_index = - sm->next_slot_track_node_by_is_ip6_is_out[node_is_ip6][0]; - output_node_index = - sm->next_slot_track_node_by_is_ip6_is_out[node_is_ip6][1]; - - if (node_is_out) - { - session_nexts[0] = output_node_index; - session_nexts[1] = input_node_index; - } - else - { - session_nexts[0] = input_node_index; - session_nexts[1] = output_node_index; - } -} - - -static inline void -swap_bytes (vlib_buffer_t * b0, int off_a, int off_b, int nbytes) -{ - u8 tmp; - u8 *pa = vlib_buffer_get_current (b0) + off_a; - u8 *pb = vlib_buffer_get_current (b0) + off_b; - while (nbytes--) - { - tmp = *pa; - *pa++ = *pb; - *pb++ = tmp; - } -} - -/* - * This quite pro[bv]ably is a terrible idea performance wise. Moreso doing it twice. - * Would having a long (ish) chunk of memory work better for this ? - * We will see when we get to the performance of this. - */ -void -l2sess_flip_l3l4_fields (vlib_buffer_t * b0, int node_is_ip6, u8 l4_proto) -{ - if (!l4_tcp_or_udp (l4_proto)) - { - return; - } - if (node_is_ip6) - { - swap_bytes (b0, 22, 38, 16); /* L3 */ - swap_bytes (b0, 54, 56, 2); /* L4 (when no EH!) */ - } - else - { - swap_bytes (b0, 26, 30, 4); /* L3 */ - swap_bytes (b0, 34, 36, 2); /* L4 */ - } -} - -void -l2sess_add_session (vlib_buffer_t * b0, int node_is_out, int node_is_ip6, - u32 session_table, u32 session_match_next, - u32 opaque_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - u32 action = 0; - u32 metadata = 0; - -#ifdef DEBUG_SESSIONS - printf ("Adding session to table %d with next %d\n", session_table, - session_match_next); -#endif - vnet_classify_add_del_session (cm, session_table, - vlib_buffer_get_current (b0), - session_match_next, opaque_index, 0, action, - metadata, 1); -} - - - -static void * -get_ptr_to_offset (vlib_buffer_t * b0, int offset) -{ - u8 *p = vlib_buffer_get_current (b0) + offset; - return p; -} - - -/* - * FIXME: Hardcoded offsets are ugly, although if casting to structs one - * would need to take care about alignment.. So let's for now be naive and simple. - */ - -void -session_store_ip4_l3l4_info (vlib_buffer_t * b0, l2s_session_t * sess, - int node_is_out) -{ - clib_memcpy (&sess->side[1 - node_is_out].addr.ip4, - get_ptr_to_offset (b0, 26), 4); - clib_memcpy (&sess->side[node_is_out].addr.ip4, get_ptr_to_offset (b0, 30), - 4); - sess->side[1 - node_is_out].port = - ntohs (*(u16 *) get_ptr_to_offset (b0, 34)); - sess->side[node_is_out].port = ntohs (*(u16 *) get_ptr_to_offset (b0, 36)); -} - -void -session_store_ip6_l3l4_info (vlib_buffer_t * b0, l2s_session_t * sess, - int node_is_out) -{ - clib_memcpy (&sess->side[1 - node_is_out].addr.ip6, - get_ptr_to_offset (b0, 22), 16); - clib_memcpy (&sess->side[node_is_out].addr.ip4, get_ptr_to_offset (b0, 38), - 16); - sess->side[1 - node_is_out].port = - ntohs (*(u16 *) get_ptr_to_offset (b0, 54)); - sess->side[node_is_out].port = ntohs (*(u16 *) get_ptr_to_offset (b0, 56)); -} - -static void -build_match_from_session (l2sess_main_t * sm, u8 * match, - l2s_session_t * sess, int is_out) -{ - if (sess->is_ip6) - { - match[20] = sess->l4_proto; - clib_memcpy (&match[22], &sess->side[1 - is_out].addr.ip6, 16); - clib_memcpy (&match[38], &sess->side[is_out].addr.ip4, 16); - *(u16 *) & match[54] = htons (sess->side[1 - is_out].port); - *(u16 *) & match[56] = htons (sess->side[is_out].port); - } - else - { - match[23] = sess->l4_proto; - clib_memcpy (&match[26], &sess->side[1 - is_out].addr.ip6, 4); - clib_memcpy (&match[30], &sess->side[is_out].addr.ip4, 4); - *(u16 *) & match[34] = htons (sess->side[1 - is_out].port); - *(u16 *) & match[36] = htons (sess->side[is_out].port); - } -} - -static void -delete_session (l2sess_main_t * sm, u32 sw_if_index, u32 session_index) -{ - vnet_classify_main_t *cm = &vnet_classify_main; - u8 match[5 * 16]; /* For building the mock of the packet to delete the classifier session */ - u32 session_tables[2] = { ~0, ~0 }; - l2s_session_t *sess = sm->sessions + session_index; - if (pool_is_free (sm->sessions, sess)) - { - sm->counter_attempted_delete_free_session++; - return; - } - l2sess_get_session_tables (sm, sw_if_index, 0, sess->is_ip6, sess->l4_proto, - session_tables); - if (session_tables[1] != ~0) - { - build_match_from_session (sm, match, sess, 1); - vnet_classify_add_del_session (cm, session_tables[1], match, 0, 0, 0, 0, - 0, 0); - } - if (session_tables[1] != ~0) - { - build_match_from_session (sm, match, sess, 1); - vnet_classify_add_del_session (cm, session_tables[1], match, 0, 0, 0, 0, - 0, 0); - } - pool_put (sm->sessions, sess); -} - -static void -udp_session_account_buffer (vlib_buffer_t * b0, l2s_session_t * s, - int which_side, u64 now) -{ - l2s_session_side_t *ss = &s->side[which_side]; - ss->active_time = now; - ss->n_packets++; - ss->n_bytes += b0->current_data + b0->current_length; -} - -static inline u64 -udp_session_get_timeout (l2sess_main_t * sm, l2s_session_t * sess, u64 now) -{ - return (sm->udp_session_idle_timeout); -} - -static void -tcp_session_account_buffer (vlib_buffer_t * b0, l2s_session_t * s, - int which_side, u64 now) -{ - l2s_session_side_t *ss = &s->side[which_side]; - ss->active_time = now; - ss->n_packets++; - ss->n_bytes += b0->current_data + b0->current_length; - /* Very very lightweight TCP state tracking: just record which flags were seen */ - s->tcp_flags_seen |= - l2sess_get_tcp_flags (b0, s->is_ip6) << (8 * which_side); -} - -/* - * Since we are tracking for the purposes of timing the sessions out, - * we mostly care about two states: established (maximize the idle timeouts) - * and transient (halfopen/halfclosed/reset) - we need to have a reasonably short timeout to - * quickly get rid of sessions but not short enough to violate the TCP specs. - */ - -static inline u64 -tcp_session_get_timeout (l2sess_main_t * sm, l2s_session_t * sess, u64 now) -{ - /* seen both SYNs and ACKs but not FINs means we are in establshed state */ - u16 masked_flags = - sess->tcp_flags_seen & ((TCP_FLAGS_RSTFINACKSYN << 8) + - TCP_FLAGS_RSTFINACKSYN); - if (((TCP_FLAGS_ACKSYN << 8) + TCP_FLAGS_ACKSYN) == masked_flags) - { - return (sm->tcp_session_idle_timeout); - } - else - { - return (sm->tcp_session_transient_timeout); - } -} - -static inline u64 -session_get_timeout (l2sess_main_t * sm, l2s_session_t * sess, u64 now) -{ - u64 timeout; - - switch (sess->l4_proto) - { - case 6: - timeout = tcp_session_get_timeout (sm, sess, now); - break; - case 17: - timeout = udp_session_get_timeout (sm, sess, now); - break; - default: - timeout = 0; - } - - return timeout; -} - -static inline u64 -get_session_last_active_time(l2s_session_t * sess) -{ - u64 last_active = - sess->side[0].active_time > - sess->side[1].active_time ? sess->side[0].active_time : sess->side[1]. - active_time; - return last_active; -} - -static int -session_is_alive (l2sess_main_t * sm, l2s_session_t * sess, u64 now, u64 *last_active_cache) -{ - u64 last_active = get_session_last_active_time(sess); - u64 timeout = session_get_timeout (sm, sess, now); - int is_alive = ((now - last_active) < timeout); - if (last_active_cache) - *last_active_cache = last_active; - return is_alive; -} - -static void -check_idle_sessions (l2sess_main_t * sm, u32 sw_if_index, u64 now) -{ - sm->timer_wheel_next_expiring_time = 0; - sm->data_from_advancing_timing_wheel - = - timing_wheel_advance (&sm->timing_wheel, now, - sm->data_from_advancing_timing_wheel, - &sm->timer_wheel_next_expiring_time); -#ifdef DEBUG_SESSIONS_VERBOSE - { - clib_time_t *ct = &sm->vlib_main->clib_time; - f64 ctime; - ctime = now * ct->seconds_per_clock; - clib_warning ("Now : %U", format_time_interval, "h:m:s:u", ctime); - ctime = sm->timer_wheel_next_expiring_time * ct->seconds_per_clock; - clib_warning ("Next expire: %U", format_time_interval, "h:m:s:u", ctime); - clib_warning ("Expired items: %d", - (int) vec_len (sm->data_from_advancing_timing_wheel)); - } -#endif - - sm->timer_wheel_next_expiring_time = now + sm->timer_wheel_tick; - if (PREDICT_FALSE ( 0 == sm->data_from_advancing_timing_wheel )) { - return; - } - - if (PREDICT_FALSE (_vec_len (sm->data_from_advancing_timing_wheel) > 0)) - { - uword i; - for (i = 0; i < _vec_len (sm->data_from_advancing_timing_wheel); i++) - { - u32 session_index = sm->data_from_advancing_timing_wheel[i]; - if (!pool_is_free_index (sm->sessions, session_index)) - { - l2s_session_t *sess = sm->sessions + session_index; - u64 last_active; - if (session_is_alive (sm, sess, now, &last_active)) - { -#ifdef DEBUG_SESSIONS - clib_warning ("Restarting timer for session %d", (int) session_index); -#endif - /* Pretend we did this in the past, at last_active moment */ - timing_wheel_insert (&sm->timing_wheel, - last_active + session_get_timeout (sm, sess, - last_active), - session_index); - } - else - { -#ifdef DEBUG_SESSIONS - clib_warning ("Deleting session %d", (int) session_index); -#endif - delete_session (sm, sw_if_index, session_index); - } - } - } - _vec_len (sm->data_from_advancing_timing_wheel) = 0; - } -} - -static uword -l2sess_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - u32 n_left_from, *from, *to_next; - l2sess_next_t next_index; - u32 pkts_swapped = 0; - u32 cached_sw_if_index = (u32) ~ 0; - u32 cached_next_index = (u32) ~ 0; - u32 feature_bitmap0; - u32 trace_flags0; - - l2sess_main_t *sm = &l2sess_main; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - /* Only a single loop for now for simplicity */ - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0 = L2SESS_NEXT_DROP; - u32 sw_if_index0; - //ethernet_header_t *en0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - //en0 = vlib_buffer_get_current (b0); - -/* - * The non-boilerplate is in the block below. - * Note first a magic macro block that sets up the behavior qualifiers: - * node_is_out : 1 = is output, 0 = is input - * node_is_ip6 : 1 = is ip6, 0 = is ip4 - * node_is_track : 1 = is a state tracking node, 0 - is a session addition node - * - * Subsequently the code adjusts its behavior depending on these variables. - * It's most probably not great performance wise but much easier to work with. - * - */ - { - int node_is_out = -1; - CLIB_UNUSED (int node_is_ip6) = -1; - CLIB_UNUSED (int node_is_track) = -1; - u32 node_index = 0; - u32 session_tables[2] = { ~0, ~0 }; - u32 session_nexts[2] = { ~0, ~0 }; - l2_output_next_nodes_st *next_nodes = 0; - u32 *input_feat_next_node_index; - u8 l4_proto; - u64 now = clib_cpu_time_now (); - -/* - * Set the variables according to which of the 8 nodes we are. - * Hopefully the compiler is smart enough to eliminate the extraneous. - */ -#define _(node_name, node_var, is_out, is_ip6, is_track) \ -if(node_var.index == node->node_index) \ - { \ - node_is_out = is_out; \ - node_is_ip6 = is_ip6; \ - node_is_track = is_track; \ - node_index = node_var.index; \ - next_nodes = &sm->node_var ## _next_nodes; \ - input_feat_next_node_index = sm->node_var ## _input_next_node_index; \ - } - foreach_l2sess_node -#undef _ - trace_flags0 = 0; - if (node_is_out) - { - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - } - else - { - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - } - /* potentially also remove the nodes here */ - feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; - - if (node_is_track) - { - u32 sess_index = vnet_buffer (b0)->l2_classify.opaque_index; - l2s_session_t *sess = sm->sessions + sess_index; - l4_proto = sess->l4_proto; - - if (session_is_alive (sm, sess, now, 0)) - { - if (6 == l4_proto) - { - tcp_session_account_buffer (b0, sess, node_is_out, - now); - } - else - { - udp_session_account_buffer (b0, sess, node_is_out, - now); - } - } - else - { - timing_wheel_delete (&sm->timing_wheel, sess_index); - delete_session (sm, sw_if_index0, sess_index); - /* FIXME: drop the packet that hit the obsolete node, for now. We really ought to recycle it. */ - next0 = 0; - } - } - else - { - /* - * "-add" node: take l2opaque which arrived to us, and deduce - * the tables out of that. ~0 means the topmost classifier table - * applied for this AF on the RX(for input)/TX(for output)) sw_if_index. - * Also add the mirrored session to the paired table. - */ - l2s_session_t *sess; - u32 sess_index; - - l4_proto = l2sess_get_l4_proto (b0, node_is_ip6); - - pool_get (sm->sessions, sess); - sess_index = sess - sm->sessions; - sess->create_time = now; - sess->side[node_is_out].active_time = now; - sess->side[1 - node_is_out].active_time = now; - sess->l4_proto = l4_proto; - sess->is_ip6 = node_is_ip6; - if (node_is_ip6) - { - session_store_ip6_l3l4_info (b0, sess, node_is_out); - } - else - { - session_store_ip4_l3l4_info (b0, sess, node_is_out); - } - - l2sess_get_session_tables (sm, sw_if_index0, node_is_out, - node_is_ip6, l4_proto, - session_tables); - l2sess_get_session_nexts (sm, sw_if_index0, node_is_out, - node_is_ip6, l4_proto, - session_nexts); - l2sess_flip_l3l4_fields (b0, node_is_ip6, l4_proto); - if (session_tables[1] != ~0) - { - l2sess_add_session (b0, node_is_out, node_is_ip6, - session_tables[1], session_nexts[1], - sess_index); - } - l2sess_flip_l3l4_fields (b0, node_is_ip6, l4_proto); - if (session_tables[0] != ~0) - { - l2sess_add_session (b0, node_is_out, node_is_ip6, - session_tables[0], session_nexts[0], - sess_index); - } - if (6 == sess->l4_proto) - { - tcp_session_account_buffer (b0, sess, node_is_out, now); - } - else - { - udp_session_account_buffer (b0, sess, node_is_out, now); - } - timing_wheel_insert (&sm->timing_wheel, - now + session_get_timeout (sm, sess, - now), - sess_index); - } - - if (now >= sm->timer_wheel_next_expiring_time) - { - check_idle_sessions (sm, sw_if_index0, now); - } - - if (node_is_out) - { - if (feature_bitmap0) - { - trace_flags0 |= 0x10; - } - if (sw_if_index0 == cached_sw_if_index) - { - trace_flags0 |= 0x20; - } - l2_output_dispatch (sm->vlib_main, - sm->vnet_main, - node, - node_index, - &cached_sw_if_index, - &cached_next_index, - next_nodes, - b0, sw_if_index0, feature_bitmap0, - &next0); - trace_flags0 |= 2; - - } - else - { - next0 = - feat_bitmap_get_next_node_index (input_feat_next_node_index, - feature_bitmap0); - trace_flags0 |= 4; - - } - - - - if (next0 >= node->n_next_nodes) - { - trace_flags0 |= 1; - } - - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - l2sess_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->trace_flags = trace_flags0; - t->l4_proto = l4_proto; - t->session_tables[0] = session_tables[0]; - t->session_tables[1] = session_tables[1]; - t->session_nexts[0] = session_nexts[0]; - t->session_nexts[1] = session_nexts[1]; - } - - } - pkts_swapped += 1; - if (next0 >= node->n_next_nodes) - { - next0 = 0; - } - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - vlib_node_increment_counter (vm, node->node_index, - L2SESS_ERROR_SWAPPED, pkts_swapped); - return frame->n_vectors; -} - - -#define _(node_name, node_var, is_out, is_ip6, is_track) \ -static uword \ -node_var ## node_fn (vlib_main_t * vm, \ - vlib_node_runtime_t * node, \ - vlib_frame_t * frame) \ -{ \ - return l2sess_node_fn(vm, node, frame); \ -} \ -VLIB_REGISTER_NODE (node_var) = { \ - .function = node_var ## node_fn, \ - .name = node_name, \ - .vector_size = sizeof (u32), \ - .format_trace = format_ ## node_var ## _trace, \ - .type = VLIB_NODE_TYPE_INTERNAL, \ - \ - .n_errors = ARRAY_LEN(l2sess_error_strings), \ - .error_strings = l2sess_error_strings, \ - \ - .n_next_nodes = L2SESS_N_NEXT, \ - .next_nodes = { \ - [L2SESS_NEXT_DROP] = "error-drop", \ - }, \ -}; -foreach_l2sess_node -#undef _ diff --git a/plugins/acl-plugin/acl/node_in.c b/plugins/acl-plugin/acl/node_in.c deleted file mode 100644 index 2a5199a9..00000000 --- a/plugins/acl-plugin/acl/node_in.c +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include "node_in.h" - -typedef struct -{ - u32 next_index; - u32 sw_if_index; - u32 match_acl_index; - u32 match_rule_index; - u32 trace_bitmap; -} acl_in_trace_t; - -/* packet trace format function */ -static u8 * -format_acl_in_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - acl_in_trace_t *t = va_arg (*args, acl_in_trace_t *); - - s = - format (s, - "ACL_IN: sw_if_index %d, next index %d, match: inacl %d rule %d trace_bits %08x", - t->sw_if_index, t->next_index, t->match_acl_index, - t->match_rule_index, t->trace_bitmap); - return s; -} - -vlib_node_registration_t acl_in_node; - -#define foreach_acl_in_error \ -_(ACL_CHECK, "InACL check packets processed") - -typedef enum -{ -#define _(sym,str) ACL_IN_ERROR_##sym, - foreach_acl_in_error -#undef _ - ACL_IN_N_ERROR, -} acl_in_error_t; - -static char *acl_in_error_strings[] = { -#define _(sym,string) string, - foreach_acl_in_error -#undef _ -}; - -static uword -acl_in_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - u32 n_left_from, *from, *to_next; - acl_in_next_t next_index; - u32 pkts_acl_checked = 0; - u32 feature_bitmap0; - u32 trace_bitmap = 0; - u32 *input_feat_next_node_index = - acl_main.acl_in_node_input_next_node_index; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0 = ~0; - u32 sw_if_index0; - u32 next = ~0; - u32 match_acl_index = ~0; - u32 match_rule_index = ~0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; - - input_acl_packet_match (sw_if_index0, b0, &next, &match_acl_index, - &match_rule_index, &trace_bitmap); - if (next != ~0) - { - next0 = next; - } - if (next0 == ~0) - { - next0 = - feat_bitmap_get_next_node_index (input_feat_next_node_index, - feature_bitmap0); - } - - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - acl_in_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->match_acl_index = match_acl_index; - t->match_rule_index = match_rule_index; - t->trace_bitmap = trace_bitmap; - } - - next0 = next0 < node->n_next_nodes ? next0 : 0; - - pkts_acl_checked += 1; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, acl_in_node.index, - ACL_IN_ERROR_ACL_CHECK, pkts_acl_checked); - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (acl_in_node) = -{ - .function = acl_in_node_fn,.name = "acl-plugin-in",.vector_size = - sizeof (u32),.format_trace = format_acl_in_trace,.type = - VLIB_NODE_TYPE_INTERNAL,.n_errors = - ARRAY_LEN (acl_in_error_strings),.error_strings = - acl_in_error_strings,.n_next_nodes = ACL_IN_N_NEXT, - /* edit / add dispositions here */ - .next_nodes = - { - [ACL_IN_ERROR_DROP] = "error-drop", - [ACL_IN_ETHERNET_INPUT] = "ethernet-input", - [ACL_IN_L2S_INPUT_IP4_ADD] = "aclp-l2s-input-ip4-add", - [ACL_IN_L2S_INPUT_IP6_ADD] = "aclp-l2s-input-ip6-add",} -,}; diff --git a/plugins/acl-plugin/acl/node_in.h b/plugins/acl-plugin/acl/node_in.h deleted file mode 100644 index 502bbf8d..00000000 --- a/plugins/acl-plugin/acl/node_in.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _NODE_IN_H_ -#define _NODE_IN_H_ - -typedef enum { - ACL_IN_ERROR_DROP, - ACL_IN_ETHERNET_INPUT, - ACL_IN_L2S_INPUT_IP4_ADD, - ACL_IN_L2S_INPUT_IP6_ADD, - ACL_IN_N_NEXT, -} acl_in_next_t; - -#endif diff --git a/plugins/acl-plugin/acl/node_out.c b/plugins/acl-plugin/acl/node_out.c deleted file mode 100644 index 50af3679..00000000 --- a/plugins/acl-plugin/acl/node_out.c +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include - -#include "node_out.h" - -typedef struct -{ - u32 next_index; - u32 sw_if_index; - u32 match_acl_index; - u32 match_rule_index; - u32 trace_bitmap; -} acl_out_trace_t; - -/* packet trace format function */ -static u8 * -format_acl_out_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - acl_out_trace_t *t = va_arg (*args, acl_out_trace_t *); - s = - format (s, - "ACL_OUT: sw_if_index %d, next index %d, match: outacl %d rule %d trace_bits %08x", - t->sw_if_index, t->next_index, t->match_acl_index, - t->match_rule_index, t->trace_bitmap); - return s; -} - -vlib_node_registration_t acl_out_node; - -#define foreach_acl_out_error \ -_(ACL_CHECK, "OutACL check packets processed") - -typedef enum -{ -#define _(sym,str) ACL_OUT_ERROR_##sym, - foreach_acl_out_error -#undef _ - ACL_OUT_N_ERROR, -} acl_out_error_t; - -static char *acl_out_error_strings[] = { -#define _(sym,string) string, - foreach_acl_out_error -#undef _ -}; - -static uword -acl_out_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - acl_main_t *am = &acl_main; - l2_output_next_nodes_st *next_nodes = &am->acl_out_output_next_nodes; - u32 n_left_from, *from, *to_next; - acl_out_next_t next_index; - u32 pkts_acl_checked = 0; - u32 feature_bitmap0; - u32 cached_sw_if_index = (u32) ~ 0; - u32 cached_next_index = (u32) ~ 0; - u32 match_acl_index = ~0; - u32 match_rule_index = ~0; - u32 trace_bitmap = 0; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0 = ~0; - u32 next = 0; - u32 sw_if_index0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; - - output_acl_packet_match (sw_if_index0, b0, &next, &match_acl_index, - &match_rule_index, &trace_bitmap); - if (next != ~0) - { - next0 = next; - } - if (next0 == ~0) - { - l2_output_dispatch (vm, - am->vnet_main, - node, - acl_out_node.index, - &cached_sw_if_index, - &cached_next_index, - next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); - } - - - - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - acl_out_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->match_acl_index = match_acl_index; - t->match_rule_index = match_rule_index; - t->trace_bitmap = trace_bitmap; - } - - pkts_acl_checked += 1; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, acl_out_node.index, - ACL_OUT_ERROR_ACL_CHECK, pkts_acl_checked); - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (acl_out_node) = -{ - .function = acl_out_node_fn,.name = "acl-plugin-out",.vector_size = - sizeof (u32),.format_trace = format_acl_out_trace,.type = - VLIB_NODE_TYPE_INTERNAL,.n_errors = - ARRAY_LEN (acl_out_error_strings),.error_strings = - acl_out_error_strings,.n_next_nodes = ACL_OUT_N_NEXT, - /* edit / add dispositions here */ - .next_nodes = - { - [ACL_OUT_ERROR_DROP] = "error-drop", - [ACL_OUT_INTERFACE_OUTPUT] = "interface-output", - [ACL_OUT_L2S_OUTPUT_IP4_ADD] = "aclp-l2s-output-ip4-add", - [ACL_OUT_L2S_OUTPUT_IP6_ADD] = "aclp-l2s-output-ip6-add",} -,}; diff --git a/plugins/acl-plugin/acl/node_out.h b/plugins/acl-plugin/acl/node_out.h deleted file mode 100644 index c919f3b7..00000000 --- a/plugins/acl-plugin/acl/node_out.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _NODE_OUT_H_ -#define _NODE_OUT_H_ - -typedef enum { - ACL_OUT_ERROR_DROP, - ACL_OUT_INTERFACE_OUTPUT, - ACL_OUT_L2S_OUTPUT_IP4_ADD, - ACL_OUT_L2S_OUTPUT_IP6_ADD, - ACL_OUT_N_NEXT, -} acl_out_next_t; - -#endif diff --git a/plugins/acl-plugin/configure.ac b/plugins/acl-plugin/configure.ac deleted file mode 100644 index 204f4e4d..00000000 --- a/plugins/acl-plugin/configure.ac +++ /dev/null @@ -1,24 +0,0 @@ -AC_INIT(acl_plugin, 1.0) -AC_CONFIG_MACRO_DIR([../../vpp-api/java/m4]) -LT_INIT -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) -AC_PREFIX_DEFAULT([/usr]) - -AC_PROG_CC - -if test -f /usr/bin/lsb_release && test `lsb_release -si` == "Ubuntu" && test `lsb_release -sr` == "14.04" && test -d /usr/lib/jvm/java-8-openjdk-amd64/ ; then - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ - JAVAC=${JAVA_HOME}/bin/javac - PATH=${JAVA_HOME}/bin/:${PATH} - break -fi - -AX_CHECK_JAVA_HOME -AX_PROG_JAVAC -AX_PROG_JAVAH -AX_PROG_JAR -AX_PROG_JAVADOC -AX_PROG_JAVA - -AC_OUTPUT([Makefile]) diff --git a/plugins/acl-plugin/test/run-python b/plugins/acl-plugin/test/run-python deleted file mode 100755 index 215eb17a..00000000 --- a/plugins/acl-plugin/test/run-python +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# -# Do all the legwork to run a scapy shell with APIs available for load -# -CURR_DIR=`pwd` -ROOT_DIR=`git rev-parse --show-toplevel` -cd $ROOT_DIR -sudo apt-get install -y python-virtualenv -# uncomment the line below to enable build of plugins and api each time -# make plugins && make build-vpp-api || exit -virtualenv virtualenv -virtualenv/bin/pip install ipaddress -virtualenv/bin/pip install scapy -# install the python API into the virtualenv -cd $ROOT_DIR/vpp-api/python/ -$ROOT_DIR/virtualenv/bin/python setup.py install -# install the python ACL plugin API into the virtualenv -ACL_PLUGIN_SETUP_DIR=`find $ROOT_DIR/build-root -name acl-plugin` -cd $ACL_PLUGIN_SETUP_DIR; -$ROOT_DIR/virtualenv/bin/python setup.py install -cd $ROOT_DIR -# figure out the shared library path and start scapy -export LD_LIBRARY_PATH=`pwd`/`find . -name "libpneum.so" -exec dirname {} \; | grep lib64 | head -n 1` -cd $CURR_DIR -sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH $ROOT_DIR/virtualenv/bin/python $1 $2 $3 $4 $5 $6 $7 $8 $9 - - - diff --git a/plugins/acl-plugin/test/run-scapy b/plugins/acl-plugin/test/run-scapy deleted file mode 100755 index 266f07d1..00000000 --- a/plugins/acl-plugin/test/run-scapy +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -# Do all the legwork to run a scapy shell with APIs available for load -# -ROOT_DIR=`git rev-parse --show-toplevel` -cd $ROOT_DIR -sudo apt-get install -y python-virtualenv -# uncomment the line below to enable the build of plugins and API each time.. -# make plugins && make build-vpp-api || exit -virtualenv virtualenv -virtualenv/bin/pip install ipaddress -virtualenv/bin/pip install scapy -# install the python API into the virtualenv -cd $ROOT_DIR/vpp-api/python/ -$ROOT_DIR/virtualenv/bin/python setup.py install -# install the python ACL plugin API into the virtualenv -ACL_PLUGIN_SETUP_DIR=`find $ROOT_DIR/build-root -name acl-plugin` -cd $ACL_PLUGIN_SETUP_DIR; -$ROOT_DIR/virtualenv/bin/python setup.py install -cd $ROOT_DIR -# figure out the shared library path and start scapy -export LD_LIBRARY_PATH=`pwd`/`find . -name "libpneum.so" -exec dirname {} \; | grep lib64 | head -n 1` -sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH virtualenv/bin/scapy - - - diff --git a/plugins/acl-plugin/test/test_acl_plugin.py b/plugins/acl-plugin/test/test_acl_plugin.py deleted file mode 100644 index 7fc72d67..00000000 --- a/plugins/acl-plugin/test/test_acl_plugin.py +++ /dev/null @@ -1,118 +0,0 @@ -from __future__ import print_function -import unittest, sys, time, threading, struct, logging, os -import vpp_papi -# import vpp_papi_plugins.acl -from ipaddress import * -papi_event = threading.Event() -print(vpp_papi.vpe.VL_API_SW_INTERFACE_SET_FLAGS) -def papi_event_handler(result): - if result.vl_msg_id == vpp_papi.vpe.VL_API_SW_INTERFACE_SET_FLAGS: - return - if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_INTERFACE_COUNTERS: - print('Interface counters', result) - return - if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_IP6_FIB_COUNTERS: - print('IPv6 FIB counters', result) - papi_event.set() - return - - print('Unknown message id:', result.vl_msg_id) - -import glob, subprocess -class TestAclPlugin(unittest.TestCase): - @classmethod - def setUpClass(cls): - print("Setup") - @classmethod - def tearDownClass(cls): - print("Teardown") - - def setUp(self): - print("Connecting API") - r = vpp_papi.connect("test_papi") - self.assertEqual(r, 0) - - def tearDown(self): - r = vpp_papi.disconnect() - self.assertEqual(r, 0) - - # - # The tests themselves - # - - # - # Basic request / reply - # - def test_show_version(self): - t = vpp_papi.show_version() - print('T', t); - program = t.program.decode().rstrip('\x00') - self.assertEqual('vpe', program) - - def x_test_acl_add(self): - print("Test ACL add") - self.assertEqual(1, 1) - - # - # Details / Dump - # - def x_test_details_dump(self): - t = vpp_papi.sw_interface_dump(0, b'') - print('Dump/details T', t) - - # - # Arrays - # - def x_test_arrays(self): - t = vpp_papi.vnet_get_summary_stats() - print('Summary stats', t) - print('Packets:', t.total_pkts[0]) - print('Packets:', t.total_pkts[1]) - # - # Variable sized arrays and counters - # - #@unittest.skip("stats") - def x_test_want_stats(self): - pid = 123 - vpp_papi.register_event_callback(papi_event_handler) - papi_event.clear() - - # Need to configure IPv6 to get som IPv6 FIB stats - t = vpp_papi.create_loopback('') - print(t) - self.assertEqual(t.retval, 0) - - ifindex = t.sw_if_index - addr = str(IPv6Address(u'1::1').packed) - t = vpp_papi.sw_interface_add_del_address(ifindex, 1, 1, 0, 16, addr) - print(t) - self.assertEqual(t.retval, 0) - - # Check if interface is up - # XXX: Add new API to query interface state based on ifindex, instead of dump all. - t = vpp_papi.sw_interface_set_flags(ifindex, 1, 1, 0) - self.assertEqual(t.retval, 0) - - t = vpp_papi.want_stats(True, pid) - - print (t) - - # - # Wait for some stats - # - self.assertEqual(papi_event.wait(15), True) - t = vpp_papi.want_stats(False, pid) - print (t) - - - # - # Plugins? - # - -if __name__ == '__main__' or __name__ == '__builtin__': - print("This is main") - suite = unittest.TestLoader().loadTestsFromTestCase(TestAclPlugin) - unittest.TextTestRunner(verbosity=2).run(suite) - #logging.basicConfig(level=logging.DEBUG) - # unittest.main() - diff --git a/plugins/configure.ac b/plugins/configure.ac index 9c631634..b6ba4152 100644 --- a/plugins/configure.ac +++ b/plugins/configure.ac @@ -53,11 +53,6 @@ AM_CONDITIONAL(ENABLE_$1_PLUGIN, test "$enable_the_plugin" = "1") # SUBDIRS += new-plugin # endif -PLUGIN_ENABLED(ioam) -PLUGIN_ENABLED(snat) -PLUGIN_ENABLED(lb) -PLUGIN_ENABLED(acl) - # Disabled plugins, require --enable-XXX-plugin PLUGIN_DISABLED(vcgn) PLUGIN_DISABLED(sample) diff --git a/plugins/ioam-plugin/Makefile.am b/plugins/ioam-plugin/Makefile.am deleted file mode 100644 index b00b088c..00000000 --- a/plugins/ioam-plugin/Makefile.am +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright (c) 2015 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AUTOMAKE_OPTIONS = foreign subdir-objects - -AM_CFLAGS = -Wall -AM_LDFLAGS = -module -shared -avoid-version - -######################################## -# iOAM Proof of Transit -######################################## - -ioam_pot_plugin_la_SOURCES = \ - ioam/lib-pot/pot_util.c \ - ioam/encap/ip6_ioam_pot.c \ - ioam/lib-pot/pot_util.h \ - ioam/lib-pot/math64.h \ - ioam/lib-pot/pot_api.c - -BUILT_SOURCES = \ - ioam/lib-pot/pot.api.h \ - ioam/lib-pot/pot.api.json - -SUFFIXES = .api.h .api - -%.api.h: %.api - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --output $@ --show-name $@ - -%.api.json: %.api - @echo " JSON APIGEN " $@ ; \ - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --json $@ - -apidir = $(prefix)/ioam/ -api_DATA = \ - ioam/lib-pot/pot.api.json \ - ioam/lib-trace/trace.api.json \ - ioam/export/ioam_export.api.json - -noinst_HEADERS = \ - ioam/lib-pot/pot_all_api_h.h \ - ioam/lib-pot/pot_msg_enum.h \ - ioam/lib-pot/pot.api.h \ - ioam/lib-pot/pot_util.h \ - ioam/lib-pot/math64.h - -ioam_pot_test_plugin_la_SOURCES = \ - ioam/lib-pot/pot_test.c \ - ioam/lib-pot/pot_plugin.api.h - -vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins -vpppluginsdir = ${libdir}/vpp_plugins - -vppapitestplugins_LTLIBRARIES = ioam_pot_test_plugin.la -vppplugins_LTLIBRARIES = ioam_pot_plugin.la - -######################################## -# iOAM trace export for IPv6 -######################################## - -ioam_export_plugin_la_SOURCES = \ -ioam/export/ioam_export.c \ -ioam/export/node.c \ -ioam/export/ioam_export.api.h \ -ioam/export/ioam_export_thread.c - -BUILT_SOURCES += \ - ioam/export/ioam_export.api.h \ - ioam/export/ioam_export.api.json - -noinst_HEADERS += \ - ioam/export/ioam_export_all_api_h.h \ - ioam/export/ioam_export_msg_enum.h \ - ioam/export/ioam_export.api.h - -ioam_export_test_plugin_la_SOURCES = \ - ioam/export/ioam_export_test.c \ - ioam/export/ioam_export_plugin.api.h - -vppapitestplugins_LTLIBRARIES += ioam_export_test_plugin.la -vppplugins_LTLIBRARIES += ioam_export_plugin.la - -######################################## -# iOAM Trace -######################################## -libioam_trace_plugin_la_SOURCES = \ - ioam/lib-trace/trace_util.c \ - ioam/encap/ip6_ioam_trace.c \ - ioam/lib-trace/trace_util.h \ - ioam/lib-trace/trace_api.c - -BUILT_SOURCES += \ - ioam/lib-trace/trace.api.h \ - ioam/lib-trace/trace.api.json - -noinst_HEADERS += \ - ioam/export/ioam_export_all_api_h.h \ - ioam/lib-trace/trace_all_api_h.h \ - ioam/lib-trace/trace_msg_enum.h \ - ioam/lib-trace/trace.api.h \ - ioam/lib-trace/trace_util.h - -ioam_trace_test_plugin_la_SOURCES = \ - ioam/lib-trace/trace_test.c \ - ioam/lib-trace/trace_plugin.api.h - -vppapitestplugins_LTLIBRARIES += ioam_trace_test_plugin.la -vppplugins_LTLIBRARIES += libioam_trace_plugin.la - -######################################## -# VxLAN-GPE -######################################## -libioam_vxlan_gpe_plugin_la_SOURCES = \ - ioam/lib-vxlan-gpe/ioam_encap.c \ - ioam/lib-vxlan-gpe/ioam_decap.c \ - ioam/lib-vxlan-gpe/ioam_transit.c \ - ioam/lib-vxlan-gpe/ioam_pop.c \ - ioam/lib-vxlan-gpe/vxlan_gpe_api.c \ - ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c \ - ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c \ - ioam/export-vxlan-gpe/vxlan_gpe_node.c \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api.h\ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c - -BUILT_SOURCES += \ - ioam/lib-vxlan-gpe/vxlan_gpe.api.h \ - ioam/lib-vxlan-gpe/vxlan_gpe.api.json \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api.h \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api.json - -noinst_HEADERS += \ - ioam/export/ioam_export_all_api_h.h \ - ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h \ - ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h \ - ioam/lib-vxlan-gpe/vxlan_gpe.api.h \ - ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h \ - ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h \ - ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api.h - -ioam_vxlan_gpe_test_plugin_la_SOURCES = \ - ioam/lib-vxlan-gpe/vxlan_gpe_test.c \ - ioam/lib-vxlan-gpe/vxlan_gpe_plugin.api.h - -libioam_vxlan_gpe_plugin_la_LIBADD = libioam_trace_plugin.la - -vppapitestplugins_LTLIBRARIES += ioam_vxlan_gpe_test_plugin.la -vppplugins_LTLIBRARIES += libioam_vxlan_gpe_plugin.la - -vxlan_gpe_ioam_export_test_plugin_la_SOURCES = \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c \ - ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_plugin.api.h - -vppapitestplugins_LTLIBRARIES += vxlan_gpe_ioam_export_test_plugin.la - -######################################## -# iOAM E2E plugin -######################################## - -ioam_e2e_plugin_la_SOURCES = \ - ioam/encap/ip6_ioam_e2e.c \ - ioam/encap/ip6_ioam_seqno.c \ - ioam/encap/ip6_ioam_seqno_analyse.c - -noinst_HEADERS += \ - ioam/encap/ip6_ioam_e2e.h \ - ioam/encap/ip6_ioam_seqno.h - -vppplugins_LTLIBRARIES += ioam_e2e_plugin.la - -# Remove *.la files -install-data-hook: - @(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) - @(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) - - -# -# Java code generation -# -jvpp_registry_root = ../../vpp-api/java -jvpp_registry_version = 17.04 -jioam_trace_jarfile = jvpp-ioam-trace-$(PACKAGE_VERSION).jar -jvpp_trace_package_dir = io/fd/vpp/jvpp/ioamtrace -jvpp_root = ioam/jvpp -jvpp_target_dir = target -jvpp_target = $(jvpp_root)/$(jvpp_target_dir) -trace_api_file=$(srcdir)/ioam/lib-trace/trace.api - - -lib_LTLIBRARIES = libjvpp_ioamtrace.la -libjvpp_ioamtrace_la_SOURCES = ioam/lib-trace/trace.api.h ioam/lib-trace/jvpp_ioam_trace.c ioam/jvpp/io_fd_vpp_jvpp_ioam_trace_JVppIoamTraceImpl.h -libjvpp_ioamtrace_la_LIBADD = -lvlibmemoryclient -lvlibapi -lvppinfra \ - -lpthread -lm -lrt -L$(jvpp_registry_root)/.libs -ljvpp_common -libjvpp_ioamtrace_la_LDFLAGS = -module -libjvpp_ioamtrace_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -I../ -I$(srcdir)/../ - -BUILT_SOURCES += $(jvpp_root)/io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl.h - -$(jvpp_root)/io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl.h: ioam_trace.api.json - dir=`pwd`; \ - mkdir -p $(jvpp_target); \ - mkdir -p $(jvpp_root)/$(jvpp_trace_package_dir); \ - cd $(jvpp_root)/$(jvpp_trace_package_dir); \ - mkdir -p dto future callfacade callback notification test; \ - @srcdir@/$(jvpp_registry_root)/jvpp/gen/jvpp_gen.py -i $${dir}/ioam_trace.api.json --plugin_name ioamtrace; \ - cd -; \ - mv -f $(jvpp_root)/$(jvpp_trace_package_dir)/jvpp_ioamtrace_gen.h $(jvpp_root)/jvpp_ioam_trace_gen.h; \ - cp $(srcdir)/$(jvpp_root)/$(jvpp_trace_package_dir)/test/*.java $(jvpp_root)/$(jvpp_trace_package_dir)/test/; \ - cd $(jvpp_root); \ - $(JAVAC) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d $(jvpp_target_dir) $(jvpp_trace_package_dir)/*.java \ - $(jvpp_trace_package_dir)/dto/*.java \ - $(jvpp_trace_package_dir)/callback/*.java \ - $(jvpp_trace_package_dir)/notification/*.java \ - $(jvpp_trace_package_dir)/future/*.java \ - $(jvpp_trace_package_dir)/callfacade/*.java \ - $(jvpp_trace_package_dir)/test/*.java \ - || (echo "ioam trace jvpp compilation failed: $$?"; exit 1); \ - $(JAVAH) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d . io.fd.vpp.jvpp.ioamtrace.JVppIoamtraceImpl ; - -$(jioam_trace_jarfile): libjvpp_ioamtrace.la - cp .libs/libjvpp_ioamtrace.so.0.0.0 $(jvpp_target); \ - cd $(jvpp_target); \ - $(JAR) cfv $(JARFLAGS) ../../../$@ libjvpp_ioamtrace.so.0.0.0 $(jvpp_trace_package_dir)/* ; cd ..; - -ioam_trace.api.json: - @echo " jIoam_trace API"; \ - vppapigen --input $(trace_api_file) --json ioam_trace.api.json; - -all-local: $(jioam_trace_jarfile) - - -jioam_pot_jarfile = jvpp-ioam-pot-$(PACKAGE_VERSION).jar -jvpp_pot_package_dir = io/fd/vpp/jvpp/ioampot -pot_api_file=$(srcdir)/ioam/lib-pot/pot.api - - -lib_LTLIBRARIES += libjvpp_ioampot.la -libjvpp_ioampot_la_SOURCES = ioam/lib-pot/pot.api.h ioam/lib-pot/jvpp_ioam_pot.c ioam/jvpp/io_fd_vpp_jvpp_ioam_pot_JVppIoamPotImpl.h -libjvpp_ioampot_la_LIBADD = -lvlibmemoryclient -lvlibapi -lvppinfra \ - -lpthread -lm -lrt -L$(jvpp_registry_root)/.libs -ljvpp_common -libjvpp_ioampot_la_LDFLAGS = -module -libjvpp_ioampot_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -I../ -I$(srcdir)/../ - -BUILT_SOURCES += $(jvpp_root)/io_fd_vpp_jvpp_ioampot_JVppIoampotImpl.h - -$(jvpp_root)/io_fd_vpp_jvpp_ioampot_JVppIoampotImpl.h: ioam_pot.api.json - dir=`pwd`; \ - mkdir -p $(jvpp_target); \ - mkdir -p $(jvpp_root)/$(jvpp_pot_package_dir); \ - cd $(jvpp_root)/$(jvpp_pot_package_dir); \ - mkdir -p dto future callfacade callback notification test; \ - @srcdir@/$(jvpp_registry_root)/jvpp/gen/jvpp_gen.py -i $${dir}/ioam_pot.api.json --plugin_name ioampot; \ - cd -; \ - mv -f $(jvpp_root)/$(jvpp_pot_package_dir)/jvpp_ioampot_gen.h $(jvpp_root)/jvpp_ioam_pot_gen.h; \ - cp $(srcdir)/$(jvpp_root)/$(jvpp_pot_package_dir)/test/*.java $(jvpp_root)/$(jvpp_pot_package_dir)/test/; \ - cd $(jvpp_root); \ - $(JAVAC) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d $(jvpp_target_dir) $(jvpp_pot_package_dir)/*.java \ - $(jvpp_pot_package_dir)/dto/*.java \ - $(jvpp_pot_package_dir)/callback/*.java \ - $(jvpp_pot_package_dir)/notification/*.java \ - $(jvpp_pot_package_dir)/future/*.java \ - $(jvpp_pot_package_dir)/callfacade/*.java \ - $(jvpp_pot_package_dir)/test/*.java \ - || (echo "ioam pot jvpp compilation failed: $$?"; exit 1); \ - $(JAVAH) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d . io.fd.vpp.jvpp.ioampot.JVppIoampotImpl ; - -$(jioam_pot_jarfile): libjvpp_ioampot.la - cp .libs/libjvpp_ioampot.so.0.0.0 $(jvpp_target); \ - cd $(jvpp_target); \ - $(JAR) cfv $(JARFLAGS) ../../../$@ libjvpp_ioampot.so.0.0.0 $(jvpp_pot_package_dir)/* ; cd ..; - -ioam_pot.api.json: - @echo " jIoam_pot API"; \ - vppapigen --input $(pot_api_file) --json ioam_pot.api.json; - -all-local: $(jioam_pot_jarfile) - -jioam_export_jarfile = jvpp-ioam-export-$(PACKAGE_VERSION).jar -jvpp_export_package_dir = io/fd/vpp/jvpp/ioamexport -export_api_file=$(srcdir)/ioam/export/ioam_export.api - - -lib_LTLIBRARIES += libjvpp_ioamexport.la -libjvpp_ioamexport_la_SOURCES = ioam/export/export.api.h ioam/export/jvpp_ioam_export.c ioam/jvpp/io_fd_vpp_jvpp_ioam_export_JVppIoamexportImpl.h -libjvpp_ioamexport_la_LIBADD = -lvlibmemoryclient -lvlibapi -lvppinfra \ - -lpthread -lm -lrt -L$(jvpp_registry_root)/.libs -ljvpp_common -libjvpp_ioamexport_la_LDFLAGS = -module -libjvpp_ioamexport_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -I../ -I$(srcdir)/../ - -BUILT_SOURCES += $(jvpp_root)/io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl.h - -$(jvpp_root)/io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl.h: ioam_export.api.json - dir=`pwd`; \ - mkdir -p $(jvpp_target); \ - mkdir -p $(jvpp_root)/$(jvpp_export_package_dir); \ - cd $(jvpp_root)/$(jvpp_export_package_dir); \ - mkdir -p dto future callfacade callback notification test; \ - @srcdir@/$(jvpp_registry_root)/jvpp/gen/jvpp_gen.py -i $${dir}/ioam_export.api.json --plugin_name ioamexport; \ - cd -; \ - mv -f $(jvpp_root)/$(jvpp_export_package_dir)/jvpp_ioamexport_gen.h $(jvpp_root)/jvpp_ioam_export_gen.h; \ - cp $(srcdir)/$(jvpp_root)/$(jvpp_export_package_dir)/test/*.java $(jvpp_root)/$(jvpp_export_package_dir)/test/; \ - cd $(jvpp_root); \ - $(JAVAC) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d $(jvpp_target_dir) $(jvpp_export_package_dir)/*.java \ - $(jvpp_export_package_dir)/dto/*.java \ - $(jvpp_export_package_dir)/callback/*.java \ - $(jvpp_export_package_dir)/notification/*.java \ - $(jvpp_export_package_dir)/future/*.java \ - $(jvpp_export_package_dir)/callfacade/*.java \ - $(jvpp_export_package_dir)/test/*.java \ - || (echo "ioam export jvpp compilation failed: $$?"; exit 1); \ - $(JAVAH) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d . io.fd.vpp.jvpp.ioamexport.JVppIoamexportImpl ; - -$(jioam_export_jarfile): libjvpp_ioamexport.la - cp .libs/libjvpp_ioamexport.so.0.0.0 $(jvpp_target); \ - cd $(jvpp_target); \ - $(JAR) cfv $(JARFLAGS) ../../../$@ libjvpp_ioamexport.so.0.0.0 $(jvpp_export_package_dir)/* ; cd ..; - -ioam_export.api.json: - @echo " jIoam_export API"; \ - vppapigen --input $(export_api_file) --json ioam_export.api.json; - -all-local: $(jioam_export_jarfile) diff --git a/plugins/ioam-plugin/configure.ac b/plugins/ioam-plugin/configure.ac deleted file mode 100644 index 287bcaab..00000000 --- a/plugins/ioam-plugin/configure.ac +++ /dev/null @@ -1,25 +0,0 @@ -AC_INIT(ioam_plugin, 1.0) -LT_INIT -AC_CONFIG_MACRO_DIR([../../vpp-api/java/m4]) -AM_INIT_AUTOMAKE -AC_PREFIX_DEFAULT([/usr]) -AM_SILENT_RULES([yes]) - -AC_PROG_CC - -if test -f /usr/bin/lsb_release && test `lsb_release -si` == "Ubuntu" && test `lsb_release -sr` == "14.04" && test -d /usr/lib/jvm/java-8-openjdk-amd64/ ; then - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ - JAVAC=${JAVA_HOME}/bin/javac - PATH=${JAVA_HOME}/bin/:${PATH} - break -fi - -AX_CHECK_JAVA_HOME -AX_PROG_JAVAC -AX_PROG_JAVAH -AX_PROG_JAR -AX_PROG_JAVADOC -AX_PROG_JAVA - - -AC_OUTPUT([Makefile]) diff --git a/plugins/ioam-plugin/ioam/dir.dox b/plugins/ioam-plugin/ioam/dir.dox deleted file mode 100644 index f3389b52..00000000 --- a/plugins/ioam-plugin/ioam/dir.dox +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - @dir - @brief Inband OAM (iOAM) implementation -*/ diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.c deleted file mode 100644 index 0839cdce..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.c +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include - -#include "ip6_ioam_e2e.h" - -ioam_e2e_main_t ioam_e2e_main; - -static u8 * ioam_e2e_trace_handler (u8 * s, - ip6_hop_by_hop_option_t *opt) -{ - ioam_e2e_option_t * e2e = (ioam_e2e_option_t *)opt; - u32 seqno = 0; - - if (e2e) - { - seqno = clib_net_to_host_u32 (e2e->e2e_data); - } - - s = format (s, "SeqNo = 0x%Lx", seqno); - return s; -} - -int -ioam_e2e_config_handler (void *data, u8 disable) -{ - int *analyse = data; - - /* Register hanlders if enabled */ - if (!disable) - { - /* If encap node register for encap handler */ - if (0 == *analyse) - { - if (ip6_hbh_register_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, - ioam_seqno_encap_handler, - ioam_e2e_trace_handler) < 0) - { - return (-1); - } - } - /* If analyze node then register for decap handler */ - else - { - if (ip6_hbh_pop_register_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, - ioam_seqno_decap_handler) < 0) - { - return (-1); - } - } - return 0; - } - - /* UnRegister handlers */ - (void) ip6_hbh_unregister_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE); - (void) ip6_hbh_pop_unregister_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE); - return 0; -} - -int -ioam_e2e_rewrite_handler (u8 *rewrite_string, - u8 *rewrite_size) -{ - ioam_e2e_option_t *e2e_option; - - if (rewrite_string && *rewrite_size == sizeof(ioam_e2e_option_t)) - { - e2e_option = (ioam_e2e_option_t *)rewrite_string; - e2e_option->hdr.type = HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE - | HBH_OPTION_TYPE_SKIP_UNKNOWN; - e2e_option->hdr.length = sizeof (ioam_e2e_option_t) - - sizeof (ip6_hop_by_hop_option_t); - return(0); - } - return(-1); -} - -u32 -ioam_e2e_flow_handler (u32 ctx, u8 add) -{ - ioam_e2e_data_t *data; - u16 i; - - if (add) - { - pool_get(ioam_e2e_main.e2e_data, data); - data->flow_ctx = ctx; - ioam_seqno_init_bitmap(&data->seqno_data); - return ((u32) (data - ioam_e2e_main.e2e_data)); - } - - /* Delete case */ - for (i = 0; i < vec_len(ioam_e2e_main.e2e_data); i++) - { - if (pool_is_free_index(ioam_e2e_main.e2e_data, i)) - continue; - - data = pool_elt_at_index(ioam_e2e_main.e2e_data, i); - if (data && (data->flow_ctx == ctx)) - { - pool_put_index(ioam_e2e_main.e2e_data, i); - return (0); - } - } - return 0; -} - -static clib_error_t * -ioam_show_e2e_cmd_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - ioam_e2e_data_t *e2e_data; - u8 *s = 0; - int i; - - vec_reset_length(s); - - s = format(0, "IOAM E2E information: \n"); - for (i = 0; i < vec_len(ioam_e2e_main.e2e_data); i++) - { - if (pool_is_free_index(ioam_e2e_main.e2e_data, i)) - continue; - - e2e_data = pool_elt_at_index(ioam_e2e_main.e2e_data, i); - s = format(s, "Flow name: %s\n", get_flow_name_from_flow_ctx(e2e_data->flow_ctx)); - - s = show_ioam_seqno_cmd_fn(s, - &e2e_data->seqno_data, - !IOAM_DEAP_ENABLED(e2e_data->flow_ctx)); - } - - vlib_cli_output(vm, "%v", s); - return 0; -} - - -VLIB_CLI_COMMAND (ioam_show_e2e_cmd, static) = { - .path = "show ioam e2e ", - .short_help = "show ioam e2e information", - .function = ioam_show_e2e_cmd_fn, -}; - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - clib_error_t * error = 0; - - ioam_e2e_main.vlib_main = vm; - ioam_e2e_main.vnet_main = h->vnet_main; - return error; -} - -/* - * Init handler E2E headet handling. - * Init hanlder registers encap, decap, trace and Rewrite handlers. - */ -static clib_error_t * -ioam_e2e_init (vlib_main_t * vm) -{ - clib_error_t * error; - - if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init))) - { - return(error); - } - - /* - * As of now we have only PPC under E2E header. - */ - if (ip6_hbh_config_handler_register(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, - ioam_e2e_config_handler) < 0) - { - return (clib_error_create("Registration of " - "HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE for rewrite failed")); - } - - if (ip6_hbh_add_register_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, - sizeof(ioam_e2e_option_t), - ioam_e2e_rewrite_handler) < 0) - { - return (clib_error_create("Registration of " - "HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE for rewrite failed")); - } - - if (ip6_hbh_flow_handler_register(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, - ioam_e2e_flow_handler) < 0) - { - return (clib_error_create("Registration of " - "HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE Flow handler failed")); - } - - return (0); -} - -/* - * Init function for the E2E lib. - * ip6_hop_by_hop_ioam_e2e_init gets called during init. - */ -VLIB_INIT_FUNCTION (ioam_e2e_init); diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.h b/plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.h deleted file mode 100644 index 18f35f80..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_e2e.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __included_ip6_ioam_e2e_h__ -#define __included_ip6_ioam_e2e_h__ - -#include "ip6_ioam_seqno.h" - -typedef struct ioam_e2e_data_t_ { - u32 flow_ctx; - u32 pad; - ioam_seqno_data seqno_data; -} ioam_e2e_data_t; - -typedef struct { - ioam_e2e_data_t *e2e_data; - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; -} ioam_e2e_main_t; - -extern ioam_e2e_main_t ioam_e2e_main; - -static inline ioam_seqno_data * -ioam_e2ec_get_seqno_data_from_flow_ctx (u32 flow_ctx) -{ - ioam_e2e_data_t *data = NULL; - u32 index; - - index = get_flow_data_from_flow_ctx(flow_ctx, - HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE); - data = &ioam_e2e_main.e2e_data[index]; - return &(data->seqno_data); -} - -#endif /* __included_ioam_e2e_h__ */ diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c deleted file mode 100644 index 05f42c91..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_pot.c +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include - -typedef CLIB_PACKED(struct { - ip6_hop_by_hop_option_t hdr; - u8 pot_type; -#define PROFILE_ID_MASK 0xF - u8 reserved_profile_id; /* 4 bits reserved, 4 bits to carry profile id */ - u64 random; - u64 cumulative; -}) ioam_pot_option_t; - -#define foreach_ip6_hop_by_hop_ioam_pot_stats \ - _(PROCESSED, "Pkts with ip6 hop-by-hop pot options") \ - _(PROFILE_MISS, "Pkts with ip6 hop-by-hop pot options but no profile set") \ - _(PASSED, "Pkts with POT in Policy") \ - _(FAILED, "Pkts with POT out of Policy") - -static char * ip6_hop_by_hop_ioam_pot_stats_strings[] = { -#define _(sym,string) string, - foreach_ip6_hop_by_hop_ioam_pot_stats -#undef _ -}; - -typedef enum { -#define _(sym,str) IP6_IOAM_POT_##sym, - foreach_ip6_hop_by_hop_ioam_pot_stats -#undef _ - IP6_IOAM_POT_N_STATS, -} ip6_ioam_pot_stats_t; - -typedef struct { - /* stats */ - u64 counters[ARRAY_LEN(ip6_hop_by_hop_ioam_pot_stats_strings)]; - - /* convenience */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; -} ip6_hop_by_hop_ioam_pot_main_t; - -ip6_hop_by_hop_ioam_pot_main_t ip6_hop_by_hop_ioam_pot_main; - -always_inline void -ip6_ioam_stats_increment_counter (u32 counter_index, u64 increment) -{ - ip6_hop_by_hop_ioam_pot_main_t *hm = &ip6_hop_by_hop_ioam_pot_main; - - hm->counters[counter_index] += increment; -} - - -static u8 * format_ioam_pot (u8 * s, va_list * args) -{ - ioam_pot_option_t * pot0 = va_arg (*args, ioam_pot_option_t *); - u64 random, cumulative; - random = cumulative = 0; - if (pot0) - { - random = clib_net_to_host_u64 (pot0->random); - cumulative = clib_net_to_host_u64 (pot0->cumulative); - } - - s = format (s, "random = 0x%Lx, Cumulative = 0x%Lx, Index = 0x%x", - random, cumulative, pot0 ? pot0->reserved_profile_id : ~0); - return s; -} - -u8 * -ip6_hbh_ioam_proof_of_transit_trace_handler (u8 *s, ip6_hop_by_hop_option_t *opt) -{ - ioam_pot_option_t *pot; - - s = format (s, " POT opt present\n"); - pot = (ioam_pot_option_t *) opt; - s = format (s, " %U\n", format_ioam_pot, pot); - return (s); -} - -int -ip6_hbh_ioam_proof_of_transit_handler (vlib_buffer_t *b, - ip6_header_t *ip, - ip6_hop_by_hop_option_t *opt0) -{ - ioam_pot_option_t * pot0; - u64 random = 0, cumulative = 0; - int rv = 0; - u8 pot_profile_index; - pot_profile *pot_profile = 0, *new_profile = 0; - u8 pot_encap = 0; - - pot0 = (ioam_pot_option_t *) opt0; - pot_encap = (pot0->random == 0); - pot_profile_index = pot_profile_get_active_id(); - pot_profile = pot_profile_get_active(); - if (pot_encap && PREDICT_FALSE(!pot_profile)) - { - ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PROFILE_MISS, 1); - return(-1); - } - if (pot_encap) - { - pot0->reserved_profile_id = - pot_profile_index & PROFILE_ID_MASK; - pot_profile_incr_usage_stats(pot_profile); - } - else - { /* Non encap node */ - if (PREDICT_FALSE(pot0->reserved_profile_id != - pot_profile_index || pot_profile == 0)) - { - /* New profile announced by encap node. */ - new_profile = - pot_profile_find(pot0->reserved_profile_id); - if (PREDICT_FALSE(new_profile == 0 || - new_profile->valid == 0)) - { - ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PROFILE_MISS, 1); - return(-1); - } - else - { - pot_profile_index = pot0->reserved_profile_id; - pot_profile = new_profile; - pot_profile_set_active(pot_profile_index); - pot_profile_reset_usage_stats(pot_profile); - } - } - pot_profile_incr_usage_stats(pot_profile); - } - - if (pot0->random == 0) - { - pot0->random = clib_host_to_net_u64(pot_generate_random(pot_profile)); - pot0->cumulative = 0; - } - random = clib_net_to_host_u64(pot0->random); - cumulative = clib_net_to_host_u64(pot0->cumulative); - pot0->cumulative = clib_host_to_net_u64( - pot_update_cumulative(pot_profile, - cumulative, - random)); - ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PROCESSED, 1); - - return (rv); -} - -int -ip6_hbh_ioam_proof_of_transit_pop_handler (vlib_buffer_t *b, ip6_header_t *ip, - ip6_hop_by_hop_option_t *opt0) -{ - ioam_pot_option_t * pot0; - u64 random = 0; - u64 cumulative = 0; - int rv = 0; - pot_profile *pot_profile = 0; - u8 result = 0; - - pot0 = (ioam_pot_option_t *) opt0; - random = clib_net_to_host_u64(pot0->random); - cumulative = clib_net_to_host_u64(pot0->cumulative); - pot_profile = pot_profile_get_active(); - result = pot_validate (pot_profile, - cumulative, random); - - if (result == 1) - { - ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PASSED, 1); - } - else - { - ip6_ioam_stats_increment_counter (IP6_IOAM_POT_FAILED, 1); - } - return (rv); -} - -int ip6_hop_by_hop_ioam_pot_rewrite_handler (u8 *rewrite_string, u8 *rewrite_size) -{ - ioam_pot_option_t * pot_option; - if (rewrite_string && *rewrite_size == sizeof(ioam_pot_option_t)) - { - pot_option = (ioam_pot_option_t *)rewrite_string; - pot_option->hdr.type = HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT - | HBH_OPTION_TYPE_DATA_CHANGE_ENROUTE; - pot_option->hdr.length = sizeof (ioam_pot_option_t) - - sizeof (ip6_hop_by_hop_option_t); - return(0); - } - return(-1); -} - -static clib_error_t * -ip6_show_ioam_pot_cmd_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - ip6_hop_by_hop_ioam_pot_main_t *hm = &ip6_hop_by_hop_ioam_pot_main; - u8 *s = 0; - int i = 0; - - for ( i = 0; i < IP6_IOAM_POT_N_STATS; i++) - { - s = format(s, " %s - %lu\n", ip6_hop_by_hop_ioam_pot_stats_strings[i], - hm->counters[i]); - } - - vlib_cli_output(vm, "%v", s); - vec_free(s); - return 0; -} - - -VLIB_CLI_COMMAND (ip6_show_ioam_pot_cmd, static) = { - .path = "show ioam pot", - .short_help = "iOAM pot statistics", - .function = ip6_show_ioam_pot_cmd_fn, -}; - - -static clib_error_t * -ip6_hop_by_hop_ioam_pot_init (vlib_main_t * vm) -{ - ip6_hop_by_hop_ioam_pot_main_t * hm = &ip6_hop_by_hop_ioam_pot_main; - clib_error_t * error; - - if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init))) - return(error); - - hm->vlib_main = vm; - hm->vnet_main = vnet_get_main(); - memset(hm->counters, 0, sizeof(hm->counters)); - - if (ip6_hbh_register_option(HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT, ip6_hbh_ioam_proof_of_transit_handler, - ip6_hbh_ioam_proof_of_transit_trace_handler) < 0) - return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT failed")); - - if (ip6_hbh_add_register_option(HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT, - sizeof(ioam_pot_option_t), - ip6_hop_by_hop_ioam_pot_rewrite_handler) < 0) - return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT for rewrite failed")); - - if (ip6_hbh_pop_register_option(HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT, - ip6_hbh_ioam_proof_of_transit_pop_handler) < 0) - return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT POP failed")); - - return (0); -} - -VLIB_INIT_FUNCTION (ip6_hop_by_hop_ioam_pot_init); - - diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.c deleted file mode 100644 index 0b4d4192..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include "ip6_ioam_seqno.h" -#include "ip6_ioam_e2e.h" - -ioam_seqno_data_main_t ioam_seqno_main; - -void ioam_seqno_init_bitmap (ioam_seqno_data *data) -{ - seqno_bitmap *bitmap = &data->seqno_rx.bitmap; - bitmap->window_size = SEQNO_WINDOW_SIZE; - bitmap->array_size = SEQNO_WINDOW_ARRAY_SIZE; - bitmap->mask = 32 * SEQNO_WINDOW_ARRAY_SIZE - 1; - bitmap->array[0] = 0x00000000;/* pretend we haven seen sequence numbers 0*/ - bitmap->highest = 0; - - data->seq_num = 0; - return ; -} - -/* - * This Routine gets called from IPv6 hop-by-hop option handling. - * Only if we are encap node, then add PPC data. - * On a Transit(MID) node we dont do anything with E2E headers. - * On decap node decap is handled by seperate function. - */ -int -ioam_seqno_encap_handler (vlib_buffer_t *b, ip6_header_t *ip, - ip6_hop_by_hop_option_t *opt) -{ - u32 opaque_index = vnet_buffer(b)->l2_classify.opaque_index; - ioam_e2e_option_t * e2e; - int rv = 0; - ioam_seqno_data *data; - - data = ioam_e2ec_get_seqno_data_from_flow_ctx(opaque_index); - e2e = (ioam_e2e_option_t *) opt; - e2e->e2e_data = clib_host_to_net_u32(++data->seq_num); - - return (rv); -} - -/* - * This Routine gets called on POP/Decap node. - */ -int -ioam_seqno_decap_handler (vlib_buffer_t *b, ip6_header_t *ip, - ip6_hop_by_hop_option_t *opt) -{ - u32 opaque_index = vnet_buffer(b)->l2_classify.opaque_index; - ioam_e2e_option_t * e2e; - int rv = 0; - ioam_seqno_data *data; - - data = ioam_e2ec_get_seqno_data_from_flow_ctx(opaque_index); - e2e = (ioam_e2e_option_t *) opt; - ioam_analyze_seqno(&data->seqno_rx, (u64) clib_net_to_host_u32(e2e->e2e_data)); - - return (rv); -} - -u8 * -show_ioam_seqno_cmd_fn (u8 *s, ioam_seqno_data *seqno_data, u8 enc) -{ - seqno_rx_info *rx; - - s = format(s, "SeqNo Data:\n"); - if (enc) - { - s = format(s, " Current Seq. Number : %llu\n", seqno_data->seq_num); - } - else - { - rx = &seqno_data->seqno_rx; - s = format(s, " Highest Seq. Number : %llu\n", rx->bitmap.highest); - s = format(s, " Packets received : %llu\n", rx->rx_packets); - s = format(s, " Lost packets : %llu\n", rx->lost_packets); - s = format(s, " Reordered packets : %llu\n", rx->reordered_packets); - s = format(s, " Duplicate packets : %llu\n", rx->dup_packets); - } - - format(s, "\n"); - return s; -} diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.h b/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.h deleted file mode 100644 index 13a84db0..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __included_ip6_ioam_seqno_h__ -#define __included_ip6_ioam_seqno_h__ - -#include -#include - -#define SEQ_CHECK_VALUE 0x80000000 /* for seq number wraparound detection */ - -#define SEQNO_WINDOW_SIZE 2048 -#define SEQNO_WINDOW_ARRAY_SIZE 64 - -typedef struct seqno_bitmap_ { - u32 window_size; - u32 array_size; - u32 mask; - u32 pad; - u64 highest; - u64 array[SEQNO_WINDOW_ARRAY_SIZE]; /* Will be alloc to array_size */ -} seqno_bitmap; - -typedef struct seqno_rx_info_ { - u64 rx_packets; - u64 lost_packets; - u64 reordered_packets; - u64 dup_packets; - seqno_bitmap bitmap; -} seqno_rx_info; - -/* This structure is 64-byte aligned */ -typedef struct ioam_seqno_data_ { - union { - u32 seq_num; /* Useful only for encap node */ - seqno_rx_info seqno_rx; - }; -} ioam_seqno_data; - -typedef struct ioam_seqno_data_main_t_ { - ioam_seqno_data *seqno_data; -} ioam_seqno_data_main_t; - -void ioam_seqno_init_bitmap(ioam_seqno_data *data); - -int ioam_seqno_encap_handler(vlib_buffer_t *b, ip6_header_t *ip, - ip6_hop_by_hop_option_t *opt); - -int -ioam_seqno_decap_handler(vlib_buffer_t *b, ip6_header_t *ip, - ip6_hop_by_hop_option_t *opt); - -void ioam_analyze_seqno(seqno_rx_info *ppc_rx, u64 seqno); - -u8 * -show_ioam_seqno_cmd_fn(u8 *s, ioam_seqno_data *seqno_data, u8 enc); - -#endif diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno_analyse.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno_analyse.c deleted file mode 100644 index 4638871c..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_seqno_analyse.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "ip6_ioam_seqno.h" - -static inline void BIT_SET (u64 *p, u32 n) -{ - p[ n>>5 ] |= (1 << (n&31)); -} - -static inline int BIT_TEST (u64 *p, u32 n) -{ - return p[ n>>5 ] & (1 << (n&31)); -} - -static void BIT_CLEAR (u64 *p, u64 start, int num_bits, u32 mask) -{ - int n, t; - int start_index = (start >> 5); - int mask_index = (mask >> 5); - - start_index &= mask_index; - if (start & 0x1f) - { - int start_bit = (start & 0x1f); - - n = (1 << start_bit)-1; - t = start_bit + num_bits; - if (t < 32) - { - n |= ~((1 << t)-1); - p[ start_index ] &= n; - return; - } - p[ start_index ] &= n; - start_index = (start_index + 1) & mask_index; - num_bits -= (32 - start_bit); - } - while (num_bits >= 32) - { - p[ start_index ] = 0; - start_index = (start_index + 1) & mask_index; - num_bits -= 32; - } - n = ~((1 << num_bits) - 1); - p[ start_index ] &= n; -} - -static inline u8 seqno_check_wraparound(u32 a, u32 b) -{ - if ((a != b) && (a > b) && ((a - b) > SEQ_CHECK_VALUE)) - { - return 1; - } - return 0; -} - -/* - * Function to analyze the PPC value recevied. - * - Updates the bitmap with received sequence number - * - counts the received/lost/duplicate/reordered packets - */ -void ioam_analyze_seqno (seqno_rx_info *seqno_rx, u64 seqno) -{ - int diff; - static int peer_dead_count; - seqno_bitmap *bitmap = &seqno_rx->bitmap; - - seqno_rx->rx_packets++; - - if (seqno > bitmap->highest) - { /* new larger sequence number */ - peer_dead_count = 0; - diff = seqno - bitmap->highest; - if (diff < bitmap->window_size) - { - if (diff > 1) - { /* diff==1 is *such* a common case it's a win to optimize it */ - BIT_CLEAR(bitmap->array, bitmap->highest+1, diff-1, bitmap->mask); - seqno_rx->lost_packets += diff -1; - } - } - else - { - seqno_rx->lost_packets += diff -1; - memset( bitmap->array, 0, bitmap->array_size * sizeof(u64) ); - } - BIT_SET(bitmap->array, seqno & bitmap->mask); - bitmap->highest = seqno; - return; - } - - /* we've seen a bigger seq number before */ - diff = bitmap->highest - seqno; - if (diff >= bitmap->window_size) - { - if (seqno_check_wraparound(bitmap->highest, seqno)) - { - memset( bitmap->array, 0, bitmap->array_size * sizeof(u64)); - BIT_SET(bitmap->array, seqno & bitmap->mask); - bitmap->highest = seqno; - return; - } - else - { - peer_dead_count++; - if (peer_dead_count > 25) - { - peer_dead_count = 0; - memset( bitmap->array, 0, bitmap->array_size * sizeof(u64) ); - BIT_SET(bitmap->array, seqno & bitmap->mask); - bitmap->highest = seqno; - } - //ppc_rx->reordered_packets++; - } - return; - } - - if (BIT_TEST(bitmap->array, seqno & bitmap->mask)) - { - seqno_rx->dup_packets++; - return; /* Already seen */ - } - seqno_rx->reordered_packets++; - seqno_rx->lost_packets--; - BIT_SET(bitmap->array, seqno & bitmap->mask); - return; -} diff --git a/plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c b/plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c deleted file mode 100644 index e63db6e4..00000000 --- a/plugins/ioam-plugin/ioam/encap/ip6_ioam_trace.c +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include - -/* Timestamp precision multipliers for seconds, milliseconds, microseconds - * and nanoseconds respectively. - */ -static f64 trace_tsp_mul[4] = { 1, 1e3, 1e6, 1e9 }; - -typedef union -{ - u64 as_u64; - u32 as_u32[2]; -} time_u64_t; - -/* *INDENT-OFF* */ -typedef CLIB_PACKED(struct { - ip6_hop_by_hop_option_t hdr; - u8 ioam_trace_type; - u8 data_list_elts_left; - u32 elts[0]; /* Variable type. So keep it generic */ -}) ioam_trace_option_t; -/* *INDENT-ON* */ - - -extern ip6_hop_by_hop_ioam_main_t ip6_hop_by_hop_ioam_main; -extern ip6_main_t ip6_main; - -#define foreach_ip6_hop_by_hop_ioam_trace_stats \ - _(PROCESSED, "Pkts with ip6 hop-by-hop trace options") \ - _(PROFILE_MISS, "Pkts with ip6 hop-by-hop trace options but no profile set") \ - _(UPDATED, "Pkts with trace updated") \ - _(FULL, "Pkts with trace options but no space") - -static char *ip6_hop_by_hop_ioam_trace_stats_strings[] = { -#define _(sym,string) string, - foreach_ip6_hop_by_hop_ioam_trace_stats -#undef _ -}; - -typedef enum -{ -#define _(sym,str) IP6_IOAM_TRACE_##sym, - foreach_ip6_hop_by_hop_ioam_trace_stats -#undef _ - IP6_IOAM_TRACE_N_STATS, -} ip6_ioam_trace_stats_t; - - -typedef struct -{ - /* stats */ - u64 counters[ARRAY_LEN (ip6_hop_by_hop_ioam_trace_stats_strings)]; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; -} ip6_hop_by_hop_ioam_trace_main_t; - -ip6_hop_by_hop_ioam_trace_main_t ip6_hop_by_hop_ioam_trace_main; - -always_inline void -ip6_ioam_trace_stats_increment_counter (u32 counter_index, u64 increment) -{ - ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main; - - hm->counters[counter_index] += increment; -} - - -static u8 * -format_ioam_data_list_element (u8 * s, va_list * args) -{ - u32 *elt = va_arg (*args, u32 *); - u8 *trace_type_p = va_arg (*args, u8 *); - u8 trace_type = *trace_type_p; - - - if (trace_type & BIT_TTL_NODEID) - { - u32 ttl_node_id_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "ttl 0x%x node id 0x%x ", - ttl_node_id_host_byte_order >> 24, - ttl_node_id_host_byte_order & 0x00FFFFFF); - - elt++; - } - - if (trace_type & BIT_ING_INTERFACE && trace_type & BIT_ING_INTERFACE) - { - u32 ingress_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "ingress 0x%x egress 0x%x ", - ingress_host_byte_order >> 16, - ingress_host_byte_order & 0xFFFF); - elt++; - } - - if (trace_type & BIT_TIMESTAMP) - { - u32 ts_in_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "ts 0x%x \n", ts_in_host_byte_order); - elt++; - } - - if (trace_type & BIT_APPDATA) - { - u32 appdata_in_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "app 0x%x ", appdata_in_host_byte_order); - elt++; - } - - return s; -} - - -int -ip6_ioam_trace_get_sizeof_handler (u32 * result) -{ - u16 size = 0; - u8 trace_data_size = 0; - trace_profile *profile = NULL; - - *result = 0; - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); - return (-1); - } - - trace_data_size = fetch_trace_data_size (profile->trace_type); - if (PREDICT_FALSE (trace_data_size == 0)) - return VNET_API_ERROR_INVALID_VALUE; - - if (PREDICT_FALSE (profile->num_elts * trace_data_size > 254)) - return VNET_API_ERROR_INVALID_VALUE; - - size += - sizeof (ioam_trace_option_t) + (profile->num_elts * trace_data_size); - *result = size; - - return 0; -} - - - -int -ip6_hop_by_hop_ioam_trace_rewrite_handler (u8 * rewrite_string, - u8 * rewrite_size) -{ - ioam_trace_option_t *trace_option = NULL; - u8 trace_data_size = 0; - u8 trace_option_elts = 0; - trace_profile *profile = NULL; - - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); - return (-1); - } - - if (PREDICT_FALSE (!rewrite_string)) - return -1; - - trace_option_elts = profile->num_elts; - trace_data_size = fetch_trace_data_size (profile->trace_type); - trace_option = (ioam_trace_option_t *) rewrite_string; - trace_option->hdr.type = HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST | - HBH_OPTION_TYPE_DATA_CHANGE_ENROUTE; - trace_option->hdr.length = 2 /*ioam_trace_type,data_list_elts_left */ + - trace_option_elts * trace_data_size; - trace_option->ioam_trace_type = profile->trace_type & TRACE_TYPE_MASK; - trace_option->data_list_elts_left = trace_option_elts; - *rewrite_size = - sizeof (ioam_trace_option_t) + (trace_option_elts * trace_data_size); - - return 0; -} - - -int -ip6_hbh_ioam_trace_data_list_handler (vlib_buffer_t * b, ip6_header_t * ip, - ip6_hop_by_hop_option_t * opt) -{ - ip6_main_t *im = &ip6_main; - ip_lookup_main_t *lm = &im->lookup_main; - ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main; - u8 elt_index = 0; - ioam_trace_option_t *trace = (ioam_trace_option_t *) opt; - u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; - ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index); - time_u64_t time_u64; - u32 *elt; - int rv = 0; - trace_profile *profile = NULL; - - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); - return (-1); - } - - - time_u64.as_u64 = 0; - - if (PREDICT_TRUE (trace->data_list_elts_left)) - { - trace->data_list_elts_left--; - /* fetch_trace_data_size returns in bytes. Convert it to 4-bytes - * to skip to this node's location. - */ - elt_index = - trace->data_list_elts_left * - fetch_trace_data_size (trace->ioam_trace_type) / 4; - elt = &trace->elts[elt_index]; - if (trace->ioam_trace_type & BIT_TTL_NODEID) - { - *elt = - clib_host_to_net_u32 ((ip->hop_limit << 24) | profile->node_id); - elt++; - } - - if (trace->ioam_trace_type & BIT_ING_INTERFACE) - { - *elt = - (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 | - (adj->rewrite_header.sw_if_index & 0xFFFF); - *elt = clib_host_to_net_u32 (*elt); - elt++; - } - - if (trace->ioam_trace_type & BIT_TIMESTAMP) - { - /* Send least significant 32 bits */ - f64 time_f64 = - (f64) (((f64) hm->unix_time_0) + - (vlib_time_now (hm->vlib_main) - hm->vlib_time_0)); - - time_u64.as_u64 = time_f64 * trace_tsp_mul[profile->trace_tsp]; - *elt = clib_host_to_net_u32 (time_u64.as_u32[0]); - elt++; - } - - if (trace->ioam_trace_type & BIT_APPDATA) - { - /* $$$ set elt0->app_data */ - *elt = clib_host_to_net_u32 (profile->app_data); - elt++; - } - ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_UPDATED, 1); - } - else - { - ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_FULL, 1); - } - return (rv); -} - -u8 * -ip6_hbh_ioam_trace_data_list_trace_handler (u8 * s, - ip6_hop_by_hop_option_t * opt) -{ - ioam_trace_option_t *trace; - u8 trace_data_size_in_words = 0; - u32 *elt; - int elt_index = 0; - - trace = (ioam_trace_option_t *) opt; - s = - format (s, " Trace Type 0x%x , %d elts left\n", trace->ioam_trace_type, - trace->data_list_elts_left); - trace_data_size_in_words = - fetch_trace_data_size (trace->ioam_trace_type) / 4; - elt = &trace->elts[0]; - while ((u8 *) elt < ((u8 *) (&trace->elts[0]) + trace->hdr.length - 2 - /* -2 accounts for ioam_trace_type,elts_left */ )) - { - s = format (s, " [%d] %U\n", elt_index, - format_ioam_data_list_element, - elt, &trace->ioam_trace_type); - elt_index++; - elt += trace_data_size_in_words; - } - return (s); -} - - -static clib_error_t * -ip6_show_ioam_trace_cmd_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main; - u8 *s = 0; - int i = 0; - - for (i = 0; i < IP6_IOAM_TRACE_N_STATS; i++) - { - s = - format (s, " %s - %lu\n", ip6_hop_by_hop_ioam_trace_stats_strings[i], - hm->counters[i]); - } - - vlib_cli_output (vm, "%v", s); - vec_free (s); - return 0; -} - - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (ip6_show_ioam_trace_cmd, static) = { - .path = "show ioam trace", - .short_help = "iOAM trace statistics", - .function = ip6_show_ioam_trace_cmd_fn, -}; -/* *INDENT-ON* */ - - -static clib_error_t * -ip6_hop_by_hop_ioam_trace_init (vlib_main_t * vm) -{ - ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main; - clib_error_t *error; - - if ((error = vlib_call_init_function (vm, ip_main_init))) - return (error); - - if ((error = vlib_call_init_function (vm, ip6_lookup_init))) - return error; - - if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init))) - return (error); - - hm->vlib_main = vm; - hm->vnet_main = vnet_get_main (); - memset (hm->counters, 0, sizeof (hm->counters)); - - - if (ip6_hbh_register_option - (HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST, - ip6_hbh_ioam_trace_data_list_handler, - ip6_hbh_ioam_trace_data_list_trace_handler) < 0) - return (clib_error_create - ("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST failed")); - - - if (ip6_hbh_add_register_option (HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST, - sizeof (ioam_trace_option_t), - ip6_hop_by_hop_ioam_trace_rewrite_handler) - < 0) - return (clib_error_create - ("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST for rewrite failed")); - - - return (0); -} - -int -ip6_trace_profile_cleanup (void) -{ - ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main; - - hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] = 0; - - return 0; - -} - - -int -ip6_trace_profile_setup (void) -{ - u32 trace_size = 0; - ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main; - - trace_profile *profile = NULL; - - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); - return (-1); - } - - - if (ip6_ioam_trace_get_sizeof_handler (&trace_size) < 0) - return (-1); - - hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] = trace_size; - - return (0); -} - - -VLIB_INIT_FUNCTION (ip6_hop_by_hop_ioam_trace_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export-common/ioam_export.h b/plugins/ioam-plugin/ioam/export-common/ioam_export.h deleted file mode 100644 index a9ec8d00..00000000 --- a/plugins/ioam-plugin/ioam/export-common/ioam_export.h +++ /dev/null @@ -1,616 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_ioam_export_h__ -#define __included_ioam_export_h__ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -typedef struct ioam_export_buffer -{ - /* Allocated buffer */ - u32 buffer_index; - u64 touched_at; - u8 records_in_this_buffer; -} ioam_export_buffer_t; - - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - - /* TODO: to support multiple collectors all this has to be grouped and create a vector here */ - u8 *record_header; - u32 sequence_number; - u32 domain_id; - - /* ipfix collector, our ip address */ - ip4_address_t ipfix_collector; - ip4_address_t src_address; - - /* Pool of ioam_export_buffer_t */ - ioam_export_buffer_t *buffer_pool; - /* Vector of per thread ioam_export_buffer_t to buffer pool index */ - u32 *buffer_per_thread; - /* Lock per thread to swap buffers between worker and timer process */ - volatile u32 **lockp; - - /* time scale transform */ - u32 unix_time_0; - f64 vlib_time_0; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; - u32 ip4_lookup_node_index; - - uword my_hbh_slot; - u32 export_process_node_index; -} ioam_export_main_t; - -ioam_export_main_t ioam_export_main; -ioam_export_main_t vxlan_gpe_ioam_export_main; - -vlib_node_registration_t export_node; - -#define DEFAULT_EXPORT_SIZE (3 * CLIB_CACHE_LINE_BYTES) -/* - * Number of records in a buffer - * ~(MTU (1500) - [ip hdr(40) + UDP(8) + ipfix (24)]) / DEFAULT_EXPORT_SIZE - */ -#define DEFAULT_EXPORT_RECORDS 7 - -always_inline ioam_export_buffer_t * -ioam_export_get_my_buffer (ioam_export_main_t * em, u32 thread_id) -{ - - if (vec_len (em->buffer_per_thread) > thread_id) - return (pool_elt_at_index - (em->buffer_pool, em->buffer_per_thread[thread_id])); - return (0); -} - -inline static int -ioam_export_buffer_add_header (ioam_export_main_t * em, vlib_buffer_t * b0) -{ - clib_memcpy (b0->data, em->record_header, vec_len (em->record_header)); - b0->current_data = 0; - b0->current_length = vec_len (em->record_header); - b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; - return (1); -} - -inline static int -ioam_export_init_buffer (ioam_export_main_t * em, vlib_main_t * vm, - ioam_export_buffer_t * eb) -{ - vlib_buffer_t *b = 0; - - if (!eb) - return (-1); - /* TODO: Perhaps buffer init from template here */ - if (vlib_buffer_alloc (vm, &(eb->buffer_index), 1) != 1) - return (-2); - eb->records_in_this_buffer = 0; - eb->touched_at = vlib_time_now (vm); - b = vlib_get_buffer (vm, eb->buffer_index); - (void) ioam_export_buffer_add_header (em, b); - vnet_buffer (b)->sw_if_index[VLIB_RX] = 0; - vnet_buffer (b)->sw_if_index[VLIB_TX] = ~0; - return (1); -} - -inline static void -ioam_export_thread_buffer_free (ioam_export_main_t * em) -{ - vlib_main_t *vm = em->vlib_main; - ioam_export_buffer_t *eb = 0; - int i; - for (i = 0; i < vec_len (em->buffer_per_thread); i++) - { - eb = pool_elt_at_index (em->buffer_pool, em->buffer_per_thread[i]); - if (eb) - vlib_buffer_free (vm, &(eb->buffer_index), 1); - } - for (i = 0; i < vec_len (em->lockp); i++) - clib_mem_free ((void *) em->lockp[i]); - vec_free (em->buffer_per_thread); - pool_free (em->buffer_pool); - vec_free (em->lockp); - em->buffer_per_thread = 0; - em->buffer_pool = 0; - em->lockp = 0; -} - -inline static int -ioam_export_thread_buffer_init (ioam_export_main_t * em, vlib_main_t * vm) -{ - int no_of_threads = vec_len (vlib_worker_threads); - int i; - ioam_export_buffer_t *eb = 0; - vlib_node_t *ip4_lookup_node; - - pool_alloc_aligned (em->buffer_pool, - no_of_threads - 1, CLIB_CACHE_LINE_BYTES); - vec_validate_aligned (em->buffer_per_thread, - no_of_threads - 1, CLIB_CACHE_LINE_BYTES); - vec_validate_aligned (em->lockp, no_of_threads - 1, CLIB_CACHE_LINE_BYTES); - ip4_lookup_node = vlib_get_node_by_name (vm, (u8 *) "ip4-lookup"); - em->ip4_lookup_node_index = ip4_lookup_node->index; - if (!em->buffer_per_thread || !em->buffer_pool || !em->lockp) - { - return (-1); - } - for (i = 0; i < no_of_threads; i++) - { - eb = 0; - pool_get_aligned (em->buffer_pool, eb, CLIB_CACHE_LINE_BYTES); - memset (eb, 0, sizeof (*eb)); - em->buffer_per_thread[i] = eb - em->buffer_pool; - if (ioam_export_init_buffer (em, vm, eb) != 1) - { - ioam_export_thread_buffer_free (em); - return (-2); - } - em->lockp[i] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, - CLIB_CACHE_LINE_BYTES); - memset ((void *) em->lockp[i], 0, CLIB_CACHE_LINE_BYTES); - } - return (1); -} - -#define IPFIX_IOAM_EXPORT_ID 272 - -/* Used to build the rewrite */ -/* data set packet */ -typedef struct -{ - ipfix_message_header_t h; - ipfix_set_header_t s; -} ipfix_data_packet_t; - -typedef struct -{ - ip4_header_t ip4; - udp_header_t udp; - ipfix_data_packet_t ipfix; -} ip4_ipfix_data_packet_t; - - -inline static void -ioam_export_header_cleanup (ioam_export_main_t * em, - ip4_address_t * collector_address, - ip4_address_t * src_address) -{ - vec_free (em->record_header); - em->record_header = 0; -} - -inline static int -ioam_export_header_create (ioam_export_main_t * em, - ip4_address_t * collector_address, - ip4_address_t * src_address) -{ - ip4_header_t *ip; - udp_header_t *udp; - ipfix_message_header_t *h; - ipfix_set_header_t *s; - u8 *rewrite = 0; - ip4_ipfix_data_packet_t *tp; - - - /* allocate rewrite space */ - vec_validate_aligned (rewrite, - sizeof (ip4_ipfix_data_packet_t) - 1, - CLIB_CACHE_LINE_BYTES); - - tp = (ip4_ipfix_data_packet_t *) rewrite; - ip = (ip4_header_t *) & tp->ip4; - udp = (udp_header_t *) (ip + 1); - h = (ipfix_message_header_t *) (udp + 1); - s = (ipfix_set_header_t *) (h + 1); - - ip->ip_version_and_header_length = 0x45; - ip->ttl = 254; - ip->protocol = IP_PROTOCOL_UDP; - ip->src_address.as_u32 = src_address->as_u32; - ip->dst_address.as_u32 = collector_address->as_u32; - udp->src_port = clib_host_to_net_u16 (4939 /* $$FIXME */ ); - udp->dst_port = clib_host_to_net_u16 (4939); - /* FIXUP: UDP length */ - udp->length = clib_host_to_net_u16 (vec_len (rewrite) + - (DEFAULT_EXPORT_RECORDS * - DEFAULT_EXPORT_SIZE) - sizeof (*ip)); - - /* FIXUP: message header export_time */ - /* FIXUP: message header sequence_number */ - h->domain_id = clib_host_to_net_u32 (em->domain_id); - - /*FIXUP: Setid length in octets if records exported are not default */ - s->set_id_length = ipfix_set_id_length (IPFIX_IOAM_EXPORT_ID, - (sizeof (*s) + - (DEFAULT_EXPORT_RECORDS * - DEFAULT_EXPORT_SIZE))); - - /* FIXUP: h version and length length in octets if records exported are not default */ - h->version_length = version_length (sizeof (*h) + - (sizeof (*s) + - (DEFAULT_EXPORT_RECORDS * - DEFAULT_EXPORT_SIZE))); - - /* FIXUP: ip length if records exported are not default */ - /* FIXUP: ip checksum if records exported are not default */ - ip->length = clib_host_to_net_u16 (vec_len (rewrite) + - (DEFAULT_EXPORT_RECORDS * - DEFAULT_EXPORT_SIZE)); - ip->checksum = ip4_header_checksum (ip); - _vec_len (rewrite) = sizeof (ip4_ipfix_data_packet_t); - em->record_header = rewrite; - return (1); -} - -inline static int -ioam_export_send_buffer (ioam_export_main_t * em, vlib_main_t * vm, - ioam_export_buffer_t * eb) -{ - ip4_header_t *ip; - udp_header_t *udp; - ipfix_message_header_t *h; - ipfix_set_header_t *s; - ip4_ipfix_data_packet_t *tp; - vlib_buffer_t *b0; - u16 new_l0, old_l0; - ip_csum_t sum0; - vlib_frame_t *nf = 0; - u32 *to_next; - - b0 = vlib_get_buffer (vm, eb->buffer_index); - tp = vlib_buffer_get_current (b0); - ip = (ip4_header_t *) & tp->ip4; - udp = (udp_header_t *) (ip + 1); - h = (ipfix_message_header_t *) (udp + 1); - s = (ipfix_set_header_t *) (h + 1); - - /* FIXUP: message header export_time */ - h->export_time = clib_host_to_net_u32 ((u32) - (((f64) em->unix_time_0) + - (vlib_time_now (em->vlib_main) - - em->vlib_time_0))); - - /* FIXUP: message header sequence_number */ - h->sequence_number = clib_host_to_net_u32 (em->sequence_number++); - - /* FIXUP: lengths if different from default */ - if (PREDICT_FALSE (eb->records_in_this_buffer != DEFAULT_EXPORT_RECORDS)) - { - s->set_id_length = - ipfix_set_id_length (IPFIX_IOAM_EXPORT_ID /* set_id */ , - b0->current_length - (sizeof (*ip) + - sizeof (*udp) + - sizeof (*h))); - h->version_length = - version_length (b0->current_length - (sizeof (*ip) + sizeof (*udp))); - sum0 = ip->checksum; - old_l0 = ip->length; - new_l0 = clib_host_to_net_u16 ((u16) b0->current_length); - sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t, - length /* changed member */ ); - ip->checksum = ip_csum_fold (sum0); - ip->length = new_l0; - udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip)); - } - - /* Enqueue pkts to ip4-lookup */ - - nf = vlib_get_frame_to_node (vm, em->ip4_lookup_node_index); - nf->n_vectors = 0; - to_next = vlib_frame_vector_args (nf); - nf->n_vectors = 1; - to_next[0] = eb->buffer_index; - vlib_put_frame_to_node (vm, em->ip4_lookup_node_index, nf); - return (1); - -} - -#define EXPORT_TIMEOUT (20.0) -#define THREAD_PERIOD (30.0) -inline static uword -ioam_export_process_common (ioam_export_main_t * em, vlib_main_t * vm, - vlib_node_runtime_t * rt, vlib_frame_t * f, - u32 index) -{ - f64 now; - f64 timeout = 30.0; - uword event_type; - uword *event_data = 0; - int i; - ioam_export_buffer_t *eb = 0, *new_eb = 0; - u32 *vec_buffer_indices = 0; - u32 *vec_buffer_to_be_sent = 0; - u32 *thread_index = 0; - u32 new_pool_index = 0; - - em->export_process_node_index = index; - /* Wait for Godot... */ - vlib_process_wait_for_event_or_clock (vm, 1e9); - event_type = vlib_process_get_events (vm, &event_data); - if (event_type != 1) - clib_warning ("bogus kickoff event received, %d", event_type); - vec_reset_length (event_data); - - while (1) - { - vlib_process_wait_for_event_or_clock (vm, timeout); - event_type = vlib_process_get_events (vm, &event_data); - switch (event_type) - { - case 2: /* Stop and Wait for kickoff again */ - timeout = 1e9; - break; - case 1: /* kickoff : Check for unsent buffers */ - timeout = THREAD_PERIOD; - break; - case ~0: /* timeout */ - break; - } - vec_reset_length (event_data); - now = vlib_time_now (vm); - /* - * Create buffers for threads that are not active enough - * to send out the export records - */ - for (i = 0; i < vec_len (em->buffer_per_thread); i++) - { - /* If the worker thread is processing export records ignore further checks */ - if (*em->lockp[i] == 1) - continue; - eb = pool_elt_at_index (em->buffer_pool, em->buffer_per_thread[i]); - if (eb->records_in_this_buffer > 0 - && now > (eb->touched_at + EXPORT_TIMEOUT)) - { - pool_get_aligned (em->buffer_pool, new_eb, - CLIB_CACHE_LINE_BYTES); - memset (new_eb, 0, sizeof (*new_eb)); - if (ioam_export_init_buffer (em, vm, new_eb) == 1) - { - new_pool_index = new_eb - em->buffer_pool; - vec_add (vec_buffer_indices, &new_pool_index, 1); - vec_add (vec_buffer_to_be_sent, &em->buffer_per_thread[i], - 1); - vec_add (thread_index, &i, 1); - } - else - { - pool_put (em->buffer_pool, new_eb); - /*Give up */ - goto CLEANUP; - } - } - } - if (vec_len (thread_index) != 0) - { - /* - * Now swap the buffers out - */ - for (i = 0; i < vec_len (thread_index); i++) - { - while (__sync_lock_test_and_set (em->lockp[thread_index[i]], 1)) - ; - em->buffer_per_thread[thread_index[i]] = - vec_pop (vec_buffer_indices); - *em->lockp[thread_index[i]] = 0; - } - - /* Send the buffers */ - for (i = 0; i < vec_len (vec_buffer_to_be_sent); i++) - { - eb = - pool_elt_at_index (em->buffer_pool, vec_buffer_to_be_sent[i]); - ioam_export_send_buffer (em, vm, eb); - pool_put (em->buffer_pool, eb); - } - } - - CLEANUP: - /* Free any leftover/unused buffers and everything that was allocated */ - for (i = 0; i < vec_len (vec_buffer_indices); i++) - { - new_eb = pool_elt_at_index (em->buffer_pool, vec_buffer_indices[i]); - vlib_buffer_free (vm, &new_eb->buffer_index, 1); - pool_put (em->buffer_pool, new_eb); - } - vec_free (vec_buffer_indices); - vec_free (vec_buffer_to_be_sent); - vec_free (thread_index); - } - return 0; /* not so much */ -} - -#define ioam_export_node_common(EM, VM, N, F, HTYPE, L, V, NEXT) \ -do { \ - u32 n_left_from, *from, *to_next; \ - export_next_t next_index; \ - u32 pkts_recorded = 0; \ - ioam_export_buffer_t *my_buf = 0; \ - vlib_buffer_t *eb0 = 0; \ - u32 ebi0 = 0; \ - from = vlib_frame_vector_args (F); \ - n_left_from = (F)->n_vectors; \ - next_index = (N)->cached_next_index; \ - while (__sync_lock_test_and_set ((EM)->lockp[(VM)->cpu_index], 1)); \ - my_buf = ioam_export_get_my_buffer (EM, (VM)->cpu_index); \ - my_buf->touched_at = vlib_time_now (VM); \ - while (n_left_from > 0) \ - { \ - u32 n_left_to_next; \ - vlib_get_next_frame (VM, N, next_index, to_next, n_left_to_next); \ - while (n_left_from >= 4 && n_left_to_next >= 2) \ - { \ - u32 next0 = NEXT; \ - u32 next1 = NEXT; \ - u32 bi0, bi1; \ - HTYPE *ip0, *ip1; \ - vlib_buffer_t *p0, *p1; \ - u32 ip_len0, ip_len1; \ - { \ - vlib_buffer_t *p2, *p3; \ - p2 = vlib_get_buffer (VM, from[2]); \ - p3 = vlib_get_buffer (VM, from[3]); \ - vlib_prefetch_buffer_header (p2, LOAD); \ - vlib_prefetch_buffer_header (p3, LOAD); \ - CLIB_PREFETCH (p2->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD); \ - CLIB_PREFETCH (p3->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD); \ - } \ - to_next[0] = bi0 = from[0]; \ - to_next[1] = bi1 = from[1]; \ - from += 2; \ - to_next += 2; \ - n_left_from -= 2; \ - n_left_to_next -= 2; \ - p0 = vlib_get_buffer (VM, bi0); \ - p1 = vlib_get_buffer (VM, bi1); \ - ip0 = vlib_buffer_get_current (p0); \ - ip1 = vlib_buffer_get_current (p1); \ - ip_len0 = \ - clib_net_to_host_u16 (ip0->L) + sizeof (HTYPE); \ - ip_len1 = \ - clib_net_to_host_u16 (ip1->L) + sizeof (HTYPE); \ - ebi0 = my_buf->buffer_index; \ - eb0 = vlib_get_buffer (VM, ebi0); \ - if (PREDICT_FALSE (eb0 == 0)) \ - goto NO_BUFFER1; \ - ip_len0 = \ - ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0; \ - ip_len1 = \ - ip_len1 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len1; \ - copy3cachelines (eb0->data + eb0->current_length, ip0, ip_len0); \ - eb0->current_length += DEFAULT_EXPORT_SIZE; \ - my_buf->records_in_this_buffer++; \ - if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS) \ - { \ - ioam_export_send_buffer (EM, VM, my_buf); \ - ioam_export_init_buffer (EM, VM, my_buf); \ - } \ - ebi0 = my_buf->buffer_index; \ - eb0 = vlib_get_buffer (VM, ebi0); \ - if (PREDICT_FALSE (eb0 == 0)) \ - goto NO_BUFFER1; \ - copy3cachelines (eb0->data + eb0->current_length, ip1, ip_len1); \ - eb0->current_length += DEFAULT_EXPORT_SIZE; \ - my_buf->records_in_this_buffer++; \ - if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS) \ - { \ - ioam_export_send_buffer (EM, VM, my_buf); \ - ioam_export_init_buffer (EM, VM, my_buf); \ - } \ - pkts_recorded += 2; \ - if (PREDICT_FALSE (((node)->flags & VLIB_NODE_FLAG_TRACE))) \ - { \ - if (p0->flags & VLIB_BUFFER_IS_TRACED) \ - { \ - export_trace_t *t = \ - vlib_add_trace (VM, node, p0, sizeof (*t)); \ - t->flow_label = \ - clib_net_to_host_u32 (ip0->V); \ - t->next_index = next0; \ - } \ - if (p1->flags & VLIB_BUFFER_IS_TRACED) \ - { \ - export_trace_t *t = \ - vlib_add_trace (VM, N, p1, sizeof (*t)); \ - t->flow_label = \ - clib_net_to_host_u32 (ip1->V); \ - t->next_index = next1; \ - } \ - } \ - NO_BUFFER1: \ - vlib_validate_buffer_enqueue_x2 (VM, N, next_index, \ - to_next, n_left_to_next, \ - bi0, bi1, next0, next1); \ - } \ - while (n_left_from > 0 && n_left_to_next > 0) \ - { \ - u32 bi0; \ - vlib_buffer_t *p0; \ - u32 next0 = NEXT; \ - HTYPE *ip0; \ - u32 ip_len0; \ - bi0 = from[0]; \ - to_next[0] = bi0; \ - from += 1; \ - to_next += 1; \ - n_left_from -= 1; \ - n_left_to_next -= 1; \ - p0 = vlib_get_buffer (VM, bi0); \ - ip0 = vlib_buffer_get_current (p0); \ - ip_len0 = \ - clib_net_to_host_u16 (ip0->L) + sizeof (HTYPE); \ - ebi0 = my_buf->buffer_index; \ - eb0 = vlib_get_buffer (VM, ebi0); \ - if (PREDICT_FALSE (eb0 == 0)) \ - goto NO_BUFFER; \ - ip_len0 = \ - ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0; \ - copy3cachelines (eb0->data + eb0->current_length, ip0, ip_len0); \ - eb0->current_length += DEFAULT_EXPORT_SIZE; \ - my_buf->records_in_this_buffer++; \ - if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS) \ - { \ - ioam_export_send_buffer (EM, VM, my_buf); \ - ioam_export_init_buffer (EM, VM, my_buf); \ - } \ - if (PREDICT_FALSE (((N)->flags & VLIB_NODE_FLAG_TRACE) \ - && (p0->flags & VLIB_BUFFER_IS_TRACED))) \ - { \ - export_trace_t *t = vlib_add_trace (VM, (N), p0, sizeof (*t)); \ - t->flow_label = \ - clib_net_to_host_u32 (ip0->V); \ - t->next_index = next0; \ - } \ - pkts_recorded += 1; \ - NO_BUFFER: \ - vlib_validate_buffer_enqueue_x1 (VM, N, next_index, \ - to_next, n_left_to_next, \ - bi0, next0); \ - } \ - vlib_put_next_frame (VM, N, next_index, n_left_to_next); \ - } \ - vlib_node_increment_counter (VM, export_node.index, \ - EXPORT_ERROR_RECORDED, pkts_recorded); \ - *(EM)->lockp[(VM)->cpu_index] = 0; \ -} while(0) - -#endif /* __included_ioam_export_h__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api deleted file mode 100644 index 7b17c3f7..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api +++ /dev/null @@ -1,42 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Define a simple binary API to control the feature */ - -define vxlan_gpe_ioam_export_enable_disable { - /* Client identifier, set from api_main.my_client_index */ - u32 client_index; - - /* Arbitrary context, so client can match reply to request */ - u32 context; - - /* Enable / disable the feature */ - u8 is_disable; - - /* Collector ip address */ - u8 collector_address[4]; - u8 src_address[4]; - - /* Src ip address */ -}; - -define vxlan_gpe_ioam_export_enable_disable_reply { - /* From the request */ - u32 context; - - /* Return value, zero means all OK */ - i32 retval; -}; \ No newline at end of file diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c deleted file mode 100644 index bab8d977..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * vxlan_gpe_ioam_export.c - ioam export API / debug CLI handling - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - - -/* List of message types that this plugin understands */ - - -#define foreach_vxlan_gpe_ioam_export_plugin_api_msg \ -_(VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE, vxlan_gpe_ioam_export_enable_disable) - -extern void vxlan_gpe_set_next_override (uword next); -/* Action function shared between message handler and debug CLI */ -int -vxlan_gpe_ioam_export_enable_disable (ioam_export_main_t * em, - u8 is_disable, - ip4_address_t * collector_address, - ip4_address_t * src_address) -{ - vlib_main_t *vm = em->vlib_main; - u32 node_index = export_node.index; - vlib_node_t *vxlan_gpe_decap_ioam_node = NULL; - - if (is_disable == 0) - { - if (em->my_hbh_slot == ~0) - { - /* Hook this export node to vxlan-gpe-decap-ioam-v4 */ - vxlan_gpe_decap_ioam_node = - vlib_get_node_by_name (vm, (u8 *) "vxlan-gpe-decap-ioam-v4"); - if (!vxlan_gpe_decap_ioam_node) - { - /* node does not exist give up */ - return (-1); - } - em->my_hbh_slot = - vlib_node_add_next (vm, vxlan_gpe_decap_ioam_node->index, - node_index); - } - if (1 == ioam_export_header_create (em, collector_address, src_address)) - { - ioam_export_thread_buffer_init (em, vm); - vxlan_gpe_set_next_override (em->my_hbh_slot); - /* Turn on the export buffer check process */ - vlib_process_signal_event (vm, em->export_process_node_index, 1, 0); - - } - else - { - return (-2); - } - } - else - { - vxlan_gpe_set_next_override (VXLAN_GPE_DECAP_IOAM_V4_NEXT_POP); - ioam_export_header_cleanup (em, collector_address, src_address); - ioam_export_thread_buffer_free (em); - /* Turn off the export buffer check process */ - vlib_process_signal_event (vm, em->export_process_node_index, 2, 0); - - } - - return 0; -} - -/* API message handler */ -static void vl_api_vxlan_gpe_ioam_export_enable_disable_t_handler - (vl_api_vxlan_gpe_ioam_export_enable_disable_t * mp) -{ - vl_api_vxlan_gpe_ioam_export_enable_disable_reply_t *rmp; - ioam_export_main_t *sm = &vxlan_gpe_ioam_export_main; - int rv; - - rv = vxlan_gpe_ioam_export_enable_disable (sm, (int) (mp->is_disable), - (ip4_address_t *) - mp->collector_address, - (ip4_address_t *) - mp->src_address); - - REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE_REPLY); -} /* API message handler */ - - - -/* Set up the API message handling tables */ -static clib_error_t * -vxlan_gpe_ioam_export_plugin_api_hookup (vlib_main_t * vm) -{ - ioam_export_main_t *sm = &vxlan_gpe_ioam_export_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vxlan_gpe_ioam_export_plugin_api_msg; -#undef _ - - return 0; -} - - -static clib_error_t * -set_vxlan_gpe_ioam_export_ipfix_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - ioam_export_main_t *em = &vxlan_gpe_ioam_export_main; - ip4_address_t collector, src; - u8 is_disable = 0; - - collector.as_u32 = 0; - src.as_u32 = 0; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "collector %U", unformat_ip4_address, &collector)) - ; - else if (unformat (input, "src %U", unformat_ip4_address, &src)) - ; - else if (unformat (input, "disable")) - is_disable = 1; - else - break; - } - - if (collector.as_u32 == 0) - return clib_error_return (0, "collector address required"); - - if (src.as_u32 == 0) - return clib_error_return (0, "src address required"); - - em->ipfix_collector.as_u32 = collector.as_u32; - em->src_address.as_u32 = src.as_u32; - - vlib_cli_output (vm, "Collector %U, src address %U", - format_ip4_address, &em->ipfix_collector, - format_ip4_address, &em->src_address); - - /* Turn on the export timer process */ - // vlib_process_signal_event (vm, flow_report_process_node.index, - //1, 0); - if (0 != - vxlan_gpe_ioam_export_enable_disable (em, is_disable, &collector, &src)) - { - return clib_error_return (0, "Unable to set ioam vxlan-gpe export"); - } - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (set_vxlan_gpe_ioam_ipfix_command, static) = -{ -.path = "set vxlan-gpe-ioam export ipfix", -.short_help = "set vxlan-gpe-ioam export ipfix collector src ", -.function = set_vxlan_gpe_ioam_export_ipfix_command_fn, -}; -/* *INDENT-ON* */ - - -static clib_error_t * -vxlan_gpe_ioam_export_init (vlib_main_t * vm) -{ - ioam_export_main_t *em = &vxlan_gpe_ioam_export_main; - clib_error_t *error = 0; - u8 *name; - - name = format (0, "vxlan_gpe_ioam_export_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - em->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - em->unix_time_0 = (u32) time (0); /* Store starting time */ - em->vlib_time_0 = vlib_time_now (vm); - - error = vxlan_gpe_ioam_export_plugin_api_hookup (vm); - em->my_hbh_slot = ~0; - em->vlib_main = vm; - em->vnet_main = vnet_get_main (); - vec_free (name); - - return error; -} - -VLIB_INIT_FUNCTION (vxlan_gpe_ioam_export_init); - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h deleted file mode 100644 index 6d93f093..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h deleted file mode 100644 index cc5698de..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_vxlan_gpe_ioam_export_msg_enum_h -#define included_vxlan_gpe_ioam_export_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_vxlan_gpe_ioam_export_msg_enum_h */ diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c deleted file mode 100644 index 494263d9..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * vxlan_gpe_ioam_export_test.c - test harness plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include - - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} export_test_main_t; - -export_test_main_t export_test_main; - -#define foreach_standard_reply_retval_handler \ -_(vxlan_gpe_ioam_export_enable_disable_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = export_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE_REPLY, vxlan_gpe_ioam_export_enable_disable_reply) - - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - -static int -api_vxlan_gpe_ioam_export_enable_disable (vat_main_t * vam) -{ - export_test_main_t *sm = &export_test_main; - unformat_input_t *i = vam->input; - f64 timeout; - int is_disable = 0; - vl_api_vxlan_gpe_ioam_export_enable_disable_t *mp; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "disable")) - is_disable = 1; - else - break; - } - - /* Construct the API message */ - M (VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE, - vxlan_gpe_ioam_export_enable_disable); - mp->is_disable = is_disable; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(vxlan_gpe_ioam_export_enable_disable, " [disable]") - -void -vat_api_hookup (vat_main_t * vam) -{ - export_test_main_t *sm = &export_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * -vat_plugin_register (vat_main_t * vam) -{ - export_test_main_t *sm = &export_test_main; - u8 *name; - - sm->vat_main = vam; - - name = format (0, "vxlan_gpe_ioam_export_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~ 0) - vat_api_hookup (vam); - - vec_free (name); - - return 0; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c deleted file mode 100644 index 58508ebf..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * ioam_export_thread.c - */ -#include -#include -#include - -static vlib_node_registration_t vxlan_gpe_ioam_export_process_node; - -static uword -vxlan_gpe_ioam_export_process (vlib_main_t * vm, - vlib_node_runtime_t * rt, vlib_frame_t * f) -{ - return (ioam_export_process_common (&vxlan_gpe_ioam_export_main, - vm, rt, f, - vxlan_gpe_ioam_export_process_node.index)); -} - - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (vxlan_gpe_ioam_export_process_node, static) = -{ - .function = vxlan_gpe_ioam_export_process, - .type = VLIB_NODE_TYPE_PROCESS, - .name = "vxlan-gpe-ioam-export-process", -}; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_node.c b/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_node.c deleted file mode 100644 index 722c2b06..00000000 --- a/plugins/ioam-plugin/ioam/export-vxlan-gpe/vxlan_gpe_node.c +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct -{ - u32 next_index; - u32 flow_label; -} export_trace_t; - -/* packet trace format function */ -static u8 * -format_export_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - export_trace_t *t = va_arg (*args, export_trace_t *); - - s = format (s, "EXPORT: flow_label %d, next index %d", - t->flow_label, t->next_index); - return s; -} - -vlib_node_registration_t export_node; - -#define foreach_export_error \ -_(RECORDED, "Packets recorded for export") - -typedef enum -{ -#define _(sym,str) EXPORT_ERROR_##sym, - foreach_export_error -#undef _ - EXPORT_N_ERROR, -} export_error_t; - -static char *export_error_strings[] = { -#define _(sym,string) string, - foreach_export_error -#undef _ -}; - -typedef enum -{ - EXPORT_NEXT_VXLAN_GPE_INPUT, - EXPORT_N_NEXT, -} export_next_t; - -always_inline void -copy3cachelines (void *dst, const void *src, size_t n) -{ -#if 0 - if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) - { - /* Copy only the first 1/2 cache lines whatever is available */ - if (n >= 64) - clib_mov64 ((u8 *) dst, (const u8 *) src); - if (n >= 128) - clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); - return; - } - clib_mov64 ((u8 *) dst, (const u8 *) src); - clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); - clib_mov64 ((u8 *) dst + 128, (const u8 *) src + 128); -#endif -#if 1 - - u64 *copy_dst, *copy_src; - int i; - copy_dst = (u64 *) dst; - copy_src = (u64 *) src; - if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) - { - for (i = 0; i < n / 64; i++) - { - copy_dst[0] = copy_src[0]; - copy_dst[1] = copy_src[1]; - copy_dst[2] = copy_src[2]; - copy_dst[3] = copy_src[3]; - copy_dst[4] = copy_src[4]; - copy_dst[5] = copy_src[5]; - copy_dst[6] = copy_src[6]; - copy_dst[7] = copy_src[7]; - copy_dst += 8; - copy_src += 8; - } - return; - } - for (i = 0; i < 3; i++) - { - copy_dst[0] = copy_src[0]; - copy_dst[1] = copy_src[1]; - copy_dst[2] = copy_src[2]; - copy_dst[3] = copy_src[3]; - copy_dst[4] = copy_src[4]; - copy_dst[5] = copy_src[5]; - copy_dst[6] = copy_src[6]; - copy_dst[7] = copy_src[7]; - copy_dst += 8; - copy_src += 8; - } -#endif -} - - -static uword -vxlan_gpe_export_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - ioam_export_main_t *em = &vxlan_gpe_ioam_export_main; - ioam_export_node_common (em, vm, node, frame, ip4_header_t, length, - ip_version_and_header_length, - EXPORT_NEXT_VXLAN_GPE_INPUT); - return frame->n_vectors; -} - -/* - * Node for VXLAN-GPE export - */ -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (export_node) = -{ - .function = vxlan_gpe_export_node_fn, - .name = "vxlan-gpe-ioam-export", - .vector_size = sizeof (u32), - .format_trace = format_export_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN (export_error_strings), - .error_strings = export_error_strings, - .n_next_nodes = EXPORT_N_NEXT, - /* edit / add dispositions here */ - .next_nodes = - {[EXPORT_NEXT_VXLAN_GPE_INPUT] = "vxlan-gpe-pop-ioam-v4"}, -}; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export/ioam_export.api b/plugins/ioam-plugin/ioam/export/ioam_export.api deleted file mode 100644 index f22d9fc8..00000000 --- a/plugins/ioam-plugin/ioam/export/ioam_export.api +++ /dev/null @@ -1,42 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Define a simple binary API to control the feature */ - -define ioam_export_ip6_enable_disable { - /* Client identifier, set from api_main.my_client_index */ - u32 client_index; - - /* Arbitrary context, so client can match reply to request */ - u32 context; - - /* Enable / disable the feature */ - u8 is_disable; - - /* Collector ip address */ - u8 collector_address[4]; - u8 src_address[4]; - - /* Src ip address */ -}; - -define ioam_export_ip6_enable_disable_reply { - /* From the request */ - u32 context; - - /* Return value, zero means all OK */ - i32 retval; -}; diff --git a/plugins/ioam-plugin/ioam/export/ioam_export.c b/plugins/ioam-plugin/ioam/export/ioam_export.c deleted file mode 100644 index b122e445..00000000 --- a/plugins/ioam-plugin/ioam/export/ioam_export.c +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * ioam_export.c - ioam export API / debug CLI handling - *------------------------------------------------------------------ - */ - -#include -#include -#include - -#include -#include -#include -#include - - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - - -/* List of message types that this plugin understands */ - -#define foreach_ioam_export_plugin_api_msg \ -_(IOAM_EXPORT_IP6_ENABLE_DISABLE, ioam_export_ip6_enable_disable) - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - ioam_export_main_t *em = &ioam_export_main; - clib_error_t *error = 0; - - em->vlib_main = vm; - em->vnet_main = h->vnet_main; - - return error; -} - -/* Action function shared between message handler and debug CLI */ - -int -ioam_export_ip6_enable_disable (ioam_export_main_t * em, - u8 is_disable, - ip4_address_t * collector_address, - ip4_address_t * src_address) -{ - vlib_main_t *vm = em->vlib_main; - - if (is_disable == 0) - { - if (1 == ioam_export_header_create (em, collector_address, src_address)) - { - ioam_export_thread_buffer_init (em, vm); - ip6_hbh_set_next_override (em->my_hbh_slot); - /* Turn on the export buffer check process */ - vlib_process_signal_event (vm, em->export_process_node_index, 1, 0); - - } - else - { - return (-2); - } - } - else - { - ip6_hbh_set_next_override (IP6_LOOKUP_NEXT_POP_HOP_BY_HOP); - ioam_export_header_cleanup (em, collector_address, src_address); - ioam_export_thread_buffer_free (em); - /* Turn off the export buffer check process */ - vlib_process_signal_event (vm, em->export_process_node_index, 2, 0); - - } - - return 0; -} - -/* API message handler */ -static void vl_api_ioam_export_ip6_enable_disable_t_handler - (vl_api_ioam_export_ip6_enable_disable_t * mp) -{ - vl_api_ioam_export_ip6_enable_disable_reply_t *rmp; - ioam_export_main_t *sm = &ioam_export_main; - int rv; - - rv = ioam_export_ip6_enable_disable (sm, (int) (mp->is_disable), - (ip4_address_t *) - mp->collector_address, - (ip4_address_t *) mp->src_address); - - REPLY_MACRO (VL_API_IOAM_EXPORT_IP6_ENABLE_DISABLE_REPLY); -} - -/* Set up the API message handling tables */ -static clib_error_t * -ioam_export_plugin_api_hookup (vlib_main_t * vm) -{ - ioam_export_main_t *sm = &ioam_export_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_ioam_export_plugin_api_msg; -#undef _ - - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (ioam_export_main_t * sm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); - foreach_vl_msg_name_crc_ioam_export; -#undef _ -} - -static clib_error_t * -set_ioam_export_ipfix_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - ioam_export_main_t *em = &ioam_export_main; - ip4_address_t collector, src; - u8 is_disable = 0; - - collector.as_u32 = 0; - src.as_u32 = 0; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "collector %U", unformat_ip4_address, &collector)) - ; - else if (unformat (input, "src %U", unformat_ip4_address, &src)) - ; - else if (unformat (input, "disable")) - is_disable = 1; - else - break; - } - - if (collector.as_u32 == 0) - return clib_error_return (0, "collector address required"); - - if (src.as_u32 == 0) - return clib_error_return (0, "src address required"); - - em->ipfix_collector.as_u32 = collector.as_u32; - em->src_address.as_u32 = src.as_u32; - - vlib_cli_output (vm, "Collector %U, src address %U", - format_ip4_address, &em->ipfix_collector, - format_ip4_address, &em->src_address); - - /* Turn on the export timer process */ - // vlib_process_signal_event (vm, flow_report_process_node.index, - //1, 0); - ioam_export_ip6_enable_disable (em, is_disable, &collector, &src); - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (set_ipfix_command, static) = -{ -.path = "set ioam export ipfix",.short_help = - "set ioam export ipfix collector src ",. - function = set_ioam_export_ipfix_command_fn,}; -/* *INDENT-ON* */ - - -static clib_error_t * -ioam_export_init (vlib_main_t * vm) -{ - ioam_export_main_t *em = &ioam_export_main; - clib_error_t *error = 0; - u8 *name; - u32 node_index = export_node.index; - vlib_node_t *ip6_hbyh_node = NULL; - - name = format (0, "ioam_export_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - em->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - em->unix_time_0 = (u32) time (0); /* Store starting time */ - em->vlib_time_0 = vlib_time_now (vm); - - error = ioam_export_plugin_api_hookup (vm); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (em, &api_main); - - /* Hook this export node to ip6-hop-by-hop */ - ip6_hbyh_node = vlib_get_node_by_name (vm, (u8 *) "ip6-hop-by-hop"); - em->my_hbh_slot = vlib_node_add_next (vm, ip6_hbyh_node->index, node_index); - vec_free (name); - - return error; -} - -VLIB_INIT_FUNCTION (ioam_export_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/export/ioam_export_all_api_h.h b/plugins/ioam-plugin/ioam/export/ioam_export_all_api_h.h deleted file mode 100644 index bc4368f2..00000000 --- a/plugins/ioam-plugin/ioam/export/ioam_export_all_api_h.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/ioam-plugin/ioam/export/ioam_export_msg_enum.h b/plugins/ioam-plugin/ioam/export/ioam_export_msg_enum.h deleted file mode 100644 index c2de7988..00000000 --- a/plugins/ioam-plugin/ioam/export/ioam_export_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_ioam_export_msg_enum_h -#define included_ioam_export_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_ioam_export_msg_enum_h */ diff --git a/plugins/ioam-plugin/ioam/export/ioam_export_test.c b/plugins/ioam-plugin/ioam/export/ioam_export_test.c deleted file mode 100644 index f991fc0c..00000000 --- a/plugins/ioam-plugin/ioam/export/ioam_export_test.c +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * ioam_export_test.c - test harness plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include - - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} export_test_main_t; - -export_test_main_t export_test_main; - -#define foreach_standard_reply_retval_handler \ -_(ioam_export_ip6_enable_disable_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = export_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(IOAM_EXPORT_IP6_ENABLE_DISABLE_REPLY, ioam_export_ip6_enable_disable_reply) - - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - -static int -api_ioam_export_ip6_enable_disable (vat_main_t * vam) -{ - export_test_main_t *sm = &export_test_main; - unformat_input_t *i = vam->input; - f64 timeout; - int is_disable = 0; - vl_api_ioam_export_ip6_enable_disable_t *mp; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "disable")) - is_disable = 1; - else - break; - } - - /* Construct the API message */ - M (IOAM_EXPORT_IP6_ENABLE_DISABLE, ioam_export_ip6_enable_disable); - mp->is_disable = is_disable; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(ioam_export_ip6_enable_disable, " [disable]") - -void -vat_api_hookup (vat_main_t * vam) -{ - export_test_main_t *sm = &export_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * -vat_plugin_register (vat_main_t * vam) -{ - export_test_main_t *sm = &export_test_main; - u8 *name; - - sm->vat_main = vam; - - name = format (0, "ioam_export_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~ 0) - vat_api_hookup (vam); - - vec_free (name); - - return 0; -} diff --git a/plugins/ioam-plugin/ioam/export/ioam_export_thread.c b/plugins/ioam-plugin/ioam/export/ioam_export_thread.c deleted file mode 100644 index d2eb2009..00000000 --- a/plugins/ioam-plugin/ioam/export/ioam_export_thread.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * ioam_export_thread.c - */ -#include -#include -#include - -static vlib_node_registration_t ioam_export_process_node; - -static uword -ioam_export_process (vlib_main_t * vm, - vlib_node_runtime_t * rt, vlib_frame_t * f) -{ - return (ioam_export_process_common(&ioam_export_main, - vm, rt, f, - ioam_export_process_node.index)); -} - -VLIB_REGISTER_NODE (ioam_export_process_node, static) = -{ - .function = ioam_export_process, - .type = VLIB_NODE_TYPE_PROCESS, - .name = "ioam-export-process", -}; diff --git a/plugins/ioam-plugin/ioam/export/jvpp_ioam_export.c b/plugins/ioam-plugin/ioam/export/jvpp_ioam_export.c deleted file mode 100644 index 27d3e214..00000000 --- a/plugins/ioam-plugin/ioam/export/jvpp_ioam_export.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#include -#include -#include - -#if VPPJNI_DEBUG == 1 - #define DEBUG_LOG(...) clib_warning(__VA_ARGS__) -#else - #define DEBUG_LOG(...) -#endif - -#include - -#include "ioam/jvpp/io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl.h" -#include "jvpp_ioam_export.h" -#include "ioam/jvpp/jvpp_ioam_export_gen.h" - -/* - * Class: io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl - * Method: init0 - * Signature: (JI)V - */ -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl_init0 - (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) { - ioamexport_main_t * plugin_main = &ioamexport_main; - u8 * name; - clib_warning ("Java_io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl_init0"); - - plugin_main->my_client_index = my_client_index; - plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address; - - name = format (0, "ioam_export_%08x%c", api_version, 0); - plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (plugin_main->msg_id_base == (u16) ~0) { - jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException"); - (*env)->ThrowNew(env, exClass, "ioam_export plugin is not loaded in VPP"); - } else { - plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback); - plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback)); - - #define _(N,n) \ - vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_api_reply_handler; - #undef _ - } -} - -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl_close0 -(JNIEnv *env, jclass clazz) { - ioamexport_main_t * plugin_main = &ioamexport_main; - - // cleanup: - (*env)->DeleteGlobalRef(env, plugin_main->callbackClass); - (*env)->DeleteGlobalRef(env, plugin_main->callbackObject); - - plugin_main->callbackClass = NULL; - plugin_main->callbackObject = NULL; -} - -/* Attach thread to JVM and cache class references when initiating JVPP iOAM EXPORT */ -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv* env; - - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return JNI_EVERSION; - } - - if (cache_class_references(env) != 0) { - clib_warning ("Failed to cache class references\n"); - return JNI_ERR; - } - - return JNI_VERSION_1_8; -} - -/* Clean up cached references when disposing JVPP iOAM EXPORT */ -void JNI_OnUnload(JavaVM *vm, void *reserved) { - JNIEnv* env; - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return; - } - delete_class_references(env); -} diff --git a/plugins/ioam-plugin/ioam/export/jvpp_ioam_export.h b/plugins/ioam-plugin/ioam/export/jvpp_ioam_export.h deleted file mode 100644 index b6c0c16e..00000000 --- a/plugins/ioam-plugin/ioam/export/jvpp_ioam_export.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_jvpp_ioam_export_h__ -#define __included_jvpp_ioam_export_h__ - -#include -#include -#include -#include -#include -#include - -/* Global state for JVPP-IOAM-EXPORT */ -typedef struct { - /* Base message index for the export plugin */ - u16 msg_id_base; - - /* Pointer to shared memory queue */ - unix_shared_memory_queue_t * vl_input_queue; - - /* VPP api client index */ - u32 my_client_index; - - /* Callback object and class references enabling asynchronous Java calls */ - jobject callbackObject; - jclass callbackClass; - -} ioamexport_main_t; - -ioamexport_main_t ioamexport_main __attribute__((aligned (64))); - - -#endif /* __included_jvpp_ioam_export_h__ */ diff --git a/plugins/ioam-plugin/ioam/export/node.c b/plugins/ioam-plugin/ioam/export/node.c deleted file mode 100644 index 19f143df..00000000 --- a/plugins/ioam-plugin/ioam/export/node.c +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -typedef struct -{ - u32 next_index; - u32 flow_label; -} export_trace_t; - -/* packet trace format function */ -static u8 * -format_export_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - export_trace_t *t = va_arg (*args, export_trace_t *); - - s = format (s, "EXPORT: flow_label %d, next index %d", - t->flow_label, t->next_index); - return s; -} - -vlib_node_registration_t export_node; - -#define foreach_export_error \ -_(RECORDED, "Packets recorded for export") - -typedef enum -{ -#define _(sym,str) EXPORT_ERROR_##sym, - foreach_export_error -#undef _ - EXPORT_N_ERROR, -} export_error_t; - -static char *export_error_strings[] = { -#define _(sym,string) string, - foreach_export_error -#undef _ -}; - -typedef enum -{ - EXPORT_NEXT_POP_HBYH, - EXPORT_N_NEXT, -} export_next_t; - -always_inline void -copy3cachelines (void *dst, const void *src, size_t n) -{ -#if 0 - if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) - { - /* Copy only the first 1/2 cache lines whatever is available */ - if (n >= 64) - clib_mov64 ((u8 *) dst, (const u8 *) src); - if (n >= 128) - clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); - return; - } - clib_mov64 ((u8 *) dst, (const u8 *) src); - clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); - clib_mov64 ((u8 *) dst + 128, (const u8 *) src + 128); -#endif -#if 1 - - u64 *copy_dst, *copy_src; - int i; - copy_dst = (u64 *) dst; - copy_src = (u64 *) src; - if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) - { - for (i = 0; i < n / 64; i++) - { - copy_dst[0] = copy_src[0]; - copy_dst[1] = copy_src[1]; - copy_dst[2] = copy_src[2]; - copy_dst[3] = copy_src[3]; - copy_dst[4] = copy_src[4]; - copy_dst[5] = copy_src[5]; - copy_dst[6] = copy_src[6]; - copy_dst[7] = copy_src[7]; - copy_dst += 8; - copy_src += 8; - } - return; - } - for (i = 0; i < 3; i++) - { - copy_dst[0] = copy_src[0]; - copy_dst[1] = copy_src[1]; - copy_dst[2] = copy_src[2]; - copy_dst[3] = copy_src[3]; - copy_dst[4] = copy_src[4]; - copy_dst[5] = copy_src[5]; - copy_dst[6] = copy_src[6]; - copy_dst[7] = copy_src[7]; - copy_dst += 8; - copy_src += 8; - } -#endif -} - -static uword -ip6_export_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - ioam_export_main_t *em = &ioam_export_main; - ioam_export_node_common(em, vm, node, frame, ip6_header_t, payload_length, - ip_version_traffic_class_and_flow_label, - EXPORT_NEXT_POP_HBYH); - return frame->n_vectors; -} - -/* - * Node for IP6 export - */ -VLIB_REGISTER_NODE (export_node) = -{ - .function = ip6_export_node_fn, - .name = "ip6-export", - .vector_size = sizeof (u32), - .format_trace = format_export_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN (export_error_strings), - .error_strings = export_error_strings, - .n_next_nodes = EXPORT_N_NEXT, - /* edit / add dispositions here */ - .next_nodes = - { - [EXPORT_NEXT_POP_HBYH] = "ip6-pop-hop-by-hop" - }, -}; diff --git a/plugins/ioam-plugin/ioam/ioam_plugin_doc.md b/plugins/ioam-plugin/ioam/ioam_plugin_doc.md deleted file mode 100644 index 343abcf7..00000000 --- a/plugins/ioam-plugin/ioam/ioam_plugin_doc.md +++ /dev/null @@ -1,464 +0,0 @@ -## VPP Inband OAM (iOAM) {#ioam_plugin_doc} - -In-band OAM (iOAM) is an implementation study to record operational -information in the packet while the packet traverses a path between -two points in the network. - -Overview of iOAM can be found in [iOAM-Devnet] page. -The following IETF drafts detail the motivation and mechanism for -recording operational information: - - [iOAM-ietf-requirements] - Describes motivation and usecases for iOAM - - [iOAM-ietf-data] - Describes data records that can be collected using iOAM - - [iOAM-ietf-transport] - Lists out the transport protocols - and mechanism to carry iOAM data records - - [iOAM-ietf-proof-of-transit] - Describes the idea of Proof of Transit (POT) - and mechanisms to operationalize the idea - -## Terminology -In-band OAM is expected to be deployed in a specific domain rather -than on the overall Internet. The part of the network which employs in-band OAM -is referred to as **"in-band OAM-domain"**. - -In-band OAM data is added to a packet on entering the in-band OAM-domain -and is removed from the packet when exiting the domain. -Within the in-band OAM-domain, network nodes that the packet traverses -may update the in-band OAM data records. - -- The node which adds in-band OAM data to the packet is called the -**"in-band OAM encapsulating node"**. - -- The node which removes the in-band OAM data is referred to as the -**"in-band OAM decapsulating node"**. - -- Nodes within the domain which are aware of in-band OAM data and read -and/or write or process the in-band OAM data are called -**"in-band OAM transit nodes"**. - -## Features supported in the current release -VPP can function as in-band OAM encapsulating, transit and decapsulating node. -In this version of VPP in-band OAM data is transported as options in an -IPv6 hop-by-hop extension header. Hence in-band OAM can be enabled -for IPv6 traffic. - -The following iOAM features are supported: - -- **In-band OAM Tracing** : In-band OAM supports multiple data records to be -recorded in the packet as the packet traverses the network. -These data records offer insights into the operational behavior of the network. -The following information can be collected in the tracing -data from the nodes a packet traverses: - - Node ID - - Ingress interface ID - - Egress interface ID - - Timestamp - - Pre-configured application data - -- **In-band OAM Proof of Transit (POT)**: Proof of transit iOAM data is -added to every packet for verifying that a packet traverses a specific -set of nodes. -In-band OAM data is updated at every node that is enabled with iOAM -proof of transit and is used to verify whether a packet traversed -all the specified nodes. When the verifier receives each packet, -it can validate whether the packet traversed the specified nodes. - - -## Configuration -Configuring iOAM involves: -- Selecting the packets for which iOAM data must be inserted, updated or removed - - Selection of packets for iOAM data insertion on iOAM encapsulating node. - Selection of packets is done by 5-tuple based classification - - Selection of packets for updating iOAM data is implicitly done on the - presence of iOAM options in the packet - - Selection of packets for removing the iOAM data is done on 5-tuple - based classification -- The kind of data to be collected - - Tracing data - - Proof of transit -- Additional details for processing iOAM data to be collected - - For trace data - trace type, number of nodes to be recorded in the trace, - time stamp precision, etc. - - For POT data - configuration of POT profile required to process the POT data - -The CLI for configuring iOAM is explained here followed by detailed steps -and examples to deploy iOAM on VPP as an encapsulating, transit or -decapsulating iOAM node in the subsequent sub-sections. - -VPP iOAM configuration for enabling trace and POT is as follows: - - set ioam rewrite trace-type <0x1f|0x7|0x9|0x11|0x19> - trace-elts trace-tsp <0|1|2|3> - node-id app-data [pot] - -A description of each of the options of the CLI follows: -- trace-type : An entry in the "Node data List" array of the trace option -can have different formats, following the needs of the a deployment. -For example: Some deployments might only be interested -in recording the node identifiers, whereas others might be interested -in recording node identifier and timestamp. -The following types are currently supported: - - 0x1f : Node data to include hop limit (8 bits), node ID (24 bits), - ingress and egress interface IDs (16 bits each), timestamp (32 bits), - application data (32 bits) - - 0x7 : Node data to include hop limit (8 bits), node ID (24 bits), - ingress and egress interface IDs (16 bits each) - - 0x9 : Node data to include hop limit (8 bits), node ID (24 bits), - timestamp (32 bits) - - 0x11: Node data to include hop limit (8 bits), node ID (24 bits), - application data (32 bits) - - 0x19: Node data to include hop limit (8 bits), node ID (24 bits), - timestamp (32 bits), application data (32 bits) -- trace-elts : Defines the length of the node data array in the trace option. -- trace-tsp : Defines the timestamp precision to use with the enumerated value - for precision as follows: - - 0 : 32bits timestamp in seconds - - 1 : 32bits timestamp in milliseconds - - 2 : 32bits timestamp in microseconds - - 3 : 32bits timestamp in nanoseconds -- node-id : Unique identifier for the node, included in the node ID - field of the node data in trace option. -- app-data : The value configured here is included as is in -application data field of node data in trace option. -- pot : Enables POT option to be included in the iOAM options. - -### Trace configuration - -#### On in-band OAM encapsulating node - - **Configure classifier and apply ACL** to select packets for - iOAM data insertion - - Example to enable iOAM data insertion for all the packets - towards IPv6 address db06::06: - - vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst - - vpp# classify session acl-hit-next node ip6-add-hop-by-hop - table-index 0 match l3 ip6 dst db06::06 - - vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 - - - **Enable tracing** : Specify node ID, maximum number of nodes for which - trace data should be recorded, type of data to be included for recording, - optionally application data to be included - - Example to enable tracing with a maximum of 4 nodes recorded - and the data to be recorded to include - hop limit, node id, - ingress and egress interface IDs, timestamp (millisecond precision), - application data (0x1234): - - - vpp# set ioam rewrite trace-type 0x1f trace-elts 4 trace-tsp 1 - node-id 0x1 app-data 0x1234 - - - -#### On in-band OAM transit node -- The transit node requires trace type, timestamp precision, node ID and -optionally application data to be configured, -to update its node data in the trace option. - -Example: - - vpp# set ioam rewrite trace-type 0x1f trace-elts 4 trace-tsp 1 - node-id 0x2 app-data 0x1234 - -#### On the In-band OAM decapsulating node -- The decapsulating node similar to encapsulating node requires -**classification** of the packets to remove iOAM data from. - - Example to decapsulate iOAM data for packets towards - db06::06, configure classifier and enable it as an ACL as follows: - - - vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst - - vpp# classify session acl-hit-next node ip6-lookup table-index 0 - match l3 ip6 dst db06::06 opaque-index 100 - - vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 - - -- Decapsulating node requires trace type, timestamp precision, -node ID and optionally application data to be configured, -to update its node data in the trace option before it is decapsulated. - -Example: - - vpp# set ioam rewrite trace-type 0x1f trace-elts 4 - trace-tsp 1 node-id 0x3 app-data 0x1234 - - -### Proof of Transit configuration - -For details on proof-of-transit, -see the IETF draft [iOAM-ietf-proof-of-transit]. -To enable Proof of Transit all the nodes that participate -and hence are verified for transit need a proof of transit profile. -A script to generate a proof of transit profile as per the mechanism -described in [iOAM-ietf-proof-of-transit] will be available at [iOAM-Devnet]. - -The Proof of transit mechanism implemented here is based on -Shamir's Secret Sharing algorithm. -The overall algorithm uses two polynomials -POLY-1 and POLY-2. The degree of polynomials depends on number of nodes -to be verified for transit. -POLY-1 is secret and constant. Each node gets a point on POLY-1 -at setup-time and keeps it secret. -POLY-2 is public, random and per packet. -Each node is assigned a point on POLY-1 and POLY-2 with the same x index. -Each node derives its point on POLY-2 each time a packet arrives at it. -A node then contributes its points on POLY-1 and POLY-2 to construct -POLY-3 (POLY-3 = POLY-1 + POLY-2) using lagrange extrapolation and -forwards it towards the verifier by updating POT data in the packet. -The verifier constructs POLY-3 from the accumulated value from all the nodes -and its own points on POLY-1 and POLY-2 and verifies whether -POLY-3 = POLY-1 + POLY-2. Only the verifier knows POLY-1. -The solution leverages finite field arithmetic in a field of size "prime number" -for reasons explained in description of Shamir's secret sharing algorithm. - -Here is an explanation of POT profile list and profile configuration CLI to -realize the above mechanism. -It is best to use the script provided at [iOAM-Devnet] to generate -this configuration. -- **Create POT profile** : set pot profile name id [0-1] -[validator-key 0xu64] prime-number 0xu64 secret_share 0xu64 -lpc 0xu64 polynomial2 0xu64 bits-in-random [0-64] - - name : Profile list name. - - id : Profile id, it can be 0 or 1. - A maximum of two profiles can be configured per profile list. - - validator-key : Secret key configured only on the - verifier/decapsulating node used to compare and verify proof of transit. - - prime-number : Prime number for finite field arithmetic as required by the - proof of transit mechanism. - - secret_share : Unique point for each node on the secret polynomial POLY-1. - - lpc : Lagrange Polynomial Constant(LPC) calculated per node based on - its point (x value used for evaluating the points on the polynomial) - on the polynomial used in lagrange extrapolation - for reconstructing polynomial (POLY-3). - - polynomial2 : Is the pre-evaluated value of the point on - 2nd polynomial(POLY-2). This is unique for each node. - It is pre-evaluated for all the coefficients of POLY-2 except - for the constant part of the polynomial that changes per packet - and is received as part of the POT data in the packet. - - bits-in-random : To control the size of the random number to be - generated. This number has to match the other numbers generated and used - in the profile as per the algorithm. - -- **Set a configured profile as active/in-use** : -set pot profile-active name ID [0-1] - - name : Name of the profile list to be used for computing - POT data per packet. - - ID : Identifier of the profile within the list to be used. - -#### On In-band OAM encapsulating node - - Configure the classifier and apply ACL to select packets for iOAM data insertion. - - Example to enable iOAM data insertion for all the packet towards - IPv6 address db06::06 - - - - vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst - - vpp# classify session acl-hit-next node - ip6-add-hop-by-hop table-index 0 match l3 ip6 dst db06::06 - - vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 - - - - Configure the proof of transit profile list with profiles. -Each profile list referred to by a name can contain 2 profiles, -only one is in use for updating proof of transit data at any time. - - Example profile list example with a profile generated from the - script to verify transit through 3 nodes is: - - - vpp# set pot profile name example id 0 prime-number 0x7fff0000fa884685 - secret_share 0x6c22eff0f45ec56d lpc 0x7fff0000fa884682 - polynomial2 0xffb543d4a9c bits-in-random 63 - - - Enable one of the profiles from the configured profile list as active - so that is will be used for calculating proof of transit - -Example enable profile ID 0 from profile list example configured above: - - - vpp# set pot profile-active name example ID 0 - - - - Enable POT option to be inserted - - - vpp# set ioam rewrite pot - - -#### On in-band OAM transit node - - Configure the proof of transit profile list with profiles for transit node. -Example: - - - vpp# set pot profile name example id 0 prime-number 0x7fff0000fa884685 - secret_share 0x564cdbdec4eb625d lpc 0x1 - polynomial2 0x23f3a227186a bits-in-random 63 - -#### On in-band OAM decapsulating node / verifier -- The decapsulating node, similar to the encapsulating node requires -classification of the packets to remove iOAM data from. - - Example to decapsulate iOAM data for packets towards db06::06 - configure classifier and enable it as an ACL as follows: - - - vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst - - vpp# classify session acl-hit-next node ip6-lookup table-index 0 - match l3 ip6 dst db06::06 opaque-index 100 - - vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 - -- To update and verify the proof of transit, POT profile list should be configured. - - Example POT profile list configured as follows: - - vpp# set pot profile name example id 0 validate-key 0x7fff0000fa88465d - prime-number 0x7fff0000fa884685 secret_share 0x7a08fbfc5b93116d lpc 0x3 - polynomial2 0x3ff738597ce bits-in-random 63 - -## Operational data - -Following CLIs are available to check iOAM operation: -- To check iOAM configuration that are effective use "show ioam summary" - -Example: - - vpp# show ioam summary - REWRITE FLOW CONFIGS - Not configured - HOP BY HOP OPTIONS - TRACE CONFIG - - Trace Type : 0x1f (31) - Trace timestamp precision : 1 (Milliseconds) - Num of trace nodes : 4 - Node-id : 0x2 (2) - App Data : 0x1234 (4660) - POT OPTION - 1 (Enabled) - Try 'show ioam pot and show pot profile' for more information - -- To find statistics about packets for which iOAM options were -added (encapsulating node) and removed (decapsulating node) execute -*show errors* - -Example on encapsulating node: - - - vpp# show error - Count Node Reason - 1208804706 ip6-inacl input ACL hits - 1208804706 ip6-add-hop-by-hop Pkts w/ added ip6 hop-by-hop options - -Example on decapsulating node: - - vpp# show error - Count Node Reason - 69508569 ip6-inacl input ACL hits - 69508569 ip6-pop-hop-by-hop Pkts w/ removed ip6 hop-by-hop options - -- To check the POT profiles use "show pot profile" - -Example: - - vpp# show pot profile - Profile list in use : example - POT Profile at index: 0 - ID : 0 - Validator : False (0) - Secret share : 0x564cdbdec4eb625d (6218586935324795485) - Prime number : 0x7fff0000fa884685 (9223090566081300101) - 2nd polynomial(eval) : 0x23f3a227186a (39529304496234) - LPC : 0x1 (1) - Bit mask : 0x7fffffffffffffff (9223372036854775807) - Profile index in use: 0 - Pkts passed : 0x36 (54) - -- To get statistics of POT for packets use "show ioam pot" - -Example at encapsulating or transit node: - - vpp# show ioam pot - Pkts with ip6 hop-by-hop POT options - 54 - Pkts with ip6 hop-by-hop POT options but no profile set - 0 - Pkts with POT in Policy - 0 - Pkts with POT out of Policy - 0 - - -Example at decapsulating/verification node: - - - vpp# show ioam pot - Pkts with ip6 hop-by-hop POT options - 54 - Pkts with ip6 hop-by-hop POT options but no profile set - 0 - Pkts with POT in Policy - 54 - Pkts with POT out of Policy - 0 - -- Tracing - enable trace of IPv6 packets to view the data inserted and -collected. - -Example when the nodes are receiving data over a DPDK interface: -Enable tracing using "trace add dpdk-input 20" and -execute "show trace" to view the iOAM data collected: - - - vpp# trace add dpdk-input 20 - - vpp# show trace - - ------------------- Start of thread 0 vpp_main ------------------- - - Packet 1 - - 00:00:19:294697: dpdk-input - GigabitEthernetb/0/0 rx queue 0 - buffer 0x10e6b: current data 0, length 214, free-list 0, totlen-nifb 0, trace 0x0 - PKT MBUF: port 0, nb_segs 1, pkt_len 214 - buf_len 2176, data_len 214, ol_flags 0x0, data_off 128, phys_addr 0xe9a35a00 - packet_type 0x0 - IP6: 00:50:56:9c:df:72 -> 00:50:56:9c:be:55 - IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 - tos 0x00, flow label 0x0, hop limit 63, payload length 160 - 00:00:19:294737: ethernet-input - IP6: 00:50:56:9c:df:72 -> 00:50:56:9c:be:55 - 00:00:19:294753: ip6-input - IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 - tos 0x00, flow label 0x0, hop limit 63, payload length 160 - 00:00:19:294757: ip6-lookup - fib 0 adj-idx 15 : indirect via db05::2 flow hash: 0x00000000 - IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 - tos 0x00, flow label 0x0, hop limit 63, payload length 160 - 00:00:19:294802: ip6-hop-by-hop - IP6_HOP_BY_HOP: next index 5 len 96 traced 96 Trace Type 0x1f , 1 elts left - [0] ttl 0x0 node ID 0x0 ingress 0x0 egress 0x0 ts 0x0 - app 0x0 - [1] ttl 0x3e node ID 0x3 ingress 0x1 egress 0x2 ts 0xb68c2213 - app 0x1234 - [2] ttl 0x3f node ID 0x2 ingress 0x1 egress 0x2 ts 0xb68c2204 - app 0x1234 - [3] ttl 0x40 node ID 0x1 ingress 0x5 egress 0x6 ts 0xb68c2200 - app 0x1234 - POT opt present - random = 0x577a916946071950, Cumulative = 0x10b46e78a35a392d, Index = 0x0 - 00:00:19:294810: ip6-rewrite - tx_sw_if_index 1 adj-idx 14 : GigabitEthernetb/0/0 - IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 flow hash: 0x00000000 - IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 - IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 - tos 0x00, flow label 0x0, hop limit 62, payload length 160 - 00:00:19:294814: GigabitEthernetb/0/0-output - GigabitEthernetb/0/0 - IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 - IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 - tos 0x00, flow label 0x0, hop limit 62, payload length 160 - 00:00:19:294820: GigabitEthernetb/0/0-tx - GigabitEthernetb/0/0 tx queue 0 - buffer 0x10e6b: current data 0, length 214, free-list 0, totlen-nifb 0, trace 0x0 - IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 - - IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 - - tos 0x00, flow label 0x0, hop limit 62, payload length 160 - - -[iOAM-Devnet]: -[iOAM-ietf-requirements]: -[iOAM-ietf-transport]: -[iOAM-ietf-data]: -[iOAM-ietf-proof-of-transit]: diff --git a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java b/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java deleted file mode 100644 index cb85f005..00000000 --- a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.ioamexport.test; - -import java.net.InetAddress; - -import io.fd.vpp.jvpp.JVpp; -import io.fd.vpp.jvpp.JVppRegistry; -import io.fd.vpp.jvpp.JVppRegistryImpl; -import io.fd.vpp.jvpp.VppCallbackException; -import io.fd.vpp.jvpp.ioamexport.JVppIoamexportImpl; -import io.fd.vpp.jvpp.ioamexport.future.FutureJVppIoamexportFacade; -import io.fd.vpp.jvpp.ioamexport.dto.IoamExportIp6EnableDisable; -import io.fd.vpp.jvpp.ioamexport.dto.IoamExportIp6EnableDisableReply; - -public class IoamExportApiTest { - - public static void main(String[] args) throws Exception { - ioamExportTestApi(); - } - - private static void ioamExportTestApi() throws Exception { - System.out.println("Testing Java API for ioam export plugin"); - try (final JVppRegistry registry = new JVppRegistryImpl("ioamExportApiTest"); - final JVpp jvpp = new JVppIoamexportImpl()) { - FutureJVppIoamexportFacade ioamexportJvpp = new FutureJVppIoamexportFacade(registry,jvpp); - System.out.println("Sending ioam export request..."); - IoamExportIp6EnableDisable request = new IoamExportIp6EnableDisable(); - request.isDisable = 0; - InetAddress collectorAddress = InetAddress.getByName("2001:0DB8:AC10:FE01:0000:0000:0000:0000"); - InetAddress srcAddress = InetAddress.getByName("2001:0DB8:AC10:FE01:0000:0000:0000:0001"); - request.collectorAddress = collectorAddress.getAddress(); - request.srcAddress = srcAddress.getAddress(); - IoamExportIp6EnableDisableReply reply = ioamexportJvpp.ioamExportIp6EnableDisable(request).toCompletableFuture().get(); - System.out.printf("IoamExportIp6EnableDisableReply = "+reply.toString()+"%n"); - - Thread.sleep(1000); - - System.out.println("Disconnecting..."); - } - } -} diff --git a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/Readme.txt b/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/Readme.txt deleted file mode 100644 index 1b38c285..00000000 --- a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamexport/test/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -sudo java -cp build-vpp_debug-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp_debug-native/plugins/ioam-plugin/jvpp-ioam-export-1.0.jar io.fd.vpp.jvpp.ioamexport.test.IoamExportApiTest diff --git a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java b/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java deleted file mode 100644 index 74eb86a1..00000000 --- a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.ioampot.test; - -import io.fd.vpp.jvpp.JVpp; -import io.fd.vpp.jvpp.JVppRegistry; -import io.fd.vpp.jvpp.JVppRegistryImpl; -import io.fd.vpp.jvpp.VppCallbackException; -import io.fd.vpp.jvpp.ioampot.JVppIoampotImpl; -import io.fd.vpp.jvpp.ioampot.callback.PotProfileAddCallback; -import io.fd.vpp.jvpp.ioampot.dto.PotProfileAdd; -import io.fd.vpp.jvpp.ioampot.dto.PotProfileAddReply; - -public class IoamPotApiTest { - - static class IoamPotTestCallback implements PotProfileAddCallback { - - @Override - public void onPotProfileAddReply(final PotProfileAddReply reply) { - System.out.printf("Received PotProfileAddReply reply: context=%d%n", - reply.context); - } - - @Override - public void onError(VppCallbackException ex) { - System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(), - ex.getCtxId(), ex.getErrorCode()); - } - } - - public static void main(String[] args) throws Exception { - ioamPotTestApi(); - } - - private static void ioamPotTestApi() throws Exception { - System.out.println("Testing Java API for ioam pot plugin"); - try (final JVppRegistry registry = new JVppRegistryImpl("ioamPotApiTest"); - final JVpp jvpp = new JVppIoampotImpl()) { - registry.register(jvpp, new IoamPotTestCallback()); - - System.out.println("Sending ioam pot profile add request..."); - PotProfileAdd request = new PotProfileAdd(); - request.id = 0; - request.validator = 4; - request.secretKey = 1; - request.secretShare = 2; - request.prime = 1234; - request.maxBits = 53; - request.lpc = 1234; - request.polynomialPublic = 1234; - request.listNameLen = (byte)"test pot profile".getBytes().length; - request.listName = "test pot profile".getBytes(); - final int result = jvpp.send(request); - System.out.printf("PotProfileAdd send result = %d%n", result); - - Thread.sleep(1000); - - System.out.println("Disconnecting..."); - } - } -} diff --git a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/Readme.txt b/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/Readme.txt deleted file mode 100644 index 2323494d..00000000 --- a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioampot/test/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -sudo java -cp build-vpp_debug-native/vpp-api/java/jvpp-registry-16.12.jar:build-vpp_debug-native/plugins/ioam-plugin/jvpp-ioam-pot-1.0.jar io.fd.vpp.jvpp.ioampot.test.IoamPotApiTest diff --git a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java b/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java deleted file mode 100644 index bc8c1c3a..00000000 --- a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.ioamtrace.test; - -import io.fd.vpp.jvpp.JVpp; -import io.fd.vpp.jvpp.JVppRegistry; -import io.fd.vpp.jvpp.JVppRegistryImpl; -import io.fd.vpp.jvpp.VppCallbackException; -import io.fd.vpp.jvpp.ioamtrace.future.FutureJVppIoamtraceFacade; -import io.fd.vpp.jvpp.ioamtrace.JVppIoamtraceImpl; -import io.fd.vpp.jvpp.ioamtrace.callback.TraceProfileAddCallback; -import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileAdd; -import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileAddReply; -import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileShowConfig; -import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileShowConfigReply; - -public class IoamTraceApiTest { - - static class IoamTraceTestCallback implements TraceProfileAddCallback { - - @Override - public void onTraceProfileAddReply(final TraceProfileAddReply reply) { - System.out.printf("Received TraceProfileAddReply reply: context=%d%n", - reply.context); - } - - @Override - public void onError(VppCallbackException ex) { - System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(), - ex.getCtxId(), ex.getErrorCode()); - } - } - - public static void main(String[] args) throws Exception { - ioamTraceTestApi(); - } - - private static void ioamTraceTestApi() throws Exception { - System.out.println("Testing Java API for ioam trace plugin"); - try (final JVppRegistry registry = new JVppRegistryImpl("ioamTraceApiTest"); - final JVpp jvpp = new JVppIoamtraceImpl()) { - FutureJVppIoamtraceFacade ioamtraceJvpp = new FutureJVppIoamtraceFacade(registry,jvpp); - - System.out.println("Sending ioam trace profile add request..."); - TraceProfileAdd request = new TraceProfileAdd(); - request.traceType = 0x1f; - request.numElts = 4; - request.nodeId = 1; - request.traceTsp = 2; - request.appData = 1234; - final int result = jvpp.send(request); - System.out.printf("TraceProfileAdd send result = %d%n", result); - - Thread.sleep(1000); - - TraceProfileShowConfig showRequest = new TraceProfileShowConfig(); - TraceProfileShowConfigReply reply = ioamtraceJvpp.traceProfileShowConfig(showRequest).toCompletableFuture().get(); - System.out.printf("TraceProfileShowConfig result = "+ reply.toString()); - - System.out.println("Disconnecting..."); - } - } -} diff --git a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt b/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt deleted file mode 100644 index 17e45a81..00000000 --- a/plugins/ioam-plugin/ioam/jvpp/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/plugins/ioam-plugin/jvpp-ioam-trace-1.0.jar io.fd.vpp.jvpp.ioamtrace.test.IoamTraceApiTest diff --git a/plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.c b/plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.c deleted file mode 100644 index a60ae60f..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#include -#include -#include - -#if VPPJNI_DEBUG == 1 - #define DEBUG_LOG(...) clib_warning(__VA_ARGS__) -#else - #define DEBUG_LOG(...) -#endif - -#include - -#include "ioam/jvpp/io_fd_vpp_jvpp_ioampot_JVppIoampotImpl.h" -#include "jvpp_ioam_pot.h" -#include "ioam/jvpp/jvpp_ioam_pot_gen.h" - -/* - * Class: io_fd_vpp_jvpp_ioampot_JVppIoampotImpl - * Method: init0 - * Signature: (JI)V - */ -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioampot_JVppIoampotImpl_init0 - (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) { - ioampot_main_t * plugin_main = &ioampot_main; - u8 * name; - clib_warning ("Java_io_fd_vpp_jvpp_ioampot_JVppIoampotImpl_init0"); - - plugin_main->my_client_index = my_client_index; - plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address; - - name = format (0, "ioam_pot_%08x%c", api_version, 0); - plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (plugin_main->msg_id_base == (u16) ~0) { - jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException"); - (*env)->ThrowNew(env, exClass, "ioam_pot plugin is not loaded in VPP"); - } else { - plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback); - plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback)); - - #define _(N,n) \ - vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_api_reply_handler; - #undef _ - } -} - -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioampot_JVppIoampotImpl_close0 -(JNIEnv *env, jclass clazz) { - ioampot_main_t * plugin_main = &ioampot_main; - - // cleanup: - (*env)->DeleteGlobalRef(env, plugin_main->callbackClass); - (*env)->DeleteGlobalRef(env, plugin_main->callbackObject); - - plugin_main->callbackClass = NULL; - plugin_main->callbackObject = NULL; -} - -/* Attach thread to JVM and cache class references when initiating JVPP iOAM POT */ -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv* env; - - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return JNI_EVERSION; - } - - if (cache_class_references(env) != 0) { - clib_warning ("Failed to cache class references\n"); - return JNI_ERR; - } - - return JNI_VERSION_1_8; -} - -/* Clean up cached references when disposing JVPP iOAM POT */ -void JNI_OnUnload(JavaVM *vm, void *reserved) { - JNIEnv* env; - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return; - } - delete_class_references(env); -} diff --git a/plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.h b/plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.h deleted file mode 100644 index 00aa51db..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/jvpp_ioam_pot.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_jvpp_ioam_pot_h__ -#define __included_jvpp_ioam_pot_h__ - -#include -#include -#include -#include -#include -#include - -/* Global state for JVPP-IOAM-POT */ -typedef struct { - /* Base message index for the pot plugin */ - u16 msg_id_base; - - /* Pointer to shared memory queue */ - unix_shared_memory_queue_t * vl_input_queue; - - /* VPP api client index */ - u32 my_client_index; - - /* Callback object and class references enabling asynchronous Java calls */ - jobject callbackObject; - jclass callbackClass; - -} ioampot_main_t; - -ioampot_main_t ioampot_main __attribute__((aligned (64))); - - -#endif /* __included_jvpp_ioam_pot_h__ */ diff --git a/plugins/ioam-plugin/ioam/lib-pot/math64.h b/plugins/ioam-plugin/ioam/lib-pot/math64.h deleted file mode 100644 index 4c608a37..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/math64.h +++ /dev/null @@ -1,159 +0,0 @@ -/* - * math64.h provides the 64 bit unsigned integer add, multiply followed by modulo operation - * The linux/math64.h provides divide and multiply 64 bit integers but: - * 1. multiply: mul_u64_u64_shr - only returns 64 bits of the result and has to be called - * twice to get the complete 128 bits of the result. - * 2. Modulo operation of the result of addition and multiplication of u64 that may result - * in integers > 64 bits is not supported - * Hence this header to combine add/multiply followed by modulo of u64 integrers - * always resulting in u64. - * - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef include_vnet_math64_h -#define include_vnet_math64_h -#include - -/* - * multiplies and returns result in hi and lo - */ -static inline void mul64by64(u64 a, u64 b, u64 * hi, u64 * lo) -{ - u64 a_lo = (u64) (uint32_t) a; - u64 a_hi = a >> 32; - u64 b_lo = (u64) (u32) b; - u64 b_hi = b >> 32; - - u64 p0 = a_lo * b_lo; - u64 p1 = a_lo * b_hi; - u64 p2 = a_hi * b_lo; - u64 p3 = a_hi * b_hi; - - u32 cy = (u32) (((p0 >> 32) + (u32) p1 + (u32) p2) >> 32); - - *lo = p0 + (p1 << 32) + (p2 << 32); - *hi = p3 + (p1 >> 32) + (p2 >> 32) + cy; - return; -} - -#define TWO64 18446744073709551616.0 - -static inline u64 mod128by64(u64 x, u64 y, u64 m, double di) -{ - u64 q1, q2, q; - u64 p1, p0; - double dq; - - /* calculate quotient first pass 53 bits */ - dq = (TWO64 * (double)x + (double)y) * di; - - if (dq >= TWO64) - q1 = 0xfffffffffffff800L; - else - q1 = dq; - - /* q1 * m to compare the product to the dividend. */ - mul64by64(q1, m, &p1, &p0); - - /* Adjust quotient. is it > actual result: */ - if (x < p1 || (x == p1 && y < p0)) - { - /* q1 > quotient. calculate abs remainder */ - x = p1 - (x + (p0 < y)); - y = p0 - y; - - /* use the remainder as new dividend to adjust quotient */ - q2 = (u64) ((TWO64 * (double)x + (double)y) * di); - mul64by64(q2, m, &p1, &p0); - - q = q1 - q2; - if (x < p1 || (x == p1 && y <= p0)) - { - y = p0 - y; - } - else - { - y = p0 - y; - y += m; - q--; - } - } - else - { - x = x - (p1 + (y < p0)); - y = y - p0; - - q2 = (u64) ((TWO64 * (double)x + (double)y) * di); - mul64by64(q2, m, &p1, &p0); - - q = q1 + q2; - if (x < p1 || (x == p1 && y < p0)) - { - y = y - p0; - y += m; - q--; - } - else - { - y = y - p0; - if (y >= m) - { - y -= m; - q++; - } - } - } - - return y; -} - -/* - * returns a % p - */ -static inline u64 mod64by64(u64 a, u64 p, u64 primeinv) -{ - return (mod128by64(0, a, p, primeinv)); -} - -static inline void add64(u64 a, u64 b, u64 * whi, u64 * wlo) -{ - *wlo = a + b; - if (*wlo < a) - *whi = 1; - -} - -/* - * returns (a + b)%p - */ -static inline u64 add64_mod(u64 a, u64 b, u64 p, double pi) -{ - u64 shi = 0, slo = 0; - - add64(a, b, &shi, &slo); - return (mod128by64(shi, slo, p, pi)); -} - -/* - * returns (ab) % p - */ -static inline u64 mul64_mod(u64 a, u64 b, u64 p, double pi) -{ - u64 phi = 0, plo = 0; - - mul64by64(a, b, &phi, &plo); - return (mod128by64(phi, plo, p, pi)); -} - -#endif diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot.api b/plugins/ioam-plugin/ioam/lib-pot/pot.api deleted file mode 100644 index fa2fc126..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot.api +++ /dev/null @@ -1,133 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/** \brief Proof of Transit(POT): Set POT profile - @param id - id of the profile - @param validator - True/False to indicate if this is verifier - @param secret_key - Verification key - @param secret_share - Share of the 1st polynomial - @param prime - Prime number used for modulo operation - @param max_bits - Max bits to be used for Random number generation - @param lpc - Lagrange basis polynomial - @param polynomial_public - pre-evaluated public polynomial - @param list_name_len - length of the name of this profile list - @param list_name - name of this profile list -*/ -define pot_profile_add { - u32 client_index; - u32 context; - u8 id; - u8 validator; - u64 secret_key; - u64 secret_share; - u64 prime; - u8 max_bits; - u64 lpc; - u64 polynomial_public; - u8 list_name_len; - u8 list_name[0]; -}; - -/** \brief Proof of Transit profile add / del response - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define pot_profile_add_reply { - u32 context; - i32 retval; -}; - - -/** \brief Proof of Transit(POT): Activate POT profile in the list - @param id - id of the profile - @param list_name_len - length of the name of this profile list - @param list_name - name of this profile list -*/ -define pot_profile_activate { - u32 client_index; - u32 context; - u8 id; - u8 list_name_len; - u8 list_name[0]; -}; - -/** \brief Proof of Transit profile activate response - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define pot_profile_activate_reply { - u32 context; - i32 retval; -}; - -/** \brief Delete POT Profile - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param list_name_len - length of the name of the profile list - @param list_name - name of profile list to delete -*/ -define pot_profile_del { - u32 client_index; - u32 context; - u8 list_name_len; - u8 list_name[0]; -}; - -/** \brief Proof of Transit profile add / del response - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define pot_profile_del_reply { - u32 context; - i32 retval; -}; - -/** \brief Show POT Profiles - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param id - id of the profile -*/ -define pot_profile_show_config_dump { - u32 client_index; - u32 context; - u8 id; -}; - -/** \brief Show POT profile reply - @param id - id of the profile - @param validator - True/False to indicate if this is verifier - @param secret_key - Verification key - @param secret_share - Share of the 1st polynomial - @param prime - Prime number used for modulo operation - @param max_bits - Max bits to be used for Random number generation - @param lpc - Lagrange basis polynomial - @param polynomial_public - pre-evaluated public polynomial - @param list_name_len - length of the name of this profile list - @param list_name - name of this profile list -*/ -define pot_profile_show_config_details { - u32 context; - i32 retval; - u8 id; - u8 validator; - u64 secret_key; - u64 secret_share; - u64 prime; - u64 bit_mask; - u64 lpc; - u64 polynomial_public; -}; diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot_all_api_h.h b/plugins/ioam-plugin/ioam/lib-pot/pot_all_api_h.h deleted file mode 100644 index 63967c45..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot_all_api_h.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot_api.c b/plugins/ioam-plugin/ioam/lib-pot/pot_api.c deleted file mode 100644 index d3af7b40..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot_api.c +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * pot_api.c - Proof of Transit related APIs to create - * and maintain profiles - *------------------------------------------------------------------ - */ - -#include -#include -#include - -#include -#include -#include - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -#define REPLY_MACRO2(t, body) \ -do { \ - unix_shared_memory_queue_t * q; \ - rv = vl_msg_api_pd_handler (mp, rv); \ - q = vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - do {body;} while (0); \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -/* List of message types that this plugin understands */ - -#define foreach_pot_plugin_api_msg \ -_(POT_PROFILE_ADD, pot_profile_add) \ -_(POT_PROFILE_ACTIVATE, pot_profile_activate) \ -_(POT_PROFILE_DEL, pot_profile_del) \ -_(POT_PROFILE_SHOW_CONFIG_DUMP, pot_profile_show_config_dump) \ - -static void vl_api_pot_profile_add_t_handler -(vl_api_pot_profile_add_t *mp) -{ - pot_main_t * sm = &pot_main; - int rv = 0; - vl_api_pot_profile_add_reply_t * rmp; - u8 id; - pot_profile *profile = NULL; - u8 *name = 0; - - if (mp->list_name_len) - name = format(0, "%s", mp->list_name); - - pot_profile_list_init(name); - id = mp->id; - profile = pot_profile_find(id); - if (profile) { - rv = pot_profile_create(profile, - clib_net_to_host_u64(mp->prime), - clib_net_to_host_u64(mp->polynomial_public), - clib_net_to_host_u64(mp->lpc), - clib_net_to_host_u64(mp->secret_share)); - if (rv != 0) - goto ERROROUT; - if (1 == mp->validator) - (void)pot_set_validator(profile, clib_net_to_host_u64(mp->secret_key)); - (void)pot_profile_set_bit_mask(profile, mp->max_bits); - } else { - rv = -3; - } - ERROROUT: - vec_free(name); - REPLY_MACRO(VL_API_POT_PROFILE_ADD_REPLY); -} - -static void send_pot_profile_details(vl_api_pot_profile_show_config_dump_t *mp, u8 id) -{ - vl_api_pot_profile_show_config_details_t * rmp; - pot_main_t * sm = &pot_main; - pot_profile *profile = pot_profile_find(id); - int rv = 0; - if(profile){ - REPLY_MACRO2(VL_API_POT_PROFILE_SHOW_CONFIG_DETAILS, - rmp->id=id; - rmp->validator=profile->validator; - rmp->secret_key=clib_host_to_net_u64(profile->secret_key); - rmp->secret_share=clib_host_to_net_u64(profile->secret_share); - rmp->prime=clib_host_to_net_u64(profile->prime); - rmp->bit_mask=clib_host_to_net_u64(profile->bit_mask); - rmp->lpc=clib_host_to_net_u64(profile->lpc); - rmp->polynomial_public=clib_host_to_net_u64(profile->poly_pre_eval); - ); - } - else{ - REPLY_MACRO2(VL_API_POT_PROFILE_SHOW_CONFIG_DETAILS, - rmp->id=id; - rmp->validator=0; - rmp->secret_key=0; - rmp->secret_share=0; - rmp->prime=0; - rmp->bit_mask=0; - rmp->lpc=0; - rmp->polynomial_public=0; - ); - } -} - -static void vl_api_pot_profile_show_config_dump_t_handler -(vl_api_pot_profile_show_config_dump_t *mp) -{ - u8 id = mp->id; - u8 dump_call_id = ~0; - if(dump_call_id==id){ - for(id=0;idlist_name_len) - name = format(0, "%s", mp->list_name); - if (!pot_profile_list_is_enabled(name)) { - rv = -1; - } else { - id = mp->id; - rv = pot_profile_set_active(id); - } - - vec_free(name); - REPLY_MACRO(VL_API_POT_PROFILE_ACTIVATE_REPLY); -} - - -static void vl_api_pot_profile_del_t_handler -(vl_api_pot_profile_del_t *mp) -{ - pot_main_t * sm = &pot_main; - int rv = 0; - vl_api_pot_profile_del_reply_t * rmp; - - clear_pot_profiles(); - - REPLY_MACRO(VL_API_POT_PROFILE_DEL_REPLY); -} - - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - pot_main_t * sm = &pot_main; - clib_error_t * error = 0; - - sm->vlib_main = vm; - sm->vnet_main = h->vnet_main; - return error; -} - -/* Set up the API message handling tables */ -static clib_error_t * -pot_plugin_api_hookup (vlib_main_t *vm) -{ - pot_main_t * sm = &pot_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_pot_plugin_api_msg; -#undef _ - - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (pot_main_t * sm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); - foreach_vl_msg_name_crc_pot; -#undef _ -} - -static clib_error_t * pot_init (vlib_main_t * vm) -{ - pot_main_t * sm = &pot_main; - clib_error_t * error = 0; - u8 * name; - - bzero(sm, sizeof(pot_main)); - (void)pot_util_init(); - name = format (0, "ioam_pot_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - sm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - - error = pot_plugin_api_hookup (vm); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (sm, &api_main); - - vec_free(name); - - return error; -} - -VLIB_INIT_FUNCTION (pot_init); diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot_msg_enum.h b/plugins/ioam-plugin/ioam/lib-pot/pot_msg_enum.h deleted file mode 100644 index a4a88bed..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_pot_msg_enum_h -#define included_pot_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_pot_msg_enum_h */ diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot_test.c b/plugins/ioam-plugin/ioam/lib-pot/pot_test.c deleted file mode 100644 index 2e870238..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot_test.c +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * pot_test.c - test harness for pot plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} pot_test_main_t; - -pot_test_main_t pot_test_main; - -#define foreach_standard_reply_retval_handler \ -_(pot_profile_add_reply) \ -_(pot_profile_activate_reply) \ -_(pot_profile_del_reply) - -#define foreach_custom_reply_retval_handler \ -_(pot_profile_show_config_details, \ - errmsg(" ID:%d\n",mp->id); \ - errmsg(" Validator:%d\n",mp->validator); \ - errmsg(" secret_key:%Lx\n",clib_net_to_host_u64(mp->secret_key)); \ - errmsg(" secret_share:%Lx\n",clib_net_to_host_u64(mp->secret_share)); \ - errmsg(" prime:%Lx\n",clib_net_to_host_u64(mp->prime)); \ - errmsg(" bitmask:%Lx\n",clib_net_to_host_u64(mp->bit_mask)); \ - errmsg(" lpc:%Lx\n",clib_net_to_host_u64(mp->lpc)); \ - errmsg(" public poly:%Lx\n",clib_net_to_host_u64(mp->polynomial_public)); \ - ) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = pot_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -#define _(n,body) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = pot_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - do{body;}while(0); \ - } -foreach_custom_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(POT_PROFILE_ADD_REPLY, pot_profile_add_reply) \ -_(POT_PROFILE_ACTIVATE_REPLY, pot_profile_activate_reply) \ -_(POT_PROFILE_DEL_REPLY, pot_profile_del_reply) \ -_(POT_PROFILE_SHOW_CONFIG_DETAILS, pot_profile_show_config_details) - - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - - -static int api_pot_profile_add (vat_main_t *vam) -{ -#define MAX_BITS 64 - pot_test_main_t * sm = &pot_test_main; - unformat_input_t *input = vam->input; - vl_api_pot_profile_add_t *mp; - u8 *name = NULL; - u64 prime = 0; - u64 secret_share = 0; - u64 secret_key = 0; - u32 bits = MAX_BITS; - u64 lpc = 0, poly2 = 0; - f64 timeout; - u8 id = 0; - int rv = 0; - - while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(input, "name %s", &name)) - ; - else if(unformat(input, "id %d", &id)) - ; - else if (unformat(input, "validator-key 0x%Lx", &secret_key)) - ; - else if (unformat(input, "prime-number 0x%Lx", &prime)) - ; - else if (unformat(input, "secret-share 0x%Lx", &secret_share)) - ; - else if (unformat(input, "polynomial-public 0x%Lx", &poly2)) - ; - else if (unformat(input, "lpc 0x%Lx", &lpc)) - ; - else if (unformat(input, "bits-in-random %u", &bits)) - { - if (bits > MAX_BITS) - bits = MAX_BITS; - } - else - break; - } - - if (!name) - { - errmsg ("name required\n"); - rv = -99; - goto OUT; - } - - M2(POT_PROFILE_ADD, pot_profile_add, vec_len(name)); - - mp->list_name_len = vec_len(name); - clib_memcpy(mp->list_name, name, mp->list_name_len); - mp->secret_share = clib_host_to_net_u64(secret_share); - mp->polynomial_public = clib_host_to_net_u64(poly2); - mp->lpc = clib_host_to_net_u64(lpc); - mp->prime = clib_host_to_net_u64(prime); - if (secret_key != 0) - { - mp->secret_key = clib_host_to_net_u64(secret_key); - mp->validator = 1; - } - else - { - mp->validator = 0; - } - mp->id = id; - mp->max_bits = bits; - - S; W; - -OUT: - vec_free(name); - return(rv); -} - -static int api_pot_profile_activate (vat_main_t *vam) -{ -#define MAX_BITS 64 - pot_test_main_t * sm = &pot_test_main; - unformat_input_t *input = vam->input; - vl_api_pot_profile_activate_t *mp; - u8 *name = NULL; - u8 id = 0; - int rv = 0; - f64 timeout; - - while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(input, "name %s", &name)) - ; - else if(unformat(input, "id %d", &id)) - ; - else - break; - } - - if (!name) - { - errmsg ("name required\n"); - rv = -99; - goto OUT; - } - - M2(POT_PROFILE_ACTIVATE, pot_profile_activate, vec_len(name)); - - mp->list_name_len = vec_len(name); - clib_memcpy(mp->list_name, name, mp->list_name_len); - mp->id = id; - - S; W; - -OUT: - vec_free(name); - return(rv); -} - - -static int api_pot_profile_del (vat_main_t *vam) -{ - pot_test_main_t * sm = &pot_test_main; - vl_api_pot_profile_del_t *mp; - f64 timeout; - - M(POT_PROFILE_DEL, pot_profile_del); - mp->list_name_len = 0; - S; W; - return 0; -} - -static int api_pot_profile_show_config_dump (vat_main_t *vam) -{ - pot_test_main_t * sm = &pot_test_main; - unformat_input_t *input = vam->input; - vl_api_pot_profile_show_config_dump_t *mp; - f64 timeout; - u8 id = 0; - - while(unformat_check_input(input) != UNFORMAT_END_OF_INPUT) - { - if(unformat(input,"id %d",&id)); - else - break; - } - M(POT_PROFILE_SHOW_CONFIG_DUMP, pot_profile_show_config_dump); - - mp->id = id; - - S; W; - return 0; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(pot_profile_add, "name id [0-1] " \ - "prime-number <0xu64> bits-in-random [0-64] " \ - "secret-share <0xu64> lpc <0xu64> polynomial-public <0xu64> " \ - "[validator-key <0xu64>] [validity <0xu64>]") \ -_(pot_profile_activate, "name id [0-1] ") \ -_(pot_profile_del, "[id ]") \ -_(pot_profile_show_config_dump, "id [0-1]") - -void vat_api_hookup (vat_main_t *vam) -{ - pot_test_main_t * sm = &pot_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * vat_plugin_register (vat_main_t *vam) -{ - pot_test_main_t * sm = &pot_test_main; - u8 * name; - - sm->vat_main = vam; - - name = format (0, "ioam_pot_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~0) - vat_api_hookup (vam); - - vec_free(name); - - return 0; -} diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot_util.c b/plugins/ioam-plugin/ioam/lib-pot/pot_util.c deleted file mode 100644 index a253ad41..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot_util.c +++ /dev/null @@ -1,445 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include "math64.h" -#include "pot_util.h" - -pot_main_t pot_main; - -static void pot_profile_cleanup(pot_profile *profile); - -static void pot_main_profiles_reset (void) -{ - pot_main_t *sm = &pot_main; - int i = 0; - - for (i = 0; i < MAX_POT_PROFILES; i++) - { - pot_profile_cleanup(&(sm->profile_list[i])); - } - sm->active_profile_id = 0; - if (sm->profile_list_name) - vec_free(sm->profile_list_name); - sm->profile_list_name = NULL; -} - -int pot_util_init (void) -{ - pot_main_profiles_reset(); - - return(0); -} - -static void pot_profile_init(pot_profile * new, u8 id) -{ - if (new) - { - memset(new, 0, sizeof(pot_profile)); - new->id = id; - } -} - -pot_profile *pot_profile_find(u8 id) -{ - pot_main_t *sm = &pot_main; - - if (id < MAX_POT_PROFILES) - { - return (&(sm->profile_list[id])); - } - return (NULL); -} -static int pot_profile_name_equal (u8 *name0, u8 *name1) -{ - int len0, len1; - - len0 = vec_len (name0); - len1 = vec_len (name1); - if (len0 != len1) - return(0); - return (0==strncmp ((char *) name0, (char *)name1, len0)); -} - -int pot_profile_list_is_enabled (u8 *name) -{ - pot_main_t *sm = &pot_main; - return (pot_profile_name_equal(sm->profile_list_name, name)); -} - -void pot_profile_list_init(u8 * profile_list_name) -{ - pot_main_t *sm = &pot_main; - int i = 0; - - /* If it is the same profile list skip reset */ - if (pot_profile_name_equal(sm->profile_list_name, profile_list_name)) - { - return; - } - - pot_main_profiles_reset(); - if (vec_len(profile_list_name)) - sm->profile_list_name = (u8 *)vec_dup(profile_list_name); - else - sm->profile_list_name = 0; - sm->active_profile_id = 0; - - for (i = 0; i < MAX_POT_PROFILES; i++) - { - pot_profile_init(&(sm->profile_list[i]), i); - } -} - -static void pot_profile_cleanup(pot_profile * profile) -{ - u16 id = profile->id; - - memset(profile, 0, sizeof(pot_profile)); - profile->id = id; /* Restore id alone */ -} - -int pot_profile_create(pot_profile * profile, u64 prime, - u64 poly2, u64 lpc, u64 secret_share) -{ - if (profile && !profile->in_use) - { - pot_profile_cleanup(profile); - profile->prime = prime; - profile->primeinv = 1.0 / prime; - profile->lpc = lpc; - profile->poly_pre_eval = poly2; - profile->secret_share = secret_share; - profile->total_pkts_using_this_profile = 0; - profile->valid = 1; - return(0); - } - - return(-1); -} - -int pot_set_validator(pot_profile * profile, u64 key) -{ - if (profile && !profile->in_use) - { - profile->validator = 1; - profile->secret_key = key; - return(0); - } - return(-1); -} - -always_inline u64 pot_update_cumulative_inline(u64 cumulative, u64 random, - u64 secret_share, u64 prime, u64 lpc, u64 pre_split, double prime_inv) -{ - u64 share_random = 0; - u64 cumulative_new = 0; - - /* - * calculate split share for random - */ - share_random = add64_mod(pre_split, random, prime, prime_inv); - - /* - * lpc * (share_secret + share_random) - */ - share_random = add64_mod(share_random, secret_share, prime, prime_inv); - share_random = mul64_mod(share_random, lpc, prime, prime_inv); - - cumulative_new = add64_mod(cumulative, share_random, prime, prime_inv); - - return (cumulative_new); -} - -u64 pot_update_cumulative(pot_profile * profile, u64 cumulative, u64 random) -{ - if (profile && profile->valid != 0) - { - return (pot_update_cumulative_inline(cumulative, random, profile->secret_share, - profile->prime, profile->lpc, profile->poly_pre_eval, - profile->primeinv)); - } - return (0); -} - -always_inline u8 pot_validate_inline(u64 secret, u64 prime, double prime_inv, - u64 cumulative, u64 random) -{ - if (cumulative == (random + secret)) - { - return (1); - } - else if (cumulative == add64_mod(random, secret, prime, prime_inv)) - { - return (1); - } - return (0); -} - -/* - * return True if the cumulative matches secret from a profile - */ -u8 pot_validate(pot_profile * profile, u64 cumulative, u64 random) -{ - if (profile && profile->validator) - { - return (pot_validate_inline(profile->secret_key, profile->prime, - profile->primeinv, cumulative, random)); - } - return (0); -} - -/* - * Utility function to get random number per pack - */ -u64 pot_generate_random(pot_profile * profile) -{ - u64 random = 0; - int32_t second_half; - static u32 seed = 0; - - if (PREDICT_FALSE(!seed)) - seed = random_default_seed(); - - /* - * Upper 4 bytes seconds - */ - random = (u64) time(NULL); - - random &= 0xffffffff; - random = random << 32; - /* - * Lower 4 bytes random number - */ - second_half = random_u32(&seed); - - random |= second_half; - - if (PREDICT_TRUE(profile != NULL)) - { - random &= profile->bit_mask; - } - return (random); -} - -int pot_profile_set_bit_mask(pot_profile * profile, u16 bits) -{ - int sizeInBits; - - if (profile && !profile->in_use) - { - sizeInBits = sizeof(profile->bit_mask) * 8; - profile->bit_mask = - (bits >= - sizeInBits ? (u64) - 1 : (u64) ((u64) 1 << (u64) bits) - 1); - return(0); - } - return(-1); -} - -clib_error_t *clear_pot_profile_command_fn(vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - - pot_main_profiles_reset(); - - return 0; -} - -void clear_pot_profiles() -{ - clear_pot_profile_command_fn(0, 0, 0); -} - -VLIB_CLI_COMMAND(clear_pot_profile_command) = -{ -.path = "clear pot profile", -.short_help = "clear pot profile [|all]", -.function = clear_pot_profile_command_fn, -}; - -static clib_error_t *set_pot_profile_command_fn(vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - u64 prime; - u64 secret_share; - u64 secret_key; - u8 validator = 0; - u32 profile_id = ~0; - u32 bits; - u64 lpc = 0, poly2 = 0; - pot_profile *profile = NULL; - u8 *profile_list_name = NULL; - - bits = MAX_BITS; - - while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(input, "name %s", - &profile_list_name)); - else if (unformat(input, "id %d", &profile_id)) - ; - else if (unformat(input, "validate-key 0x%Lx", &secret_key)) - validator = 1; - else if (unformat(input, "prime-number 0x%Lx", &prime)) - ; - else if (unformat(input, "secret_share 0x%Lx", &secret_share)) - ; - else if (unformat(input, "polynomial2 0x%Lx", &poly2)) - ; - else if (unformat(input, "lpc 0x%Lx", &lpc)) - ; - else if (unformat(input, "bits-in-random %d", &bits)) - { - if (bits > MAX_BITS) - bits = MAX_BITS; - } - else - break; - } - if (profile_list_name == 0) - { - return clib_error_return(0, "Name cannot be null"); - } - pot_profile_list_init(profile_list_name); - profile = pot_profile_find(profile_id); - - if (profile) - { - pot_profile_create(profile, prime, poly2, lpc, secret_share); - if (validator) - pot_set_validator(profile, secret_key); - pot_profile_set_bit_mask(profile, bits); - } - vec_free(profile_list_name); - return 0; -} - -VLIB_CLI_COMMAND(set_pot_profile_command) = -{ -.path = "set pot profile", -.short_help = "set pot profile name id [0-1] [validator-key 0xu64] \ - prime-number 0xu64 secret_share 0xu64 lpc 0xu64 \ - polynomial2 0xu64 bits-in-random [0-64] ", -.function = set_pot_profile_command_fn, -}; - -static clib_error_t *set_pot_profile_activate_command_fn(vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - pot_main_t *sm = &pot_main; - u8 *profile_list_name = NULL; - u32 id = 0; - clib_error_t *result = NULL; - - while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(input, "name %s", - &profile_list_name)); - else if (unformat(input, "id %d", &id)) - ; - else - return clib_error_return(0, "unknown input `%U'", - format_unformat_error, input); - } - if (profile_list_name == 0) - { - return clib_error_return(0, "Name cannot be null"); - } - - if (!pot_profile_list_is_enabled(profile_list_name)) { - result = clib_error_return(0, "%s list is not enabled, profile in use %s", - profile_list_name, sm->profile_list_name); - } else if (0 != pot_profile_set_active((u8)id)) { - result = clib_error_return(0, "Profile %d not defined in %s", - id, sm->profile_list_name); - } - vec_free(profile_list_name); - return result; -} - -VLIB_CLI_COMMAND(set_pot_profile_activate_command) = -{ -.path = "set pot profile-active", -.short_help = "set pot profile-active name id [0-1]", -.function = set_pot_profile_activate_command_fn, -}; - -static clib_error_t *show_pot_profile_command_fn(vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - pot_main_t *sm = &pot_main; - pot_profile *p = NULL; - u16 i; - u8 *s = 0; - - if (vec_len(sm->profile_list_name) == 0) - { - s = format(s, "POT Profiles not configured\n"); - vlib_cli_output(vm, "%v", s); - return 0; - } - s = format(s, "Profile list in use : %s\n",sm->profile_list_name); - for (i = 0; i < MAX_POT_PROFILES; i++) - { - p = pot_profile_find(i); - if (p->valid == 0) - continue; - s = format(s, "POT Profile at index: %d\n", i); - s = format(s, " Id : %d\n", p->id); - s = format(s, " Validator : %s (%d)\n", - (p->validator) ? "True" : "False", p->validator); - if (p->validator == 1) - s = format(s, " Secret key : 0x%Lx (%Ld)\n", - p->secret_key, p->secret_key); - s = format(s, " Secret share : 0x%Lx (%Ld)\n", - p->secret_share, p->secret_share); - s = format(s, " Prime number : 0x%Lx (%Ld)\n", - p->prime, p->prime); - s = format(s, "2nd polynomial(eval) : 0x%Lx (%Ld)\n", - p->poly_pre_eval, p->poly_pre_eval); - s = format(s, " LPC : 0x%Lx (%Ld)\n", p->lpc, p->lpc); - - s = format(s, " Bit mask : 0x%Lx (%Ld)\n", - p->bit_mask, p->bit_mask); - } - - p = pot_profile_find(sm->active_profile_id); - - if (p && p->valid && p->in_use) { - s = format(s, "\nProfile index in use: %d\n", sm->active_profile_id); - s = format(s, "Pkts passed : 0x%Lx (%Ld)\n", - p->total_pkts_using_this_profile, - p->total_pkts_using_this_profile); - if (pot_is_decap(p)) - s = format(s, " This is Decap node. \n"); - } else { - s = format(s, "\nProfile index in use: None\n"); - } - vlib_cli_output(vm, "%v", s); - vec_free(s); - - return 0; -} - -VLIB_CLI_COMMAND(show_pot_profile_command) = -{ -.path = "show pot profile", -.short_help = "show pot profile", -.function = show_pot_profile_command_fn, -}; diff --git a/plugins/ioam-plugin/ioam/lib-pot/pot_util.h b/plugins/ioam-plugin/ioam/lib-pot/pot_util.h deleted file mode 100644 index 9df31fae..00000000 --- a/plugins/ioam-plugin/ioam/lib-pot/pot_util.h +++ /dev/null @@ -1,195 +0,0 @@ -/* - * pot_util.h -- Proof Of Transit Utility Header - * - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef include_vnet_pot_util_h -#define include_vnet_pot_util_h - -#include -#define debug_ioam debug_ioam_fn -/* Dont change this size 256. This is there across multiple components */ -#define PATH_NAME_SIZE 256 - -/* Ring size. this should be same as the one in ODL. Do not change this - without change in ODL. */ -#define MAX_POT_PROFILES 2 - -/** - * Usage: - * - * On any node that participates in Proof of Transit: - * - * Step 1: Initialize this library by calling pot_init() - * Step 2: Setup a proof of transit profile that contains all the parameters needed to compute cumulative: - * Call these functions: - * pot_profile_find - * pot_profile_create - * pot_profile_set_bit_mask - To setup how large we want the numbers used in the computation and random number <= 64 bits - * Step 2a: For validator do this: - * pot_set_validator - * Step 2b: On initial node enable the profile to be used: - * pot_profile_set_active / pot_profile_get_active will return the profile - * Step 3a: At the initial node to generate Random number that will be read by all other nodes: - * pot_generate_random - * Step 3b: At all nodes including initial and verifier call this to compute cumulative: - * pot_update_cumulative - * Step 4: At the verifier: - * pot_validate - * - */ - -typedef struct pot_profile_ -{ - u8 id : 1; - u8 valid : 1; - u8 in_use : 1; - u64 random; - u8 validator; - u64 secret_key; - u64 secret_share; - u64 prime; - u64 lpc; - u64 poly_pre_eval; - u64 bit_mask; - u64 limit; - double primeinv; - u64 total_pkts_using_this_profile; -} pot_profile; - -typedef struct { - /* Name of the default profile list in use*/ - u8 *profile_list_name; - pot_profile profile_list[MAX_POT_PROFILES]; - /* number of profiles in the list */ - u8 active_profile_id : 1; - - /* API message ID base */ - u16 msg_id_base; - - /* convenience */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; -} pot_main_t; - -extern pot_main_t pot_main; - -/* - * Initialize proof of transit - */ -int pot_util_init(void); -void pot_profile_list_init(u8 * name); - - -/* - * Find a pot profile by ID - */ -pot_profile *pot_profile_find(u8 id); - -static inline u16 pot_profile_get_id(pot_profile * profile) -{ - if (profile) - { - return (profile->id); - } - return (0); -} - -/* setup and clean up profile */ -int pot_profile_create(pot_profile * profile, u64 prime, - u64 poly2, u64 lpc, u64 secret_share); -/* - * Setup profile as a validator - */ -int pot_set_validator(pot_profile * profile, u64 key); - -/* - * Setup max bits to be used for random number generation - */ -#define MAX_BITS 64 -int pot_profile_set_bit_mask(pot_profile * profile, u16 bits); - -/* - * Given a random and cumulative compute the new cumulative for a given profile - */ -u64 pot_update_cumulative(pot_profile * profile, u64 cumulative, u64 random); - -/* - * return True if the cumulative matches secret from a profile - */ -u8 pot_validate(pot_profile * profile, u64 cumulative, u64 random); - -/* - * Utility function to get random number per pack - */ -u64 pot_generate_random(pot_profile * profile); - - -extern void clear_pot_profiles(); -extern int pot_profile_list_is_enabled(u8 *name); - -static inline u8 pot_is_decap(pot_profile * p) -{ - return (p->validator == 1); -} - -static inline int pot_profile_set_active (u8 id) -{ - pot_main_t *sm = &pot_main; - pot_profile *profile = NULL; - pot_profile *current_active_prof = NULL; - - current_active_prof = pot_profile_find(sm->active_profile_id); - profile = pot_profile_find(id); - if (profile && profile->valid) { - sm->active_profile_id = id; - current_active_prof->in_use = 0; - profile->in_use = 1; - return(0); - } - return(-1); -} -static inline u8 pot_profile_get_active_id (void) -{ - pot_main_t *sm = &pot_main; - return (sm->active_profile_id); -} - -static inline pot_profile * pot_profile_get_active (void) -{ - pot_main_t *sm = &pot_main; - pot_profile *profile = NULL; - profile = pot_profile_find(sm->active_profile_id); - if (profile && profile->in_use) - return(profile); - return (NULL); -} - -static inline void pot_profile_reset_usage_stats (pot_profile *pow) -{ - if (pow) { - pow->total_pkts_using_this_profile = 0; - } -} - -static inline void pot_profile_incr_usage_stats (pot_profile *pow) -{ - if (pow) { - pow->total_pkts_using_this_profile++; - } -} - - -#endif diff --git a/plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.c b/plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.c deleted file mode 100644 index 1d878ea3..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#include -#include -#include - -#if VPPJNI_DEBUG == 1 - #define DEBUG_LOG(...) clib_warning(__VA_ARGS__) -#else - #define DEBUG_LOG(...) -#endif - -#include - -#include "ioam/jvpp/io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl.h" -#include "jvpp_ioam_trace.h" -#include "ioam/jvpp/jvpp_ioam_trace_gen.h" - -/* - * Class: io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl - * Method: init0 - * Signature: (JI)V - */ -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl_init0 - (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) { - ioamtrace_main_t * plugin_main = &ioamtrace_main; - u8 * name; - clib_warning ("Java_io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl_init0"); - - plugin_main->my_client_index = my_client_index; - plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address; - - name = format (0, "ioam_trace_%08x%c", api_version, 0); - plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (plugin_main->msg_id_base == (u16) ~0) { - jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException"); - (*env)->ThrowNew(env, exClass, "ioam_trace plugin is not loaded in VPP"); - } else { - plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback); - plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback)); - - #define _(N,n) \ - vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_api_reply_handler; - #undef _ - } -} - -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl_close0 -(JNIEnv *env, jclass clazz) { - ioamtrace_main_t * plugin_main = &ioamtrace_main; - - // cleanup: - (*env)->DeleteGlobalRef(env, plugin_main->callbackClass); - (*env)->DeleteGlobalRef(env, plugin_main->callbackObject); - - plugin_main->callbackClass = NULL; - plugin_main->callbackObject = NULL; -} - -/* Attach thread to JVM and cache class references when initiating JVPP iOAM Trace */ -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv* env; - - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return JNI_EVERSION; - } - - if (cache_class_references(env) != 0) { - clib_warning ("Failed to cache class references\n"); - return JNI_ERR; - } - - return JNI_VERSION_1_8; -} - -/* Clean up cached references when disposing JVPP iOAM Trace */ -void JNI_OnUnload(JavaVM *vm, void *reserved) { - JNIEnv* env; - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return; - } - delete_class_references(env); -} diff --git a/plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.h b/plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.h deleted file mode 100644 index 9fc16c15..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/jvpp_ioam_trace.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_jvpp_ioam_trace_h__ -#define __included_jvpp_ioam_trace_h__ - -#include -#include -#include -#include -#include -#include - -/* Global state for JVPP-IOAM-TRACE */ -typedef struct { - /* Base message index for the trace plugin */ - u16 msg_id_base; - - /* Pointer to shared memory queue */ - unix_shared_memory_queue_t * vl_input_queue; - - /* VPP api client index */ - u32 my_client_index; - - /* Callback object and class references enabling asynchronous Java calls */ - jobject callbackObject; - jclass callbackClass; - -} ioamtrace_main_t; - -ioamtrace_main_t ioamtrace_main __attribute__((aligned (64))); - - -#endif /* __included_jvpp_ioam_trace_h__ */ diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace.api b/plugins/ioam-plugin/ioam/lib-trace/trace.api deleted file mode 100644 index cb958325..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace.api +++ /dev/null @@ -1,92 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/** \brief iOAM6 Trace - Set the iOAM6 trace profile - @param trace_type - Type of trace requested - @param num_elts - Number of trace elements to be inserted - @param node_id - Trace Node ID - @param trace_tsp- Timestamp resolution - @param app_data - Application specific opaque -*/ -define trace_profile_add { - u32 client_index; - u32 context; - u8 trace_type; - u8 num_elts; - u8 trace_tsp; - u32 node_id; - u32 app_data; -}; - -/** \brief Trace profile add / del response - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define trace_profile_add_reply { - u32 context; - i32 retval; -}; - - - -/** \brief Delete trace Profile - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define trace_profile_del { - u32 client_index; - u32 context; -}; - -/** \brief Trace profile add / del response - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define trace_profile_del_reply { - u32 context; - i32 retval; -}; - - - -/** \brief Show trace Profile - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define trace_profile_show_config { - u32 client_index; - u32 context; -}; - -/** \brief Show trace config response - @param context - sender context, to match reply w/ request - @param retval - return value for request - @param trace_type - Type of trace requested - @param num_elts - Number of trace elements to be inserted - @param node_id - Trace Node ID - @param trace_tsp- Timestamp resolution - @param app_data - Application specific opaque -*/ -define trace_profile_show_config_reply { - u32 context; - i32 retval; - u8 trace_type; - u8 num_elts; - u8 trace_tsp; - u32 node_id; - u32 app_data; -}; diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h b/plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h deleted file mode 100644 index 223f9545..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace_all_api_h.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_api.c b/plugins/ioam-plugin/ioam/lib-trace/trace_api.c deleted file mode 100644 index 7e0d708e..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace_api.c +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * trace_api.c - iOAM Trace related APIs to create - * and maintain profiles - *------------------------------------------------------------------ - */ - -#include -#include -#include - -#include -#include -#include - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define TRACE_REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -/* *INDENT-OFF* */ -#define TRACE_REPLY_MACRO2(t, body) \ -do { \ - unix_shared_memory_queue_t * q; \ - rv = vl_msg_api_pd_handler (mp, rv); \ - q = vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - do {body;} while (0); \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); -/* *INDENT-ON* */ - -/* List of message types that this plugin understands */ - -#define foreach_trace_plugin_api_msg \ -_(TRACE_PROFILE_ADD, trace_profile_add) \ -_(TRACE_PROFILE_DEL, trace_profile_del) \ -_(TRACE_PROFILE_SHOW_CONFIG, trace_profile_show_config) - -static void vl_api_trace_profile_add_t_handler - (vl_api_trace_profile_add_t * mp) -{ - trace_main_t *sm = &trace_main; - int rv = 0; - vl_api_trace_profile_add_reply_t *rmp; - trace_profile *profile = NULL; - - profile = trace_profile_find (); - if (profile) - { - rv = - trace_profile_create (profile, mp->trace_type, mp->num_elts, - mp->trace_tsp, ntohl (mp->node_id), - ntohl (mp->app_data)); - if (rv != 0) - goto ERROROUT; - } - else - { - rv = -3; - } -ERROROUT: - TRACE_REPLY_MACRO (VL_API_TRACE_PROFILE_ADD_REPLY); -} - - -static void vl_api_trace_profile_del_t_handler - (vl_api_trace_profile_del_t * mp) -{ - trace_main_t *sm = &trace_main; - int rv = 0; - vl_api_trace_profile_del_reply_t *rmp; - - clear_trace_profiles (); - - TRACE_REPLY_MACRO (VL_API_TRACE_PROFILE_DEL_REPLY); -} - -static void vl_api_trace_profile_show_config_t_handler - (vl_api_trace_profile_show_config_t * mp) -{ - trace_main_t *sm = &trace_main; - vl_api_trace_profile_show_config_reply_t *rmp; - int rv = 0; - trace_profile *profile = trace_profile_find (); - if (profile->valid) - { - TRACE_REPLY_MACRO2 (VL_API_TRACE_PROFILE_SHOW_CONFIG_REPLY, - rmp->trace_type = profile->trace_type; - rmp->num_elts = profile->num_elts; - rmp->trace_tsp = profile->trace_tsp; - rmp->node_id = htonl (profile->node_id); - rmp->app_data = htonl (profile->app_data); - ); - } - else - { - TRACE_REPLY_MACRO2 (VL_API_TRACE_PROFILE_SHOW_CONFIG_REPLY, - rmp->trace_type = 0; - rmp->num_elts = 0; rmp->trace_tsp = 0; - rmp->node_id = 0; rmp->app_data = 0; - ); - } -} - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - trace_main_t *sm = &trace_main; - clib_error_t *error = 0; - - sm->vlib_main = vm; - sm->vnet_main = h->vnet_main; - return error; -} - -/* Set up the API message handling tables */ -static clib_error_t * -trace_plugin_api_hookup (vlib_main_t * vm) -{ - trace_main_t *sm = &trace_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_trace_plugin_api_msg; -#undef _ - - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (trace_main_t * sm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); - foreach_vl_msg_name_crc_trace; -#undef _ -} - -static clib_error_t * -trace_init (vlib_main_t * vm) -{ - trace_main_t *sm = &trace_main; - clib_error_t *error = 0; - u8 *name; - - bzero (sm, sizeof (trace_main)); - (void) trace_util_init (); - name = format (0, "ioam_trace_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - sm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - - error = trace_plugin_api_hookup (vm); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (sm, &api_main); - - vec_free (name); - - return error; -} - -VLIB_INIT_FUNCTION (trace_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h b/plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h deleted file mode 100644 index 78c35665..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_trace_msg_enum_h -#define included_trace_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_trace_msg_enum_h */ diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_test.c b/plugins/ioam-plugin/ioam/lib-trace/trace_test.c deleted file mode 100644 index 111dd461..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace_test.c +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * trace_test.c - test harness for trace plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} trace_test_main_t; - -trace_test_main_t trace_test_main; - -#define foreach_standard_reply_retval_handler \ -_(trace_profile_add_reply) \ -_(trace_profile_del_reply) - -#define foreach_custom_reply_handler \ -_(trace_profile_show_config_reply, \ - if(mp->trace_type) \ - { \ - errmsg(" Trace Type : 0x%x (%d)\n",mp->trace_type, mp->trace_type); \ - errmsg(" Trace timestamp precision : %d \n",mp->trace_tsp); \ - errmsg(" Node Id : 0x%x (%d)\n",htonl(mp->node_id), htonl(mp->node_id)); \ - errmsg(" App Data : 0x%x (%d)\n",htonl(mp->app_data), htonl(mp->app_data)); \ - } \ - else errmsg("No valid trace profile configuration found\n");) -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = trace_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -#define _(n,body) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = trace_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - if(retval>=0)do{body;} while(0); \ - else errmsg("Error, retval: %d",retval); \ - } -foreach_custom_reply_handler; -#undef _ -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(TRACE_PROFILE_ADD_REPLY, trace_profile_add_reply) \ -_(TRACE_PROFILE_DEL_REPLY, trace_profile_del_reply) \ -_(TRACE_PROFILE_SHOW_CONFIG_REPLY, trace_profile_show_config_reply) - - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - - -static int -api_trace_profile_add (vat_main_t * vam) -{ - trace_test_main_t *sm = &trace_test_main; - unformat_input_t *input = vam->input; - vl_api_trace_profile_add_t *mp; - u8 trace_type = 0; - u8 num_elts = 0; - int rv = 0; - u32 node_id = 0; - u32 app_data = 0; - u8 trace_tsp = 0; - f64 timeout; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "trace-type 0x%x", &trace_type)) - ; - else if (unformat (input, "trace-elts %d", &num_elts)) - ; - else if (unformat (input, "trace-tsp %d", &trace_tsp)) - ; - else if (unformat (input, "node-id 0x%x", &node_id)) - ; - else if (unformat (input, "app-data 0x%x", &app_data)) - ; - - else - break; - } - - - M (TRACE_PROFILE_ADD, trace_profile_add); - - mp->trace_type = trace_type; - mp->trace_tsp = trace_tsp; - mp->node_id = htonl (node_id); - mp->app_data = htonl (app_data); - mp->num_elts = num_elts; - - S; - W; - - return (rv); -} - - - -static int -api_trace_profile_del (vat_main_t * vam) -{ - trace_test_main_t *sm = &trace_test_main; - vl_api_trace_profile_del_t *mp; - f64 timeout; - - M (TRACE_PROFILE_DEL, trace_profile_del); - S; - W; - return 0; -} - -static int -api_trace_profile_show_config (vat_main_t * vam) -{ - trace_test_main_t *sm = &trace_test_main; - vl_api_trace_profile_show_config_t *mp; - f64 timeout; - M (TRACE_PROFILE_SHOW_CONFIG, trace_profile_show_config); - S; - W; - return 0; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(trace_profile_add, ""\ - "trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts trace-tsp <0|1|2|3> node-id app-data ") \ -_(trace_profile_del, "[id ]") \ -_(trace_profile_show_config, "[id ]") - - -void -vat_api_hookup (vat_main_t * vam) -{ - trace_test_main_t *sm = &trace_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * -vat_plugin_register (vat_main_t * vam) -{ - trace_test_main_t *sm = &trace_test_main; - u8 *name; - - sm->vat_main = vam; - - name = format (0, "ioam_trace_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~ 0) - vat_api_hookup (vam); - - vec_free (name); - - return 0; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_util.c b/plugins/ioam-plugin/ioam/lib-trace/trace_util.c deleted file mode 100644 index 5c7f1eef..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace_util.c +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include "trace_util.h" - -trace_main_t trace_main; - -static int -trace_profile_cleanup (trace_profile * profile) -{ - - memset (profile, 0, sizeof (trace_profile)); - profile->trace_tsp = TSP_MICROSECONDS; /* Micro seconds */ - ip6_trace_profile_cleanup (); /* lib-trace_TODO: Remove this once IOAM-IPv6 transport is a plugin */ - return 0; - -} - -static int -trace_main_profiles_reset (void) -{ - int rv; - - trace_main_t *sm = &trace_main; - rv = trace_profile_cleanup (&(sm->profile)); - return (rv); -} - -int -trace_util_init (void) -{ - int rv; - - rv = trace_main_profiles_reset (); - return (rv); -} - - -int -trace_profile_create (trace_profile * profile, u8 trace_type, u8 num_elts, - u32 trace_tsp, u32 node_id, u32 app_data) -{ - - if (!trace_type || !num_elts || !(node_id)) - { - return (-1); - } - if (profile && !profile->valid) - { - //rv = trace_profile_cleanup (profile); - profile->trace_type = trace_type; - profile->num_elts = num_elts; - profile->trace_tsp = trace_tsp; - profile->node_id = node_id; - profile->app_data = app_data; - profile->valid = 1; - - /* lib-trace_TODO: Remove this once IOAM-IPv6 transport is a plugin */ - ip6_trace_profile_setup (); - return (0); - } - - return (-1); -} - - - -clib_error_t * -clear_trace_profile_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - - trace_main_profiles_reset (); - return 0; -} - -void -clear_trace_profiles (void) -{ - clear_trace_profile_command_fn (0, 0, 0); -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND(clear_trace_profile_command) = -{ -.path = "clear ioam-trace profile", -.short_help = "clear ioam-trace profile [|all]", -.function = clear_trace_profile_command_fn, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_trace_profile_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - u8 trace_type = 0; - u8 num_elts = 0; - u32 node_id = 0; - u32 app_data = 0; - u32 trace_tsp = 0; - trace_profile *profile = NULL; - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "trace-type 0x%x", &trace_type)); - else if (unformat (input, "trace-elts %d", &num_elts)); - else if (unformat (input, "trace-tsp %d", &trace_tsp)); - else if (unformat (input, "node-id 0x%x", &node_id)); - else if (unformat (input, "app-data 0x%x", &app_data)); - else - break; - } - profile = trace_profile_find (); - if (profile) - { - trace_profile_create (profile, trace_type, num_elts, trace_tsp, - node_id, app_data); - } - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (set_trace_profile_command, static) = -{ -.path = "set ioam-trace profile", -.short_help = "set ioam-trace \ - trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts trace-tsp <0|1|2|3> \ - node-id app-data ", -.function = set_trace_profile_command_fn, -}; -/* *INDENT-ON* */ - -static clib_error_t * -show_trace_profile_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - trace_profile *p = NULL; - u8 *s = 0; - p = trace_profile_find (); - if (!(p && p->valid)) - { - s = format (s, "\nTrace configuration not valid\n"); - vlib_cli_output (vm, "%v", s); - vec_free (s); - return 0; - } - s = format (s, " HOP BY HOP OPTIONS - TRACE CONFIG - \n"); - s = format (s, " Trace Type : 0x%x (%d)\n", - p->trace_type, p->trace_type); - s = - format (s, " Trace timestamp precision : %d (%s)\n", - p->trace_tsp, - (p->trace_tsp == - TSP_SECONDS) ? "Seconds" : ((p->trace_tsp == - TSP_MILLISECONDS) ? - "Milliseconds" - : (((p->trace_tsp == - TSP_MICROSECONDS) ? - "Microseconds" : - "Nanoseconds")))); - s = format (s, " Num of trace nodes : %d\n", p->num_elts); - s = - format (s, " Node-id : 0x%x (%d)\n", - p->node_id, p->node_id); - s = - format (s, " App Data : 0x%x (%d)\n", - p->app_data, p->app_data); - vlib_cli_output (vm, "%v", s); - vec_free (s); - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (show_trace_profile_command, static) = -{ -.path = "show ioam-trace profile", -.short_help = "show ioam-trace profile", -.function = show_trace_profile_command_fn, -}; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-trace/trace_util.h b/plugins/ioam-plugin/ioam/lib-trace/trace_util.h deleted file mode 100644 index 556f07ee..00000000 --- a/plugins/ioam-plugin/ioam/lib-trace/trace_util.h +++ /dev/null @@ -1,247 +0,0 @@ -/* - * trace_util.h -- Trace Profile Utility header - * - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef include_vnet_trace_util_h -#define include_vnet_trace_util_h - -#define debug_ioam debug_ioam_fn - - -/** - * Usage: - * - * On any node that participates in iOAM Trace. - * - * Step 1: Initialize this library by calling trace_init() - * Step 2: Setup a trace profile that contains all the parameters needed to compute cumulative: - * Call these functions: - * trace_profile_find - * trace_profile_create - * Step 2a: On initial node enable the profile to be used: - * trace_profile_set_active / trace_profile_get_active will return the profile - * Step 4: TBD - * trace_validate - * - */ - -typedef struct trace_profile_ -{ - u8 valid:1; - u8 trace_type; - u8 num_elts; - /* Configured node-id */ - u32 node_id; - u32 app_data; - u32 trace_tsp; -} trace_profile; - -typedef struct -{ - /* Name of the default profile list in use */ - trace_profile profile; - - /* API message ID base */ - u16 msg_id_base; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; -} trace_main_t; - -extern trace_main_t trace_main; - -/* - * Initialize Trace profile - */ -int trace_util_init (void); - - -/* - * Find a trace profile - */ - -always_inline trace_profile * -trace_profile_find (void) -{ - trace_main_t *sm = &trace_main; - - return (&(sm->profile)); -} - - -/* setup and clean up profile */ -int trace_profile_create (trace_profile * profile, u8 trace_type, u8 num_elts, - u32 trace_tsp, u32 node_id, u32 app_data); - -void clear_trace_profiles (void); - - - -#define BIT_TTL_NODEID (1<<0) -#define BIT_ING_INTERFACE (1<<1) -#define BIT_EGR_INTERFACE (1<<2) -#define BIT_TIMESTAMP (1<<3) -#define BIT_APPDATA (1<<4) -#define TRACE_TYPE_MASK 0x1F /* Mask of all above bits */ - -/* - 0x00011111 iOAM-trace-type is 0x00011111 then the format of node - data is: - - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Hop_Lim | node_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | ingress_if_id | egress_if_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - + timestamp + - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | app_data | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -*/ -#define TRACE_TYPE_IF_TS_APP 0x1f -typedef struct -{ - u32 ttl_node_id; - u16 ingress_if; - u16 egress_if; - u32 timestamp; - u32 app_data; -} ioam_trace_if_ts_app_t; - -/* - 0x00000111 iOAM-trace-type is 0x00000111 then the format is: - - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Hop_Lim | node_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | ingress_if_id | egress_if_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -*/ - -#define TRACE_TYPE_IF 0x03 -typedef struct -{ - u32 ttl_node_id; - u16 ingress_if; - u16 egress_if; -} ioam_trace_if_t; - -/* - 0x00001001 iOAM-trace-type is 0x00001001 then the format is: - - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Hop_Lim | node_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - + timestamp + - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -*/ - -#define TRACE_TYPE_TS 0x09 -typedef struct -{ - u32 ttl_node_id; - u32 timestamp; -} ioam_trace_ts_t; - -/* - 0x00010001 iOAM-trace-type is 0x00010001 then the format is: - - - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Hop_Lim | node_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | app_data | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -*/ - - -#define TRACE_TYPE_APP 0x11 -typedef struct -{ - u32 ttl_node_id; - u32 app_data; -} ioam_trace_app_t; - -/* - - 0x00011001 iOAM-trace-type is 0x00011001 then the format is: - - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Hop_Lim | node_id | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - + timestamp + - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | app_data | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -*/ - -#define TRACE_TYPE_TS_APP 0x19 -typedef struct -{ - u32 ttl_node_id; - u32 timestamp; - u32 app_data; -} ioam_trace_ts_app_t; - - - -static inline u8 -fetch_trace_data_size (u8 trace_type) -{ - u8 trace_data_size = 0; - - if (trace_type == TRACE_TYPE_IF_TS_APP) - trace_data_size = sizeof (ioam_trace_if_ts_app_t); - else if (trace_type == TRACE_TYPE_IF) - trace_data_size = sizeof (ioam_trace_if_t); - else if (trace_type == TRACE_TYPE_TS) - trace_data_size = sizeof (ioam_trace_ts_t); - else if (trace_type == TRACE_TYPE_APP) - trace_data_size = sizeof (ioam_trace_app_t); - else if (trace_type == TRACE_TYPE_TS_APP) - trace_data_size = sizeof (ioam_trace_ts_app_t); - - return trace_data_size; -} - -int ioam_trace_get_sizeof_handler (u32 * result); -int ip6_trace_profile_setup (void); -int ip6_trace_profile_cleanup (void); - -#define TSP_SECONDS 0 -#define TSP_MILLISECONDS 1 -#define TSP_MICROSECONDS 2 -#define TSP_NANOSECONDS 3 - -#endif - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_decap.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_decap.c deleted file mode 100644 index fd308657..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_decap.c +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Statistics (not really errors) */ -#define foreach_vxlan_gpe_decap_ioam_v4_error \ -_(DECAPSULATED, "good packets decapsulated") - -static char *vxlan_gpe_decap_ioam_v4_error_strings[] = { -#define _(sym,string) string, - foreach_vxlan_gpe_decap_ioam_v4_error -#undef _ -}; - -typedef enum -{ -#define _(sym,str) VXLAN_GPE_DECAP_IOAM_V4_ERROR_##sym, - foreach_vxlan_gpe_decap_ioam_v4_error -#undef _ - VXLAN_GPE_DECAP_IOAM_V4_N_ERROR, -} vxlan_gpe_decap_ioam_v4_error_t; - - -always_inline void -vxlan_gpe_decap_ioam_v4_two_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vxlan_gpe_main_t * ngm, - vlib_buffer_t * b0, vlib_buffer_t * b1, - u32 * next0, u32 * next1) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - next0[0] = next1[0] = hm->decap_v4_next_override; - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, &next0[0], - VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, - 0 /* use_adj */ ); - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b1, &next0[1], - VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, - 0 /* use_adj */ ); -} - - - -static uword -vxlan_gpe_decap_ioam (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame, u8 is_ipv6) -{ - u32 n_left_from, next_index, *from, *to_next; - vxlan_gpe_main_t *ngm = &vxlan_gpe_main; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 bi0, bi1; - vlib_buffer_t *b0, *b1; - u32 next0, next1; - - next0 = next1 = hm->decap_v4_next_override; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - } - - bi0 = from[0]; - bi1 = from[1]; - to_next[0] = bi0; - to_next[1] = bi1; - from += 2; - to_next += 2; - n_left_to_next -= 2; - n_left_from -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - - vlib_buffer_advance (b0, - -(word) (sizeof (udp_header_t) + - sizeof (ip4_header_t) + - sizeof (vxlan_gpe_header_t))); - vlib_buffer_advance (b1, - -(word) (sizeof (udp_header_t) + - sizeof (ip4_header_t) + - sizeof (vxlan_gpe_header_t))); - - vxlan_gpe_decap_ioam_v4_two_inline (vm, node, ngm, b0, b1, - &next0, &next1); - - - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, - n_left_to_next, bi0, bi1, next0, - next1); - - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - vxlan_gpe_ioam_v4_trace_t *tr = vlib_add_trace (vm, node, b0, - sizeof (*tr)); - } - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0 = hm->decap_v4_next_override; - - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - - vlib_buffer_advance (b0, - -(word) (sizeof (udp_header_t) + - sizeof (ip4_header_t) + - sizeof (vxlan_gpe_header_t))); - - next0 = hm->decap_v4_next_override; - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, - &next0, - VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, - 0 /* use_adj */ ); - - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - vxlan_gpe_ioam_v4_trace_t *tr = vlib_add_trace (vm, node, b0, - sizeof (*tr)); - } - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - return from_frame->n_vectors; -} - - -static uword -vxlan_gpe_decap_ioam_v4 (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) -{ - return vxlan_gpe_decap_ioam (vm, node, from_frame, 0); -} - - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (vxlan_gpe_decap_ioam_v4_node) = { - .function = vxlan_gpe_decap_ioam_v4, - .name = "vxlan-gpe-decap-ioam-v4", - .vector_size = sizeof (u32), - .format_trace = format_vxlan_gpe_ioam_v4_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(vxlan_gpe_decap_ioam_v4_error_strings), - .error_strings = vxlan_gpe_decap_ioam_v4_error_strings, - - .n_next_nodes = VXLAN_GPE_DECAP_IOAM_V4_N_NEXT, - - .next_nodes = { - [VXLAN_GPE_DECAP_IOAM_V4_NEXT_POP] = "vxlan-gpe-pop-ioam-v4", - [VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP] = "error-drop", - }, -}; -/* *INDENT-ON* */ - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_encap.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_encap.c deleted file mode 100644 index 4b18bfea..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_encap.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Statistics (not really errors) */ -#define foreach_vxlan_gpe_encap_ioam_v4_error \ -_(ENCAPSULATED, "good packets encapsulated") - -static char *vxlan_gpe_encap_ioam_v4_error_strings[] = { -#define _(sym,string) string, - foreach_vxlan_gpe_encap_ioam_v4_error -#undef _ -}; - -typedef enum -{ -#define _(sym,str) VXLAN_GPE_ENCAP_IOAM_V4_ERROR_##sym, - foreach_vxlan_gpe_encap_ioam_v4_error -#undef _ - VXLAN_GPE_ENCAP_IOAM_V4_N_ERROR, -} vxlan_gpe_encap_ioam_v4_error_t; - -typedef enum -{ - VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP, - VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, - VXLAN_GPE_ENCAP_IOAM_V4_N_NEXT -} vxlan_gpe_encap_ioam_v4_next_t; - - -always_inline void -vxlan_gpe_encap_ioam_v4_two_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vxlan_gpe_main_t * ngm, - vlib_buffer_t * b0, vlib_buffer_t * b1, - u32 * next0, u32 * next1) -{ - next0[0] = next1[0] = VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP; - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, next0, - VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, - 0 /* use_adj */ ); - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b1, next1, - VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, - 0 /* use_adj */ ); -} - - -static uword -vxlan_gpe_encap_ioam_v4 (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) -{ - u32 n_left_from, next_index, *from, *to_next; - vxlan_gpe_main_t *ngm = &vxlan_gpe_main; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 bi0, bi1; - vlib_buffer_t *b0, *b1; - u32 next0, next1; - - next0 = next1 = VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - } - - bi0 = from[0]; - bi1 = from[1]; - to_next[0] = bi0; - to_next[1] = bi1; - from += 2; - to_next += 2; - n_left_to_next -= 2; - n_left_from -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - vxlan_gpe_encap_ioam_v4_two_inline (vm, node, ngm, b0, b1, - &next0, &next1); - - - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, - n_left_to_next, bi0, bi1, next0, - next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0 = VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP; - - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, - &next0, - VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, - 0 /* use_adj */ ); - - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - vxlan_gpe_ioam_v4_trace_t *tr = vlib_add_trace (vm, node, b0, - sizeof (*tr)); - } - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - return from_frame->n_vectors; -} - - - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (vxlan_gpe_encap_ioam_v4_node) = { - .function = vxlan_gpe_encap_ioam_v4, - .name = "vxlan-gpe-encap-ioam-v4", - .vector_size = sizeof (u32), - .format_trace = format_vxlan_gpe_ioam_v4_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(vxlan_gpe_encap_ioam_v4_error_strings), - .error_strings = vxlan_gpe_encap_ioam_v4_error_strings, - - .n_next_nodes = VXLAN_GPE_ENCAP_IOAM_V4_N_NEXT, - - .next_nodes = { - [VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP] = "ip4-lookup", - [VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP] = "error-drop", - }, -}; -/* *INDENT-ON* */ - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_pop.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_pop.c deleted file mode 100644 index 55c33b14..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_pop.c +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include - -/* Statistics (not really errors) */ -#define foreach_vxlan_gpe_pop_ioam_v4_error \ -_(POPPED, "good packets popped") - -static char *vxlan_gpe_pop_ioam_v4_error_strings[] = { -#define _(sym,string) string, - foreach_vxlan_gpe_pop_ioam_v4_error -#undef _ -}; - -typedef enum -{ -#define _(sym,str) VXLAN_GPE_POP_IOAM_V4_ERROR_##sym, - foreach_vxlan_gpe_pop_ioam_v4_error -#undef _ - VXLAN_GPE_POP_IOAM_V4_N_ERROR, -} vxlan_gpe_pop_ioam_v4_error_t; - -typedef struct -{ - ioam_trace_t fmt_trace; -} vxlan_gpe_pop_ioam_v4_trace_t; - - -u8 * -format_vxlan_gpe_pop_ioam_v4_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - vxlan_gpe_pop_ioam_v4_trace_t *t1 - = va_arg (*args, vxlan_gpe_pop_ioam_v4_trace_t *); - ioam_trace_t *t = &(t1->fmt_trace); - vxlan_gpe_ioam_option_t *fmt_trace0; - vxlan_gpe_ioam_option_t *opt0, *limit0; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - u8 type0; - - fmt_trace0 = (vxlan_gpe_ioam_option_t *) t->option_data; - - s = format (s, "VXLAN_GPE_IOAM_POP: next_index %d len %d traced %d", - t->next_index, fmt_trace0->length, t->trace_len); - - opt0 = (vxlan_gpe_ioam_option_t *) (fmt_trace0 + 1); - limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) fmt_trace0) + t->trace_len; - - while (opt0 < limit0) - { - type0 = opt0->type; - switch (type0) - { - case 0: /* Pad, just stop */ - opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; - break; - - default: - if (hm->trace[type0]) - { - s = (*hm->trace[type0]) (s, opt0); - } - else - { - s = - format (s, "\n unrecognized option %d length %d", type0, - opt0->length); - } - opt0 = - (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + - sizeof (vxlan_gpe_ioam_option_t)); - break; - } - } - - return s; -} - -always_inline void -vxlan_gpe_ioam_pop_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_buffer_t * b0) -{ - ip4_header_t *ip0; - udp_header_t *udp_hdr0; - vxlan_gpe_header_t *gpe_hdr0; - vxlan_gpe_ioam_hdr_t *gpe_ioam0; - - ip0 = vlib_buffer_get_current (b0); - - udp_hdr0 = (udp_header_t *) (ip0 + 1); - gpe_hdr0 = (vxlan_gpe_header_t *) (udp_hdr0 + 1); - gpe_ioam0 = (vxlan_gpe_ioam_hdr_t *) (gpe_hdr0 + 1); - - /* Pop the iOAM data */ - vlib_buffer_advance (b0, - (word) (sizeof (udp_header_t) + - sizeof (ip4_header_t) + - sizeof (vxlan_gpe_header_t) + - gpe_ioam0->length)); - - return; -} - - - -always_inline void -vxlan_gpe_pop_ioam_v4_one_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vxlan_gpe_main_t * ngm, - vlib_buffer_t * b0, u32 * next0) -{ - CLIB_UNUSED (ip4_header_t * ip0); - CLIB_UNUSED (udp_header_t * udp_hdr0); - CLIB_UNUSED (vxlan_gpe_header_t * gpe_hdr0); - CLIB_UNUSED (vxlan_gpe_ioam_hdr_t * gpe_ioam0); - CLIB_UNUSED (vxlan_gpe_ioam_option_t * opt0); - CLIB_UNUSED (vxlan_gpe_ioam_option_t * limit0); - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - - /* Pop the iOAM header */ - ip0 = vlib_buffer_get_current (b0); - udp_hdr0 = (udp_header_t *) (ip0 + 1); - gpe_hdr0 = (vxlan_gpe_header_t *) (udp_hdr0 + 1); - gpe_ioam0 = (vxlan_gpe_ioam_hdr_t *) (gpe_hdr0 + 1); - opt0 = (vxlan_gpe_ioam_option_t *) (gpe_ioam0 + 1); - limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) gpe_ioam0 + gpe_ioam0->length); - - /* - * Basic validity checks - */ - if (gpe_ioam0->length > clib_net_to_host_u16 (ip0->length)) - { - next0[0] = VXLAN_GPE_INPUT_NEXT_DROP; - goto trace00; - } - - /* Scan the set of h-b-h options, process ones that we understand */ - while (opt0 < limit0) - { - u8 type0; - type0 = opt0->type; - switch (type0) - { - case 0: /* Pad1 */ - opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; - continue; - case 1: /* PadN */ - break; - default: - if (hm->pop_options[type0]) - { - if ((*hm->pop_options[type0]) (ip0, opt0) < 0) - { - next0[0] = VXLAN_GPE_INPUT_NEXT_DROP; - goto trace00; - } - } - break; - } - opt0 = - (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + - sizeof (vxlan_gpe_ioam_hdr_t)); - } - - - next0[0] = - (gpe_ioam0->protocol < VXLAN_GPE_PROTOCOL_MAX) ? - ngm-> - decap_next_node_list[gpe_ioam0->protocol] : VXLAN_GPE_INPUT_NEXT_DROP; - -trace00: - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - vxlan_gpe_pop_ioam_v4_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - u32 trace_len = gpe_ioam0->length; - t->fmt_trace.next_index = next0[0]; - /* Capture the h-b-h option verbatim */ - trace_len = - trace_len < - ARRAY_LEN (t->fmt_trace. - option_data) ? trace_len : ARRAY_LEN (t->fmt_trace. - option_data); - t->fmt_trace.trace_len = trace_len; - clib_memcpy (&(t->fmt_trace.option_data), gpe_ioam0, trace_len); - } - - /* Remove the iOAM header inside the VxLAN-GPE header */ - vxlan_gpe_ioam_pop_v4 (vm, node, b0); - return; -} - -always_inline void -vxlan_gpe_pop_ioam_v4_two_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vxlan_gpe_main_t * ngm, - vlib_buffer_t * b0, vlib_buffer_t * b1, - u32 * next0, u32 * next1) -{ - - vxlan_gpe_pop_ioam_v4_one_inline (vm, node, ngm, b0, next0); - vxlan_gpe_pop_ioam_v4_one_inline (vm, node, ngm, b1, next1); -} - - - -static uword -vxlan_gpe_pop_ioam (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame, u8 is_ipv6) -{ - u32 n_left_from, next_index, *from, *to_next; - vxlan_gpe_main_t *ngm = &vxlan_gpe_main; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 bi0, bi1; - vlib_buffer_t *b0, *b1; - u32 next0, next1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - } - - bi0 = from[0]; - bi1 = from[1]; - to_next[0] = bi0; - to_next[1] = bi1; - from += 2; - to_next += 2; - n_left_to_next -= 2; - n_left_from -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - vxlan_gpe_pop_ioam_v4_two_inline (vm, node, ngm, b0, b1, &next0, - &next1); - - - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, - n_left_to_next, bi0, bi1, next0, - next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0; - - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - vxlan_gpe_pop_ioam_v4_one_inline (vm, node, ngm, b0, &next0); - - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - return from_frame->n_vectors; -} - - -static uword -vxlan_gpe_pop_ioam_v4 (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * from_frame) -{ - return vxlan_gpe_pop_ioam (vm, node, from_frame, 0); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (vxlan_gpe_pop_ioam_v4_node) = { - .function = vxlan_gpe_pop_ioam_v4, - .name = "vxlan-gpe-pop-ioam-v4", - .vector_size = sizeof (u32), - .format_trace = format_vxlan_gpe_pop_ioam_v4_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(vxlan_gpe_pop_ioam_v4_error_strings), - .error_strings = vxlan_gpe_pop_ioam_v4_error_strings, - - .n_next_nodes = VXLAN_GPE_INPUT_N_NEXT, - - .next_nodes = { -#define _(s,n) [VXLAN_GPE_INPUT_NEXT_##s] = n, - foreach_vxlan_gpe_input_next -#undef _ - }, -}; -/* *INDENT-ON* */ - - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_transit.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_transit.c deleted file mode 100644 index b42c357c..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/ioam_transit.c +++ /dev/null @@ -1,188 +0,0 @@ - /* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Statistics (not really errors) */ -#define foreach_vxlan_gpe_transit_ioam_error \ -_(ENCAPSULATED, "good packets encapsulated") - -static char *vxlan_gpe_transit_ioam_error_strings[] = { -#define _(sym,string) string, - foreach_vxlan_gpe_transit_ioam_error -#undef _ -}; - -typedef enum -{ -#define _(sym,str) VXLAN_GPE_TRANSIT_IOAM_ERROR_##sym, - foreach_vxlan_gpe_transit_ioam_error -#undef _ - VXLAN_GPE_TRANSIT_IOAM_N_ERROR, -} vxlan_gpe_transit_ioam_error_t; - -typedef enum -{ - VXLAN_GPE_TRANSIT_IOAM_NEXT_OUTPUT, - VXLAN_GPE_TRANSIT_IOAM_NEXT_DROP, - VXLAN_GPE_TRANSIT_IOAM_N_NEXT -} vxlan_gpe_transit_ioam_next_t; - - -/* *INDENT-OFF* */ -VNET_FEATURE_INIT (vxlan_gpe_transit_ioam, static) = -{ - .arc_name = "ip4-output", - .node_name = "vxlan-gpe-transit-ioam", - .runs_before = VNET_FEATURES ("interface-output"), -}; -/* *INDENT-ON* */ - -static uword -vxlan_gpe_transit_ioam (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * from_frame) -{ - u32 n_left_from, next_index, *from, *to_next; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 next0 = VXLAN_GPE_TRANSIT_IOAM_NEXT_OUTPUT; - - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - ip4_header_t *ip0; - u32 iph_offset = 0; - - b0 = vlib_get_buffer (vm, bi0); - iph_offset = vnet_buffer (b0)->ip.save_rewrite_length; - ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) - + iph_offset); - - /* just forward non ipv4 packets */ - if (PREDICT_FALSE - ((ip0->ip_version_and_header_length & 0xF0) == 0x40)) - { - /* ipv4 packets */ - udp_header_t *udp_hdr0 = (udp_header_t *) (ip0 + 1); - if (PREDICT_FALSE - ((ip0->protocol == IP_PROTOCOL_UDP) && - (clib_net_to_host_u16 (udp_hdr0->dst_port) == - UDP_DST_PORT_vxlan_gpe))) - { - - /* Check the iOAM header */ - vxlan_gpe_header_t *gpe_hdr0 = - (vxlan_gpe_header_t *) (udp_hdr0 + 1); - - if (PREDICT_FALSE - (gpe_hdr0->protocol == VXLAN_GPE_PROTOCOL_IOAM)) - { - uword *t = NULL; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - fib_prefix_t key4; - memset (&key4, 0, sizeof (key4)); - key4.fp_proto = FIB_PROTOCOL_IP4; - key4.fp_addr.ip4.as_u32 = ip0->dst_address.as_u32; - t = hash_get_mem (hm->dst_by_ip4, &key4); - if (t) - { - - - vlib_buffer_advance (b0, - (word) (sizeof - (ethernet_header_t))); - vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, - b0, - &next0, - VXLAN_GPE_TRANSIT_IOAM_NEXT_DROP, - 1 - /* use_adj */ - ); - vlib_buffer_advance (b0, - -(word) (sizeof - (ethernet_header_t))); - } - } - } - } - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - return from_frame->n_vectors; -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (vxlan_gpe_transit_ioam_node) = { - .function = vxlan_gpe_transit_ioam, - .name = "vxlan-gpe-transit-ioam", - .vector_size = sizeof (u32), - .format_trace = format_vxlan_gpe_ioam_v4_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(vxlan_gpe_transit_ioam_error_strings), - .error_strings = vxlan_gpe_transit_ioam_error_strings, - - .n_next_nodes = VXLAN_GPE_TRANSIT_IOAM_N_NEXT, - - .next_nodes = { - [VXLAN_GPE_TRANSIT_IOAM_NEXT_OUTPUT] = "interface-output", - [VXLAN_GPE_TRANSIT_IOAM_NEXT_DROP] = "error-drop", - }, - -}; -/* *INDENT-ON* */ - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe.api b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe.api deleted file mode 100644 index 056529a4..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe.api +++ /dev/null @@ -1,181 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/** \brief iOAM Over VxLAN-GPE - Set iOAM transport for VxLAN-GPE - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param id - profile id - @param trace_ppc - Trace PPC (none/encap/decap) - @param pow_enable - Proof of Work enabled or not flag - @param trace_enable - iOAM Trace enabled or not flag - -*/ -define vxlan_gpe_ioam_enable { - u32 client_index; - u32 context; - u16 id; - u8 trace_ppc; - u8 pow_enable; - u8 trace_enable; -}; - -/** \brief iOAM Over VxLAN-GPE - Set iOAM transport for VXLAN-GPE reply - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define vxlan_gpe_ioam_enable_reply { - u32 context; - i32 retval; -}; - - -/** \brief iOAM for VxLAN-GPE disable - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param id - profile id -*/ -define vxlan_gpe_ioam_disable -{ - u32 client_index; - u32 context; - u16 id; -}; - -/** \brief vxlan_gpe_ioam disable response - @param context - sender context, to match reply w/ request - @param retval - return value for request -*/ -define vxlan_gpe_ioam_disable_reply -{ - u32 context; - i32 retval; -}; - -/** \brief Enable iOAM for a VNI (VXLAN-GPE) - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param vni - VXLAN-GPE VNI - @param local - IPv4/6 Address of the local VTEP - @param remote - IPv4/6 Address of the remote VTEP - -*/ -define vxlan_gpe_ioam_vni_enable { - u32 client_index; - u32 context; - u32 vni; - u8 local[16]; - u8 remote[16]; - u8 is_ipv6; -}; - -/** \brief Reply to enable iOAM for a VNI (VXLAN-GPE) - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return value for request - -*/ -define vxlan_gpe_ioam_vni_enable_reply { - u32 client_index; - u32 context; - i32 retval; -}; - -/** \brief Disable iOAM for a VNI (VXLAN-GPE) - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param vni - VXLAN-GPE VNI - @param local - IPv4/6 Address of the local VTEP - @param remote - IPv4/6 Address of the remote VTEP - -*/ -define vxlan_gpe_ioam_vni_disable { - u32 client_index; - u32 context; - u32 vni; - u8 local[16]; - u8 remote[16]; - u8 is_ipv6; -}; - -/** \brief Reply to disable iOAM for a VNI (VXLAN-GPE) - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return value for request - -*/ -define vxlan_gpe_ioam_vni_disable_reply { - u32 client_index; - u32 context; - i32 retval; -}; - - -/** \brief Enable iOAM for a VXLAN-GPE transit - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param dst_addr - IPv4/6 Address of the local VTEP - @param outer_fib_index- FIB index - -*/ -define vxlan_gpe_ioam_transit_enable { - u32 client_index; - u32 context; - u32 outer_fib_index; - u8 dst_addr[16]; - u8 is_ipv6; -}; - -/** \brief Reply to enable iOAM for VXLAN-GPE transit - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return value for request - -*/ -define vxlan_gpe_ioam_transit_enable_reply { - u32 client_index; - u32 context; - i32 retval; -}; - -/** \brief Disable iOAM for VXLAN-GPE transit - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param dst_addr - IPv4/6 Address of the local VTEP - @param outer_fib_index- FIB index - -*/ -define vxlan_gpe_ioam_transit_disable { - u32 client_index; - u32 context; - u32 outer_fib_index; - u8 dst_addr[16]; - u8 is_ipv6; -}; - -/** \brief Reply to disable iOAM for VXLAN-GPE transit - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return value for request - -*/ -define vxlan_gpe_ioam_transit_disable_reply { - u32 client_index; - u32 context; - i32 retval; -}; - - diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h deleted file mode 100644 index bbf2c101..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_api.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_api.c deleted file mode 100644 index 68752365..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_api.c +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * vxlan_gpe_api.c - iOAM VxLAN-GPE related APIs to create - * and maintain profiles - *------------------------------------------------------------------ - */ - -#include -#include -#include - -#include -#include -#include - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define VXLAN_GPE_REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -/* *INDENT-OFF* */ -#define VXLAN_GPE_REPLY_MACRO2(t, body) \ -do { \ - unix_shared_memory_queue_t * q; \ - rv = vl_msg_api_pd_handler (mp, rv); \ - q = vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - do {body;} while (0); \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); -/* *INDENT-ON* */ - -/* List of message types that this plugin understands */ - -#define foreach_vxlan_gpe_plugin_api_msg \ -_(VXLAN_GPE_IOAM_ENABLE, vxlan_gpe_ioam_enable) \ -_(VXLAN_GPE_IOAM_DISABLE, vxlan_gpe_ioam_disable) \ -_(VXLAN_GPE_IOAM_VNI_ENABLE, vxlan_gpe_ioam_vni_enable) \ -_(VXLAN_GPE_IOAM_VNI_DISABLE, vxlan_gpe_ioam_vni_disable) \ -_(VXLAN_GPE_IOAM_TRANSIT_ENABLE, vxlan_gpe_ioam_transit_enable) \ -_(VXLAN_GPE_IOAM_TRANSIT_DISABLE, vxlan_gpe_ioam_transit_disable) \ - - -static void vl_api_vxlan_gpe_ioam_enable_t_handler - (vl_api_vxlan_gpe_ioam_enable_t * mp) -{ - int rv = 0; - vl_api_vxlan_gpe_ioam_enable_reply_t *rmp; - clib_error_t *error; - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - - /* Ignoring the profile id as currently a single profile - * is supported */ - error = - vxlan_gpe_ioam_enable (mp->trace_enable, mp->pow_enable, mp->trace_ppc); - if (error) - { - clib_error_report (error); - rv = clib_error_get_code (error); - } - - VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_ENABLE_REPLY); -} - -static void vl_api_vxlan_gpe_ioam_disable_t_handler - (vl_api_vxlan_gpe_ioam_disable_t * mp) -{ - int rv = 0; - vl_api_vxlan_gpe_ioam_disable_reply_t *rmp; - clib_error_t *error; - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - - /* Ignoring the profile id as currently a single profile - * is supported */ - error = vxlan_gpe_ioam_disable (0, 0, 0); - if (error) - { - clib_error_report (error); - rv = clib_error_get_code (error); - } - - VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_DISABLE_REPLY); -} - -static void vl_api_vxlan_gpe_ioam_vni_enable_t_handler - (vl_api_vxlan_gpe_ioam_vni_enable_t * mp) -{ - int rv = 0; - vl_api_vxlan_gpe_ioam_vni_enable_reply_t *rmp; - clib_error_t *error; - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - vxlan4_gpe_tunnel_key_t key4; - uword *p = NULL; - vxlan_gpe_main_t *gm = &vxlan_gpe_main; - vxlan_gpe_tunnel_t *t = 0; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - u32 vni; - - - if (!mp->is_ipv6) - { - clib_memcpy (&key4.local, &mp->local, sizeof (key4.local)); - clib_memcpy (&key4.remote, &mp->remote, sizeof (key4.remote)); - vni = clib_net_to_host_u32 (mp->vni); - key4.vni = clib_host_to_net_u32 (vni << 8); - key4.pad = 0; - - p = hash_get_mem (gm->vxlan4_gpe_tunnel_by_key, &key4); - } - else - { - return; - } - - if (!p) - return; - - t = pool_elt_at_index (gm->tunnels, p[0]); - - error = vxlan_gpe_ioam_set (t, hm->has_trace_option, - hm->has_pot_option, - hm->has_ppc_option, mp->is_ipv6); - - - if (error) - { - clib_error_report (error); - rv = clib_error_get_code (error); - } - - VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_VNI_ENABLE_REPLY); -} - - -static void vl_api_vxlan_gpe_ioam_vni_disable_t_handler - (vl_api_vxlan_gpe_ioam_vni_disable_t * mp) -{ - int rv = 0; - vl_api_vxlan_gpe_ioam_vni_enable_reply_t *rmp; - clib_error_t *error; - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - vxlan4_gpe_tunnel_key_t key4; - uword *p = NULL; - vxlan_gpe_main_t *gm = &vxlan_gpe_main; - vxlan_gpe_tunnel_t *t = 0; - u32 vni; - - - if (!mp->is_ipv6) - { - clib_memcpy (&key4.local, &mp->local, sizeof (key4.local)); - clib_memcpy (&key4.remote, &mp->remote, sizeof (key4.remote)); - vni = clib_net_to_host_u32 (mp->vni); - key4.vni = clib_host_to_net_u32 (vni << 8); - key4.pad = 0; - - p = hash_get_mem (gm->vxlan4_gpe_tunnel_by_key, &key4); - } - else - { - return; - } - - if (!p) - return; - - t = pool_elt_at_index (gm->tunnels, p[0]); - - error = vxlan_gpe_ioam_clear (t, 0, 0, 0, 0); - - - if (error) - { - clib_error_report (error); - rv = clib_error_get_code (error); - } - - - VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_VNI_DISABLE_REPLY); -} - -static void vl_api_vxlan_gpe_ioam_transit_enable_t_handler - (vl_api_vxlan_gpe_ioam_transit_enable_t * mp) -{ - int rv = 0; - vl_api_vxlan_gpe_ioam_transit_enable_reply_t *rmp; - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - ip46_address_t dst_addr; - - memset (&dst_addr.ip4, 0, sizeof (dst_addr.ip4)); - if (!mp->is_ipv6) - { - clib_memcpy (&dst_addr.ip4, &mp->dst_addr, sizeof (dst_addr.ip4)); - } - rv = vxlan_gpe_enable_disable_ioam_for_dest (sm->vlib_main, - dst_addr, - ntohl (mp->outer_fib_index), - mp->is_ipv6 ? 0 : 1, - 1 /* is_add */ ); - - VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_TRANSIT_ENABLE_REPLY); -} - -static void vl_api_vxlan_gpe_ioam_transit_disable_t_handler - (vl_api_vxlan_gpe_ioam_transit_disable_t * mp) -{ - int rv = 0; - vl_api_vxlan_gpe_ioam_transit_disable_reply_t *rmp; - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - ip46_address_t dst_addr; - - memset (&dst_addr.ip4, 0, sizeof (dst_addr.ip4)); - if (!mp->is_ipv6) - { - clib_memcpy (&dst_addr.ip4, &mp->dst_addr, sizeof (dst_addr.ip4)); - } - - rv = vxlan_gpe_ioam_disable_for_dest (sm->vlib_main, - dst_addr, - ntohl (mp->outer_fib_index), - mp->is_ipv6 ? 0 : 1); - VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_TRANSIT_DISABLE_REPLY); -} - - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - clib_error_t *error = 0; - - sm->vlib_main = vm; - sm->vnet_main = h->vnet_main; - sm->unix_time_0 = (u32) time (0); /* Store starting time */ - sm->vlib_time_0 = vlib_time_now (vm); - return error; -} - -/* Set up the API message handling tables */ -static clib_error_t * -vxlan_gpe_plugin_api_hookup (vlib_main_t * vm) -{ - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vxlan_gpe_plugin_api_msg; -#undef _ - - return 0; -} - -static clib_error_t * -vxlan_gpe_init (vlib_main_t * vm) -{ - vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; - clib_error_t *error = 0; - u8 *name; - u32 encap_node_index = vxlan_gpe_encap_ioam_v4_node.index; - u32 decap_node_index = vxlan_gpe_decap_ioam_v4_node.index; - vlib_node_t *vxlan_gpe_encap_node = NULL; - vlib_node_t *vxlan_gpe_decap_node = NULL; - uword next_node = 0; - - name = format (0, "ioam_vxlan_gpe_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - sm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - - error = vxlan_gpe_plugin_api_hookup (vm); - - /* Hook the ioam-encap node to vxlan-gpe-encap */ - vxlan_gpe_encap_node = vlib_get_node_by_name (vm, (u8 *) "vxlan-gpe-encap"); - sm->encap_v4_next_node = - vlib_node_add_next (vm, vxlan_gpe_encap_node->index, encap_node_index); - - vxlan_gpe_decap_node = - vlib_get_node_by_name (vm, (u8 *) "vxlan4-gpe-input"); - next_node = - vlib_node_add_next (vm, vxlan_gpe_decap_node->index, decap_node_index); - vxlan_gpe_register_decap_protocol (VXLAN_GPE_PROTOCOL_IOAM, next_node); - - vec_new (vxlan_gpe_ioam_sw_interface_t, pool_elts (sm->sw_interfaces)); - sm->dst_by_ip4 = hash_create_mem (0, sizeof (fib_prefix_t), sizeof (uword)); - - sm->dst_by_ip6 = hash_create_mem (0, sizeof (fib_prefix_t), sizeof (uword)); - - vxlan_gpe_ioam_interface_init (); - vec_free (name); - - return error; -} - -VLIB_INIT_FUNCTION (vxlan_gpe_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c deleted file mode 100644 index 6c04d9af..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c +++ /dev/null @@ -1,773 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include - -vxlan_gpe_ioam_main_t vxlan_gpe_ioam_main; - -int -vxlan_gpe_ioam_set_rewrite (vxlan_gpe_tunnel_t * t, int has_trace_option, - int has_pot_option, int has_ppc_option, - u8 ipv6_set) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - u32 size; - vxlan_gpe_ioam_hdr_t *vxlan_gpe_ioam_hdr; - u8 *current; - u8 trace_data_size = 0; - u8 pot_data_size = 0; - - if (has_trace_option == 0 && has_pot_option == 0) - return -1; - - /* Work out how much space we need */ - size = sizeof (vxlan_gpe_ioam_hdr_t); - - if (has_trace_option - && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] != 0) - { - size += sizeof (vxlan_gpe_ioam_option_t); - size += hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE]; - } - if (has_pot_option - && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0) - { - size += sizeof (vxlan_gpe_ioam_option_t); - size += hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]; - } - - t->rewrite_size = size; - - if (!ipv6_set) - { - vxlan4_gpe_rewrite (t, size, VXLAN_GPE_PROTOCOL_IOAM, - hm->encap_v4_next_node); - vxlan_gpe_ioam_hdr = - (vxlan_gpe_ioam_hdr_t *) (t->rewrite + - sizeof (ip4_vxlan_gpe_header_t)); - } - else - { - vxlan6_gpe_rewrite (t, size, VXLAN_GPE_PROTOCOL_IOAM, - VXLAN_GPE_ENCAP_NEXT_IP6_LOOKUP); - vxlan_gpe_ioam_hdr = - (vxlan_gpe_ioam_hdr_t *) (t->rewrite + - sizeof (ip6_vxlan_gpe_header_t)); - } - - - vxlan_gpe_ioam_hdr->type = VXLAN_GPE_PROTOCOL_IOAM; - /* Length of the header in octets */ - vxlan_gpe_ioam_hdr->length = size; - vxlan_gpe_ioam_hdr->protocol = t->protocol; - current = (u8 *) vxlan_gpe_ioam_hdr + sizeof (vxlan_gpe_ioam_hdr_t); - - if (has_trace_option - && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] != 0) - { - if (0 != hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] (current, - &trace_data_size)) - return -1; - current += trace_data_size; - } - if (has_pot_option - && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0) - { - pot_data_size = - hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]; - if (0 == - hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] - (current, &pot_data_size)) - current += pot_data_size; - } - - return 0; -} - -int -vxlan_gpe_ioam_clear_rewrite (vxlan_gpe_tunnel_t * t, int has_trace_option, - int has_pot_option, int has_ppc_option, - u8 ipv6_set) -{ - - t->rewrite_size = 0; - - if (!ipv6_set) - { - vxlan4_gpe_rewrite (t, 0, 0, VXLAN_GPE_ENCAP_NEXT_IP4_LOOKUP); - } - else - { - vxlan6_gpe_rewrite (t, 0, 0, VXLAN_GPE_ENCAP_NEXT_IP6_LOOKUP); - } - - - return 0; -} - -clib_error_t * -vxlan_gpe_ioam_clear (vxlan_gpe_tunnel_t * t, - int has_trace_option, int has_pot_option, - int has_ppc_option, u8 ipv6_set) -{ - int rv; - rv = vxlan_gpe_ioam_clear_rewrite (t, 0, 0, 0, 0); - - if (rv == 0) - { - return (0); - } - else - { - return clib_error_return_code (0, rv, 0, - "vxlan_gpe_ioam_clear_rewrite returned %d", - rv); - } - -} - - -clib_error_t * -vxlan_gpe_ioam_set (vxlan_gpe_tunnel_t * t, - int has_trace_option, int has_pot_option, - int has_ppc_option, u8 ipv6_set) -{ - int rv; - rv = vxlan_gpe_ioam_set_rewrite (t, has_trace_option, - has_pot_option, has_ppc_option, ipv6_set); - - if (rv == 0) - { - return (0); - } - else - { - return clib_error_return_code (0, rv, 0, - "vxlan_gpe_ioam_set_rewrite returned %d", - rv); - } - -} - -static void -vxlan_gpe_set_clear_output_feature_on_intf (vlib_main_t * vm, - u32 sw_if_index0, u8 is_add) -{ - - - - vnet_feature_enable_disable ("ip4-output", "vxlan-gpe-transit-ioam", - sw_if_index0, is_add, - 0 /* void *feature_config */ , - 0 /* u32 n_feature_config_bytes */ ); - return; -} - -void -vxlan_gpe_clear_output_feature_on_all_intfs (vlib_main_t * vm) -{ - vnet_sw_interface_t *si = 0; - vnet_main_t *vnm = vnet_get_main (); - vnet_interface_main_t *im = &vnm->interface_main; - - pool_foreach (si, im->sw_interfaces, ( - { - vxlan_gpe_set_clear_output_feature_on_intf - (vm, si->sw_if_index, 0); - })); - return; -} - - -extern fib_forward_chain_type_t -fib_entry_get_default_chain_type (const fib_entry_t * fib_entry); - -int -vxlan_gpe_enable_disable_ioam_for_dest (vlib_main_t * vm, - ip46_address_t dst_addr, - u32 outer_fib_index, - u8 is_ipv4, u8 is_add) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - u32 fib_index0 = 0; - u32 sw_if_index0 = ~0; - - fib_node_index_t fei = ~0; - fib_entry_t *fib_entry; - u32 adj_index0; - ip_adjacency_t *adj0; - fib_prefix_t fib_prefix; - //fib_forward_chain_type_t fct; - load_balance_t *lb_m, *lb_b; - const dpo_id_t *dpo0, *dpo1; - u32 i, j; - //vnet_hw_interface_t *hw; - - if (is_ipv4) - { - memset (&fib_prefix, 0, sizeof (fib_prefix_t)); - fib_prefix.fp_len = 32; - fib_prefix.fp_proto = FIB_PROTOCOL_IP4; - fib_prefix.fp_addr = dst_addr; - } - else - { - return 0; - } - - fei = fib_table_lookup (fib_index0, &fib_prefix); - fib_entry = fib_entry_get (fei); - - //fct = fib_entry_get_default_chain_type (fib_entry); - - if (!dpo_id_is_valid (&fib_entry->fe_lb /*[fct] */ )) - { - return (-1); - } - - lb_m = load_balance_get (fib_entry->fe_lb /*[fct] */ .dpoi_index); - - for (i = 0; i < lb_m->lb_n_buckets; i++) - { - dpo0 = load_balance_get_bucket_i (lb_m, i); - - if (dpo0->dpoi_type == DPO_LOAD_BALANCE) - { - lb_b = load_balance_get (dpo0->dpoi_index); - - for (j = 0; j < lb_b->lb_n_buckets; j++) - { - dpo1 = load_balance_get_bucket_i (lb_b, j); - - if (dpo1->dpoi_type == DPO_ADJACENCY) - { - adj_index0 = dpo1->dpoi_index; - - if (ADJ_INDEX_INVALID == adj_index0) - { - continue; - } - - adj0 = - ip_get_adjacency (&(ip4_main.lookup_main), adj_index0); - sw_if_index0 = adj0->rewrite_header.sw_if_index; - - if (~0 == sw_if_index0) - { - continue; - } - - - if (is_add) - { - vnet_feature_enable_disable ("ip4-output", - "vxlan-gpe-transit-ioam", - sw_if_index0, is_add, 0 - /* void *feature_config */ - , 0 /* u32 n_feature_config_bytes */ - ); - - vec_validate_init_empty (hm->bool_ref_by_sw_if_index, - sw_if_index0, ~0); - hm->bool_ref_by_sw_if_index[sw_if_index0] = 1; - } - else - { - hm->bool_ref_by_sw_if_index[sw_if_index0] = ~0; - } - } - } - } - } - - if (is_ipv4) - { - - uword *t = NULL; - vxlan_gpe_ioam_dest_tunnels_t *t1; - fib_prefix_t key4, *key4_copy; - hash_pair_t *hp; - memset (&key4, 0, sizeof (key4)); - key4.fp_proto = FIB_PROTOCOL_IP4; - key4.fp_addr.ip4.as_u32 = fib_prefix.fp_addr.ip4.as_u32; - t = hash_get_mem (hm->dst_by_ip4, &key4); - if (is_add) - { - if (t) - { - return 0; - } - pool_get_aligned (hm->dst_tunnels, t1, CLIB_CACHE_LINE_BYTES); - memset (t1, 0, sizeof (*t1)); - t1->fp_proto = FIB_PROTOCOL_IP4; - t1->dst_addr.ip4.as_u32 = fib_prefix.fp_addr.ip4.as_u32; - key4_copy = clib_mem_alloc (sizeof (*key4_copy)); - clib_memcpy (key4_copy, &key4, sizeof (*key4_copy)); - hash_set_mem (hm->dst_by_ip4, key4_copy, t1 - hm->dst_tunnels); - /* - * Attach to the FIB entry for the VxLAN-GPE destination - * and become its child. The dest route will invoke a callback - * when the fib entry changes, it can be used to - * re-program the output feature on the egress interface. - */ - - const fib_prefix_t tun_dst_pfx = { - .fp_len = 32, - .fp_proto = FIB_PROTOCOL_IP4, - .fp_addr = {.ip4 = t1->dst_addr.ip4,} - }; - - t1->fib_entry_index = - fib_table_entry_special_add (outer_fib_index, - &tun_dst_pfx, - FIB_SOURCE_RR, - FIB_ENTRY_FLAG_NONE, - ADJ_INDEX_INVALID); - t1->sibling_index = - fib_entry_child_add (t1->fib_entry_index, - hm->fib_entry_type, t1 - hm->dst_tunnels); - t1->outer_fib_index = outer_fib_index; - - } - else - { - if (!t) - { - return 0; - } - t1 = pool_elt_at_index (hm->dst_tunnels, t[0]); - hp = hash_get_pair (hm->dst_by_ip4, &key4); - key4_copy = (void *) (hp->key); - hash_unset_mem (hm->dst_by_ip4, &key4); - clib_mem_free (key4_copy); - pool_put (hm->dst_tunnels, t1); - } - } - else - { - // TBD for IPv6 - } - - return 0; -} - -void -vxlan_gpe_refresh_output_feature_on_all_dest (void) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - vxlan_gpe_ioam_dest_tunnels_t *t; - u32 i; - if (pool_elts (hm->dst_tunnels) == 0) - return; - vxlan_gpe_clear_output_feature_on_all_intfs (hm->vlib_main); - i = vec_len (hm->bool_ref_by_sw_if_index); - vec_free (hm->bool_ref_by_sw_if_index); - vec_validate_init_empty (hm->bool_ref_by_sw_if_index, i, ~0); - pool_foreach (t, hm->dst_tunnels, ( - { - vxlan_gpe_enable_disable_ioam_for_dest - (hm->vlib_main, - t->dst_addr, - t->outer_fib_index, - (t->fp_proto == FIB_PROTOCOL_IP4), 1 - /* is_add */ - ); - } - )); - return; -} - -void -vxlan_gpe_clear_output_feature_on_select_intfs (void) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - u32 sw_if_index0 = 0; - for (sw_if_index0 = 0; - sw_if_index0 < vec_len (hm->bool_ref_by_sw_if_index); sw_if_index0++) - { - if (hm->bool_ref_by_sw_if_index[sw_if_index0] == 0xFF) - { - vxlan_gpe_set_clear_output_feature_on_intf - (hm->vlib_main, sw_if_index0, 0); - } - } - - return; -} - -static clib_error_t * -vxlan_gpe_set_ioam_rewrite_command_fn (vlib_main_t * - vm, - unformat_input_t - * input, vlib_cli_command_t * cmd) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - ip46_address_t local, remote; - u8 local_set = 0; - u8 remote_set = 0; - u8 ipv4_set = 0; - u8 ipv6_set = 0; - u32 vni; - u8 vni_set = 0; - u8 disable = 0; - clib_error_t *rv = 0; - vxlan4_gpe_tunnel_key_t key4; - vxlan6_gpe_tunnel_key_t key6; - uword *p; - vxlan_gpe_main_t *gm = &vxlan_gpe_main; - vxlan_gpe_tunnel_t *t = 0; - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "local %U", unformat_ip4_address, &local.ip4)) - { - local_set = 1; - ipv4_set = 1; - } - else - if (unformat (input, "remote %U", unformat_ip4_address, &remote.ip4)) - { - remote_set = 1; - ipv4_set = 1; - } - else if (unformat (input, "local %U", unformat_ip6_address, &local.ip6)) - { - local_set = 1; - ipv6_set = 1; - } - else - if (unformat (input, "remote %U", unformat_ip6_address, &remote.ip6)) - { - remote_set = 1; - ipv6_set = 1; - } - else if (unformat (input, "vni %d", &vni)) - vni_set = 1; - else if (unformat (input, "disable")) - disable = 1; - else - break; - } - - if (local_set == 0) - return clib_error_return (0, "tunnel local address not specified"); - if (remote_set == 0) - return clib_error_return (0, "tunnel remote address not specified"); - if (ipv4_set && ipv6_set) - return clib_error_return (0, "both IPv4 and IPv6 addresses specified"); - if ((ipv4_set - && memcmp (&local.ip4, &remote.ip4, - sizeof (local.ip4)) == 0) || (ipv6_set - && - memcmp - (&local.ip6, - &remote.ip6, - sizeof (local.ip6)) == 0)) - return clib_error_return (0, "src and dst addresses are identical"); - if (vni_set == 0) - return clib_error_return (0, "vni not specified"); - if (!ipv6_set) - { - key4.local = local.ip4.as_u32; - key4.remote = remote.ip4.as_u32; - key4.vni = clib_host_to_net_u32 (vni << 8); - key4.pad = 0; - p = hash_get_mem (gm->vxlan4_gpe_tunnel_by_key, &key4); - } - else - { - key6.local.as_u64[0] = local.ip6.as_u64[0]; - key6.local.as_u64[1] = local.ip6.as_u64[1]; - key6.remote.as_u64[0] = remote.ip6.as_u64[0]; - key6.remote.as_u64[1] = remote.ip6.as_u64[1]; - key6.vni = clib_host_to_net_u32 (vni << 8); - p = hash_get_mem (gm->vxlan6_gpe_tunnel_by_key, &key6); - } - - if (!p) - return clib_error_return (0, "VxLAN Tunnel not found"); - t = pool_elt_at_index (gm->tunnels, p[0]); - if (!disable) - { - rv = - vxlan_gpe_ioam_set (t, hm->has_trace_option, - hm->has_pot_option, hm->has_ppc_option, ipv6_set); - } - else - { - rv = vxlan_gpe_ioam_clear (t, 0, 0, 0, 0); - } - return rv; -} - - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (vxlan_gpe_set_ioam_rewrite_cmd, static) = { - .path = "set vxlan-gpe-ioam", - .short_help = "set vxlan-gpe-ioam vxlan [disable]", - .function = vxlan_gpe_set_ioam_rewrite_command_fn, -}; -/* *INDENT-ON* */ - - - -clib_error_t * -vxlan_gpe_ioam_enable (int has_trace_option, - int has_pot_option, int has_ppc_option) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - hm->has_trace_option = has_trace_option; - hm->has_pot_option = has_pot_option; - hm->has_ppc_option = has_ppc_option; - if (hm->has_trace_option) - { - vxlan_gpe_trace_profile_setup (); - } - - return 0; -} - -clib_error_t * -vxlan_gpe_ioam_disable (int - has_trace_option, - int has_pot_option, int has_ppc_option) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - hm->has_trace_option = has_trace_option; - hm->has_pot_option = has_pot_option; - hm->has_ppc_option = has_ppc_option; - if (!hm->has_trace_option) - { - vxlan_gpe_trace_profile_cleanup (); - } - - return 0; -} - -void -vxlan_gpe_set_next_override (uword next) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - hm->decap_v4_next_override = next; - return; -} - -static clib_error_t * -vxlan_gpe_set_ioam_flags_command_fn (vlib_main_t * vm, - unformat_input_t - * input, vlib_cli_command_t * cmd) -{ - int has_trace_option = 0; - int has_pot_option = 0; - int has_ppc_option = 0; - clib_error_t *rv = 0; - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "trace")) - has_trace_option = 1; - else if (unformat (input, "pot")) - has_pot_option = 1; - else if (unformat (input, "ppc encap")) - has_ppc_option = PPC_ENCAP; - else if (unformat (input, "ppc decap")) - has_ppc_option = PPC_DECAP; - else if (unformat (input, "ppc none")) - has_ppc_option = PPC_NONE; - else - break; - } - - - rv = - vxlan_gpe_ioam_enable (has_trace_option, has_pot_option, has_ppc_option); - return rv; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (vxlan_gpe_set_ioam_flags_cmd, static) = -{ -.path = "set vxlan-gpe-ioam rewrite", -.short_help = "set vxlan-gpe-ioam [trace] [pot] [ppc ]", -.function = vxlan_gpe_set_ioam_flags_command_fn,}; -/* *INDENT-ON* */ - - -int vxlan_gpe_ioam_disable_for_dest - (vlib_main_t * vm, ip46_address_t dst_addr, u32 outer_fib_index, - u8 ipv4_set) -{ - vxlan_gpe_ioam_dest_tunnels_t *t; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - vxlan_gpe_enable_disable_ioam_for_dest (hm->vlib_main, - dst_addr, outer_fib_index, ipv4_set, - 0); - if (pool_elts (hm->dst_tunnels) == 0) - { - vxlan_gpe_clear_output_feature_on_select_intfs (); - return 0; - } - - pool_foreach (t, hm->dst_tunnels, ( - { - vxlan_gpe_enable_disable_ioam_for_dest - (hm->vlib_main, - t->dst_addr, - t->outer_fib_index, - (t->fp_proto == - FIB_PROTOCOL_IP4), 1 /* is_add */ ); - } - )); - vxlan_gpe_clear_output_feature_on_select_intfs (); - return (0); - -} - -static clib_error_t *vxlan_gpe_set_ioam_transit_rewrite_command_fn - (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - ip46_address_t dst_addr; - u8 dst_addr_set = 0; - u8 ipv4_set = 0; - u8 ipv6_set = 0; - u8 disable = 0; - clib_error_t *rv = 0; - u32 outer_fib_index = 0; - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "dst-ip %U", unformat_ip4_address, &dst_addr.ip4)) - { - dst_addr_set = 1; - ipv4_set = 1; - } - else - if (unformat - (input, "dst-ip %U", unformat_ip6_address, &dst_addr.ip6)) - { - dst_addr_set = 1; - ipv6_set = 1; - } - else if (unformat (input, "outer-fib-index %d", &outer_fib_index)) - { - } - - else if (unformat (input, "disable")) - disable = 1; - else - break; - } - - if (dst_addr_set == 0) - return clib_error_return (0, "tunnel destination address not specified"); - if (ipv4_set && ipv6_set) - return clib_error_return (0, "both IPv4 and IPv6 addresses specified"); - if (!disable) - { - vxlan_gpe_enable_disable_ioam_for_dest (hm->vlib_main, - dst_addr, outer_fib_index, - ipv4_set, 1); - } - else - { - vxlan_gpe_ioam_disable_for_dest - (vm, dst_addr, outer_fib_index, ipv4_set); - } - return rv; -} - - /* *INDENT-OFF* */ -VLIB_CLI_COMMAND (vxlan_gpe_set_ioam_transit_rewrite_cmd, static) = { - .path = "set vxlan-gpe-ioam-transit", - .short_help = "set vxlan-gpe-ioam-transit dst-ip [outer-fib-index ] [disable]", - .function = vxlan_gpe_set_ioam_transit_rewrite_command_fn, -}; -/* *INDENT-ON* */ - -clib_error_t *clear_vxlan_gpe_ioam_rewrite_command_fn - (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) -{ - return (vxlan_gpe_ioam_disable (0, 0, 0)); -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (vxlan_gpe_clear_ioam_flags_cmd, static) = -{ -.path = "clear vxlan-gpe-ioam rewrite", -.short_help = "clear vxlan-gpe-ioam rewrite", -.function = clear_vxlan_gpe_ioam_rewrite_command_fn, -}; -/* *INDENT-ON* */ - - -/** - * Function definition to backwalk a FIB node - */ -static fib_node_back_walk_rc_t -vxlan_gpe_ioam_back_walk (fib_node_t * node, fib_node_back_walk_ctx_t * ctx) -{ - vxlan_gpe_refresh_output_feature_on_all_dest (); - return (FIB_NODE_BACK_WALK_CONTINUE); -} - -/** - * Function definition to get a FIB node from its index - */ -static fib_node_t * -vxlan_gpe_ioam_fib_node_get (fib_node_index_t index) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - return (&hm->node); -} - -/** - * Function definition to inform the FIB node that its last lock has gone. - */ -static void -vxlan_gpe_ioam_last_lock_gone (fib_node_t * node) -{ - ASSERT (0); -} - - -/* - * Virtual function table registered by MPLS GRE tunnels - * for participation in the FIB object graph. - */ -const static fib_node_vft_t vxlan_gpe_ioam_vft = { - .fnv_get = vxlan_gpe_ioam_fib_node_get, - .fnv_last_lock = vxlan_gpe_ioam_last_lock_gone, - .fnv_back_walk = vxlan_gpe_ioam_back_walk, -}; - -void -vxlan_gpe_ioam_interface_init (void) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - hm->fib_entry_type = fib_node_register_new_type (&vxlan_gpe_ioam_vft); - return; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h deleted file mode 100644 index 3b7d72cf..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_vxlan_gpe_ioam_h__ -#define __included_vxlan_gpe_ioam_h__ - -#include -#include -#include -#include - - -typedef struct vxlan_gpe_sw_interface_ -{ - u32 sw_if_index; -} vxlan_gpe_ioam_sw_interface_t; - -typedef struct vxlan_gpe_dest_tunnels_ -{ - ip46_address_t dst_addr; - u32 fp_proto; - u32 sibling_index; - fib_node_index_t fib_entry_index; - u32 outer_fib_index; -} vxlan_gpe_ioam_dest_tunnels_t; - -typedef struct vxlan_gpe_ioam_main_ -{ - /** - * Linkage into the FIB object graph - */ - fib_node_t node; - - /* time scale transform. Joy. */ - u32 unix_time_0; - f64 vlib_time_0; - - - /* Trace option */ - u8 has_trace_option; - - /* Pot option */ - u8 has_pot_option; - -#define PPC_NONE 0 -#define PPC_ENCAP 1 -#define PPC_DECAP 2 - u8 has_ppc_option; - -#define TSP_SECONDS 0 -#define TSP_MILLISECONDS 1 -#define TSP_MICROSECONDS 2 -#define TSP_NANOSECONDS 3 - - /* Array of function pointers to ADD and POP VxLAN-GPE iOAM option handling routines */ - u8 options_size[256]; - int (*add_options[256]) (u8 * rewrite_string, u8 * rewrite_size); - int (*pop_options[256]) (ip4_header_t * ip, vxlan_gpe_ioam_option_t * opt); - - /* Array of function pointers to iOAM option handling routines */ - int (*options[256]) (vlib_buffer_t * b, vxlan_gpe_ioam_option_t * opt, - u8 is_ipv4, u8 use_adj); - u8 *(*trace[256]) (u8 * s, vxlan_gpe_ioam_option_t * opt); - - /* API message ID base */ - u16 msg_id_base; - - /* Override to export for iOAM */ - uword decap_v4_next_override; - uword decap_v6_next_override; - - /* sequence of node graph for encap */ - uword encap_v4_next_node; - uword encap_v6_next_node; - - /* Software interfaces. */ - vxlan_gpe_ioam_sw_interface_t *sw_interfaces; - - /* hash ip4/ip6 -> list of destinations for doing transit iOAM operation */ - vxlan_gpe_ioam_dest_tunnels_t *dst_tunnels; - uword *dst_by_ip4; - uword *dst_by_ip6; - - /** per sw_if_index, to maintain bitmap */ - u8 *bool_ref_by_sw_if_index; - fib_node_type_t fib_entry_type; - - /** State convenience vlib_main_t */ - vlib_main_t *vlib_main; - /** State convenience vnet_main_t */ - vnet_main_t *vnet_main; - - -} vxlan_gpe_ioam_main_t; -extern vxlan_gpe_ioam_main_t vxlan_gpe_ioam_main; - -/* - * Primary h-b-h handler trace support - */ -typedef struct -{ - u32 next_index; - u32 trace_len; - u8 option_data[256]; -} ioam_trace_t; - - -vlib_node_registration_t vxlan_gpe_encap_ioam_v4_node; -vlib_node_registration_t vxlan_gpe_decap_ioam_v4_node; -vlib_node_registration_t vxlan_gpe_transit_ioam_v4_node; - -clib_error_t *vxlan_gpe_ioam_enable (int has_trace_option, int has_pot_option, - int has_ppc_option); - -clib_error_t *vxlan_gpe_ioam_disable (int has_trace_option, - int has_pot_option, int has_ppc_option); - -clib_error_t *vxlan_gpe_ioam_set (vxlan_gpe_tunnel_t * t, - int has_trace_option, - int has_pot_option, - int has_ppc_option, u8 ipv6_set); -clib_error_t *vxlan_gpe_ioam_clear (vxlan_gpe_tunnel_t * t, - int has_trace_option, int has_pot_option, - int has_ppc_option, u8 ipv6_set); - -int vxlan_gpe_ioam_add_register_option (u8 option, - u8 size, - int rewrite_options (u8 * - rewrite_string, - u8 * - rewrite_size)); - -int vxlan_gpe_add_unregister_option (u8 option); - -int vxlan_gpe_ioam_register_option (u8 option, - int options (vlib_buffer_t * b, - vxlan_gpe_ioam_option_t * - opt, u8 is_ipv4, u8 use_adj), - u8 * trace (u8 * s, - vxlan_gpe_ioam_option_t * - opt)); -int vxlan_gpe_ioam_unregister_option (u8 option); - -int vxlan_gpe_trace_profile_setup (void); - -int vxlan_gpe_trace_profile_cleanup (void); -extern void vxlan_gpe_ioam_interface_init (void); -int -vxlan_gpe_enable_disable_ioam_for_dest (vlib_main_t * vm, - ip46_address_t dst_addr, - u32 outer_fib_index, - u8 is_ipv4, u8 is_add); -int vxlan_gpe_ioam_disable_for_dest - (vlib_main_t * vm, ip46_address_t dst_addr, u32 outer_fib_index, - u8 ipv4_set); - -typedef enum -{ - VXLAN_GPE_DECAP_IOAM_V4_NEXT_POP, - VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, - VXLAN_GPE_DECAP_IOAM_V4_N_NEXT -} vxlan_gpe_decap_ioam_v4_next_t; - -#endif - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h deleted file mode 100644 index a7ef859e..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_vxlan_gpe_ioam_packet_h__ -#define __included_vxlan_gpe_ioam_packet_h__ - -#include -#include -#include - - - -#define VXLAN_GPE_OPTION_TYPE_IOAM_TRACE 59 -#define VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT 60 - -/** - * @brief VXLAN GPE Extension (iOAM) Header definition - */ -typedef struct -{ - u8 type; - u8 length; - /** Reserved */ - u8 reserved; - /** see vxlan_gpe_protocol_t */ - u8 protocol; -} vxlan_gpe_ioam_hdr_t; - -/* - * @brief VxLAN GPE iOAM Option definition - */ -typedef struct -{ - /* Option Type */ - u8 type; - /* Length in octets of the option data field */ - u8 length; -} vxlan_gpe_ioam_option_t; - - -#endif - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c deleted file mode 100644 index e37b1642..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c +++ /dev/null @@ -1,552 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -/* Timestamp precision multipliers for seconds, milliseconds, microseconds - * and nanoseconds respectively. - */ -static f64 trace_tsp_mul[4] = { 1, 1e3, 1e6, 1e9 }; - -typedef union -{ - u64 as_u64; - u32 as_u32[2]; -} time_u64_t; - - -/* *INDENT-OFF* */ -typedef CLIB_PACKED(struct { - vxlan_gpe_ioam_option_t hdr; - u8 ioam_trace_type; - u8 data_list_elts_left; - u32 elts[0]; /* Variable type. So keep it generic */ -}) vxlan_gpe_ioam_trace_option_t; -/* *INDENT-ON* */ - - -#define foreach_vxlan_gpe_ioam_trace_stats \ - _(SUCCESS, "Pkts updated with TRACE records") \ - _(FAILED, "Errors in TRACE due to lack of TRACE records") - -static char *vxlan_gpe_ioam_trace_stats_strings[] = { -#define _(sym,string) string, - foreach_vxlan_gpe_ioam_trace_stats -#undef _ -}; - -typedef enum -{ -#define _(sym,str) VXLAN_GPE_IOAM_TRACE_##sym, - foreach_vxlan_gpe_ioam_trace_stats -#undef _ - VXLAN_GPE_IOAM_TRACE_N_STATS, -} vxlan_gpe_ioam_trace_stats_t; - - -typedef struct -{ - /* stats */ - u64 counters[ARRAY_LEN (vxlan_gpe_ioam_trace_stats_strings)]; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; -} vxlan_gpe_ioam_trace_main_t; - -vxlan_gpe_ioam_trace_main_t vxlan_gpe_ioam_trace_main; - -int -vxlan_gpe_ioam_add_register_option (u8 option, - u8 size, - int rewrite_options (u8 * rewrite_string, - u8 * rewrite_size)) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - ASSERT (option < ARRAY_LEN (hm->add_options)); - - /* Already registered */ - if (hm->add_options[option]) - return (-1); - - hm->add_options[option] = rewrite_options; - hm->options_size[option] = size; - - return (0); -} - -int -vxlan_gpe_add_unregister_option (u8 option) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - ASSERT (option < ARRAY_LEN (hm->add_options)); - - /* Not registered */ - if (!hm->add_options[option]) - return (-1); - - hm->add_options[option] = NULL; - hm->options_size[option] = 0; - return (0); -} - - -int -vxlan_gpe_ioam_register_option (u8 option, - int options (vlib_buffer_t * b, - vxlan_gpe_ioam_option_t * opt, - u8 is_ipv4, u8 use_adj), - u8 * trace (u8 * s, - vxlan_gpe_ioam_option_t * opt)) -{ - vxlan_gpe_ioam_main_t *im = &vxlan_gpe_ioam_main; - - ASSERT (option < ARRAY_LEN (im->options)); - - /* Already registered */ - if (im->options[option]) - return (-1); - - im->options[option] = options; - im->trace[option] = trace; - - return (0); -} - -int -vxlan_gpe_ioam_unregister_option (u8 option) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - ASSERT (option < ARRAY_LEN (hm->options)); - - /* Not registered */ - if (!hm->options[option]) - return (-1); - - hm->options[option] = NULL; - hm->trace[option] = NULL; - - return (0); -} - - -always_inline void -vxlan_gpe_ioam_trace_stats_increment_counter (u32 counter_index, - u64 increment) -{ - vxlan_gpe_ioam_trace_main_t *hm = &vxlan_gpe_ioam_trace_main; - - hm->counters[counter_index] += increment; -} - - -static u8 * -format_ioam_data_list_element (u8 * s, va_list * args) -{ - u32 *elt = va_arg (*args, u32 *); - u8 *trace_type_p = va_arg (*args, u8 *); - u8 trace_type = *trace_type_p; - - - if (trace_type & BIT_TTL_NODEID) - { - u32 ttl_node_id_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "ttl 0x%x node id 0x%x ", - ttl_node_id_host_byte_order >> 24, - ttl_node_id_host_byte_order & 0x00FFFFFF); - - elt++; - } - - if (trace_type & BIT_ING_INTERFACE && trace_type & BIT_ING_INTERFACE) - { - u32 ingress_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "ingress 0x%x egress 0x%x ", - ingress_host_byte_order >> 16, - ingress_host_byte_order & 0xFFFF); - elt++; - } - - if (trace_type & BIT_TIMESTAMP) - { - u32 ts_in_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "ts 0x%x \n", ts_in_host_byte_order); - elt++; - } - - if (trace_type & BIT_APPDATA) - { - u32 appdata_in_host_byte_order = clib_net_to_host_u32 (*elt); - s = format (s, "app 0x%x ", appdata_in_host_byte_order); - elt++; - } - - return s; -} - - - -int -vxlan_gpe_ioam_trace_rewrite_handler (u8 * rewrite_string, u8 * rewrite_size) -{ - vxlan_gpe_ioam_trace_option_t *trace_option = NULL; - u8 trace_data_size = 0; - u8 trace_option_elts = 0; - trace_profile *profile = NULL; - - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - return (-1); - } - - if (PREDICT_FALSE (!rewrite_string)) - return -1; - - trace_option_elts = profile->num_elts; - trace_data_size = fetch_trace_data_size (profile->trace_type); - trace_option = (vxlan_gpe_ioam_trace_option_t *) rewrite_string; - trace_option->hdr.type = VXLAN_GPE_OPTION_TYPE_IOAM_TRACE; - trace_option->hdr.length = 2 /*ioam_trace_type,data_list_elts_left */ + - trace_option_elts * trace_data_size; - trace_option->ioam_trace_type = profile->trace_type & TRACE_TYPE_MASK; - trace_option->data_list_elts_left = trace_option_elts; - *rewrite_size = - sizeof (vxlan_gpe_ioam_trace_option_t) + - (trace_option_elts * trace_data_size); - - return 0; -} - - -int -vxlan_gpe_ioam_trace_data_list_handler (vlib_buffer_t * b, - vxlan_gpe_ioam_option_t * opt, - u8 is_ipv4, u8 use_adj) -{ - u8 elt_index = 0; - vxlan_gpe_ioam_trace_option_t *trace = - (vxlan_gpe_ioam_trace_option_t *) opt; - time_u64_t time_u64; - u32 *elt; - int rv = 0; - trace_profile *profile = NULL; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - return (-1); - } - - - time_u64.as_u64 = 0; - - if (PREDICT_TRUE (trace->data_list_elts_left)) - { - trace->data_list_elts_left--; - /* fetch_trace_data_size returns in bytes. Convert it to 4-bytes - * to skip to this node's location. - */ - elt_index = - trace->data_list_elts_left * - fetch_trace_data_size (trace->ioam_trace_type) / 4; - elt = &trace->elts[elt_index]; - if (is_ipv4) - { - if (trace->ioam_trace_type & BIT_TTL_NODEID) - { - ip4_header_t *ip0 = vlib_buffer_get_current (b); - /* The transit case is the only case where the TTL decrement happens - * before iOAM processing. For now, use the use_adj flag as an overload. - * We can probably use a separate flag instead of overloading the use_adj flag. - */ - *elt = clib_host_to_net_u32 (((ip0->ttl - 1 + use_adj) << 24) | - profile->node_id); - elt++; - } - - if (trace->ioam_trace_type & BIT_ING_INTERFACE) - { - u16 tx_if = 0; - u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; - ip4_main_t *im4 = &ip4_main; - ip_lookup_main_t *lm = &im4->lookup_main; - if (use_adj) - { - ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index); - tx_if = adj->rewrite_header.sw_if_index & 0xFFFF; - } - - *elt = - (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 | - tx_if; - *elt = clib_host_to_net_u32 (*elt); - elt++; - } - } - else - { - if (trace->ioam_trace_type & BIT_TTL_NODEID) - { - ip6_header_t *ip0 = vlib_buffer_get_current (b); - *elt = clib_host_to_net_u32 ((ip0->hop_limit << 24) | - profile->node_id); - elt++; - } - if (trace->ioam_trace_type & BIT_ING_INTERFACE) - { - u16 tx_if = 0; - u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; - ip6_main_t *im6 = &ip6_main; - ip_lookup_main_t *lm = &im6->lookup_main; - if (use_adj) - { - ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index); - tx_if = adj->rewrite_header.sw_if_index & 0xFFFF; - } - - *elt = - (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 | - tx_if; - *elt = clib_host_to_net_u32 (*elt); - elt++; - } - } - - if (trace->ioam_trace_type & BIT_TIMESTAMP) - { - /* Send least significant 32 bits */ - f64 time_f64 = - (f64) (((f64) hm->unix_time_0) + - (vlib_time_now (hm->vlib_main) - hm->vlib_time_0)); - - time_u64.as_u64 = time_f64 * trace_tsp_mul[profile->trace_tsp]; - *elt = clib_host_to_net_u32 (time_u64.as_u32[0]); - elt++; - } - - if (trace->ioam_trace_type & BIT_APPDATA) - { - /* $$$ set elt0->app_data */ - *elt = clib_host_to_net_u32 (profile->app_data); - elt++; - } - vxlan_gpe_ioam_trace_stats_increment_counter - (VXLAN_GPE_IOAM_TRACE_SUCCESS, 1); - } - else - { - vxlan_gpe_ioam_trace_stats_increment_counter - (VXLAN_GPE_IOAM_TRACE_FAILED, 1); - } - return (rv); -} - -u8 * -vxlan_gpe_ioam_trace_data_list_trace_handler (u8 * s, - vxlan_gpe_ioam_option_t * opt) -{ - vxlan_gpe_ioam_trace_option_t *trace; - u8 trace_data_size_in_words = 0; - u32 *elt; - int elt_index = 0; - - trace = (vxlan_gpe_ioam_trace_option_t *) opt; - s = - format (s, " Trace Type 0x%x , %d elts left\n", trace->ioam_trace_type, - trace->data_list_elts_left); - trace_data_size_in_words = - fetch_trace_data_size (trace->ioam_trace_type) / 4; - elt = &trace->elts[0]; - while ((u8 *) elt < ((u8 *) (&trace->elts[0]) + trace->hdr.length - 2 - /* -2 accounts for ioam_trace_type,elts_left */ )) - { - s = format (s, " [%d] %U\n", elt_index, - format_ioam_data_list_element, - elt, &trace->ioam_trace_type); - elt_index++; - elt += trace_data_size_in_words; - } - return (s); -} - - -static clib_error_t * -vxlan_gpe_show_ioam_trace_cmd_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - vxlan_gpe_ioam_trace_main_t *hm = &vxlan_gpe_ioam_trace_main; - u8 *s = 0; - int i = 0; - - for (i = 0; i < VXLAN_GPE_IOAM_TRACE_N_STATS; i++) - { - s = format (s, " %s - %lu\n", vxlan_gpe_ioam_trace_stats_strings[i], - hm->counters[i]); - } - - vlib_cli_output (vm, "%v", s); - vec_free (s); - return 0; -} - - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (vxlan_gpe_show_ioam_trace_cmd, static) = { - .path = "show ioam vxlan-gpe trace", - .short_help = "iOAM trace statistics", - .function = vxlan_gpe_show_ioam_trace_cmd_fn, -}; -/* *INDENT-ON* */ - - -static clib_error_t * -vxlan_gpe_ioam_trace_init (vlib_main_t * vm) -{ - vxlan_gpe_ioam_trace_main_t *hm = &vxlan_gpe_ioam_trace_main; - clib_error_t *error; - - if ((error = vlib_call_init_function (vm, ip_main_init))) - return (error); - - if ((error = vlib_call_init_function (vm, ip6_lookup_init))) - return error; - - if ((error = vlib_call_init_function (vm, vxlan_gpe_init))) - return (error); - - hm->vlib_main = vm; - hm->vnet_main = vnet_get_main (); - memset (hm->counters, 0, sizeof (hm->counters)); - - if (vxlan_gpe_ioam_register_option - (VXLAN_GPE_OPTION_TYPE_IOAM_TRACE, - vxlan_gpe_ioam_trace_data_list_handler, - vxlan_gpe_ioam_trace_data_list_trace_handler) < 0) - return (clib_error_create - ("registration of VXLAN_GPE_OPTION_TYPE_IOAM_TRACE failed")); - - - if (vxlan_gpe_ioam_add_register_option - (VXLAN_GPE_OPTION_TYPE_IOAM_TRACE, - sizeof (vxlan_gpe_ioam_trace_option_t), - vxlan_gpe_ioam_trace_rewrite_handler) < 0) - return (clib_error_create - ("registration of VXLAN_GPE_OPTION_TYPE_IOAM_TRACE for rewrite failed")); - - - return (0); -} - -VLIB_INIT_FUNCTION (vxlan_gpe_ioam_trace_init); - -int -vxlan_gpe_trace_profile_cleanup (void) -{ - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] = 0; - - return 0; - -} - -static int -vxlan_gpe_ioam_trace_get_sizeof_handler (u32 * result) -{ - u16 size = 0; - u8 trace_data_size = 0; - trace_profile *profile = NULL; - - *result = 0; - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - return (-1); - } - - trace_data_size = fetch_trace_data_size (profile->trace_type); - if (PREDICT_FALSE (trace_data_size == 0)) - return VNET_API_ERROR_INVALID_VALUE; - - if (PREDICT_FALSE (profile->num_elts * trace_data_size > 254)) - return VNET_API_ERROR_INVALID_VALUE; - - size += - sizeof (vxlan_gpe_ioam_trace_option_t) + - profile->num_elts * trace_data_size; - *result = size; - - return 0; -} - - -int -vxlan_gpe_trace_profile_setup (void) -{ - u32 trace_size = 0; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - trace_profile *profile = NULL; - - - profile = trace_profile_find (); - - if (PREDICT_FALSE (!profile)) - { - return (-1); - } - - - if (vxlan_gpe_ioam_trace_get_sizeof_handler (&trace_size) < 0) - return (-1); - - hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] = trace_size; - - return (0); -} - - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h deleted file mode 100644 index c0ad8d9d..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_vxlan_gpe_ioam_util_h__ -#define __included_vxlan_gpe_ioam_util_h__ - -#include -#include -#include - - -typedef struct -{ - u32 tunnel_index; - ioam_trace_t fmt_trace; -} vxlan_gpe_ioam_v4_trace_t; - - -static u8 * -format_vxlan_gpe_ioam_v4_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - vxlan_gpe_ioam_v4_trace_t *t1 = va_arg (*args, vxlan_gpe_ioam_v4_trace_t *); - ioam_trace_t *t = &(t1->fmt_trace); - vxlan_gpe_ioam_option_t *fmt_trace0; - vxlan_gpe_ioam_option_t *opt0, *limit0; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - u8 type0; - - fmt_trace0 = (vxlan_gpe_ioam_option_t *) t->option_data; - - s = format (s, "VXLAN-GPE-IOAM: next_index %d len %d traced %d", - t->next_index, fmt_trace0->length, t->trace_len); - - opt0 = (vxlan_gpe_ioam_option_t *) (fmt_trace0 + 1); - limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) fmt_trace0) + t->trace_len; - - while (opt0 < limit0) - { - type0 = opt0->type; - switch (type0) - { - case 0: /* Pad, just stop */ - opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; - break; - - default: - if (hm->trace[type0]) - { - s = (*hm->trace[type0]) (s, opt0); - } - else - { - s = - format (s, "\n unrecognized option %d length %d", type0, - opt0->length); - } - opt0 = - (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + - sizeof (vxlan_gpe_ioam_option_t)); - break; - } - } - - s = format (s, "VXLAN-GPE-IOAM: tunnel %d", t1->tunnel_index); - return s; -} - - -always_inline void -vxlan_gpe_encap_decap_ioam_v4_one_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_buffer_t * b0, - u32 * next0, u32 drop_node_val, - u8 use_adj) -{ - ip4_header_t *ip0; - udp_header_t *udp_hdr0; - vxlan_gpe_header_t *gpe_hdr0; - vxlan_gpe_ioam_hdr_t *gpe_ioam0; - vxlan_gpe_ioam_option_t *opt0; - vxlan_gpe_ioam_option_t *limit0; - vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; - - /* Populate the iOAM header */ - ip0 = vlib_buffer_get_current (b0); - udp_hdr0 = (udp_header_t *) (ip0 + 1); - gpe_hdr0 = (vxlan_gpe_header_t *) (udp_hdr0 + 1); - gpe_ioam0 = (vxlan_gpe_ioam_hdr_t *) (gpe_hdr0 + 1); - opt0 = (vxlan_gpe_ioam_option_t *) (gpe_ioam0 + 1); - limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) gpe_ioam0 + gpe_ioam0->length); - - /* - * Basic validity checks - */ - if (gpe_ioam0->length > clib_net_to_host_u16 (ip0->length)) - { - *next0 = drop_node_val; - return; - } - - /* Scan the set of h-b-h options, process ones that we understand */ - while (opt0 < limit0) - { - u8 type0; - type0 = opt0->type; - switch (type0) - { - case 0: /* Pad1 */ - opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; - continue; - case 1: /* PadN */ - break; - default: - if (hm->options[type0]) - { - if ((*hm->options[type0]) (b0, opt0, 1 /* is_ipv4 */ , - use_adj) < 0) - { - *next0 = drop_node_val; - return; - } - } - break; - } - opt0 = - (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + - sizeof (vxlan_gpe_ioam_hdr_t)); - } - - - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - vxlan_gpe_ioam_v4_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - u32 trace_len = gpe_ioam0->length; - t->fmt_trace.next_index = *next0; - /* Capture the ioam option verbatim */ - trace_len = - trace_len < - ARRAY_LEN (t->fmt_trace. - option_data) ? trace_len : ARRAY_LEN (t->fmt_trace. - option_data); - t->fmt_trace.trace_len = trace_len; - clib_memcpy (&(t->fmt_trace.option_data), gpe_ioam0, trace_len); - } - return; -} - - -#endif - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h deleted file mode 100644 index cc0a10a3..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_vxlan_gpe_msg_enum_h -#define included_vxlan_gpe_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_vxlan_gpe_msg_enum_h */ diff --git a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_test.c b/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_test.c deleted file mode 100644 index 47253eb6..00000000 --- a/plugins/ioam-plugin/ioam/lib-vxlan-gpe/vxlan_gpe_test.c +++ /dev/null @@ -1,600 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * vxlan_gpe_test.c - test harness for vxlan_gpe plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version -#include -#include - -typedef struct -{ - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} vxlan_gpe_test_main_t; - -vxlan_gpe_test_main_t vxlan_gpe_test_main; - -#define foreach_standard_reply_retval_handler \ -_(vxlan_gpe_ioam_enable_reply) \ -_(vxlan_gpe_ioam_disable_reply) \ -_(vxlan_gpe_ioam_vni_enable_reply) \ -_(vxlan_gpe_ioam_vni_disable_reply) \ -_(vxlan_gpe_ioam_transit_enable_reply) \ -_(vxlan_gpe_ioam_transit_disable_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = vxlan_gpe_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(VXLAN_GPE_IOAM_ENABLE_REPLY, vxlan_gpe_ioam_enable_reply) \ -_(VXLAN_GPE_IOAM_DISABLE_REPLY, vxlan_gpe_ioam_disable_reply) \ -_(VXLAN_GPE_IOAM_VNI_ENABLE_REPLY, vxlan_gpe_ioam_vni_enable_reply) \ -_(VXLAN_GPE_IOAM_VNI_DISABLE_REPLY, vxlan_gpe_ioam_vni_disable_reply) \ -_(VXLAN_GPE_IOAM_TRANSIT_ENABLE_REPLY, vxlan_gpe_ioam_transit_enable_reply) \ -_(VXLAN_GPE_IOAM_TRANSIT_DISABLE_REPLY, vxlan_gpe_ioam_transit_disable_reply) \ - - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - - -static int -api_vxlan_gpe_ioam_enable (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - - unformat_input_t *input = vam->input; - vl_api_vxlan_gpe_ioam_enable_t *mp; - f64 timeout; - u32 id = 0; - int has_trace_option = 0; - int has_pow_option = 0; - int has_ppc_option = 0; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "trace")) - has_trace_option = 1; - else if (unformat (input, "pow")) - has_pow_option = 1; - else if (unformat (input, "ppc encap")) - has_ppc_option = PPC_ENCAP; - else if (unformat (input, "ppc decap")) - has_ppc_option = PPC_DECAP; - else if (unformat (input, "ppc none")) - has_ppc_option = PPC_NONE; - else - break; - } - M (VXLAN_GPE_IOAM_ENABLE, vxlan_gpe_ioam_enable); - mp->id = htons (id); - mp->trace_ppc = has_ppc_option; - mp->pow_enable = has_pow_option; - mp->trace_enable = has_trace_option; - - - S; - W; - - return (0); -} - - -static int -api_vxlan_gpe_ioam_disable (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - vl_api_vxlan_gpe_ioam_disable_t *mp; - f64 timeout; - - M (VXLAN_GPE_IOAM_DISABLE, vxlan_gpe_ioam_disable); - S; - W; - return 0; -} - -static int -api_vxlan_gpe_ioam_vni_enable (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - - unformat_input_t *line_input = vam->input; - vl_api_vxlan_gpe_ioam_vni_enable_t *mp; - ip4_address_t local4, remote4; - ip6_address_t local6, remote6; - u8 ipv4_set = 0, ipv6_set = 0; - u8 local_set = 0; - u8 remote_set = 0; - u32 vni; - u8 vni_set = 0; - f64 timeout; - - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "local %U", unformat_ip4_address, &local4)) - { - local_set = 1; - ipv4_set = 1; - } - else if (unformat (line_input, "remote %U", - unformat_ip4_address, &remote4)) - { - remote_set = 1; - ipv4_set = 1; - } - else if (unformat (line_input, "local %U", - unformat_ip6_address, &local6)) - { - local_set = 1; - ipv6_set = 1; - } - else if (unformat (line_input, "remote %U", - unformat_ip6_address, &remote6)) - { - remote_set = 1; - ipv6_set = 1; - } - - else if (unformat (line_input, "vni %d", &vni)) - vni_set = 1; - else - { - errmsg ("parse error '%U'\n", format_unformat_error, line_input); - return -99; - } - } - - if (local_set == 0) - { - errmsg ("tunnel local address not specified\n"); - return -99; - } - if (remote_set == 0) - { - errmsg ("tunnel remote address not specified\n"); - return -99; - } - if (ipv4_set && ipv6_set) - { - errmsg ("both IPv4 and IPv6 addresses specified"); - return -99; - } - - if (vni_set == 0) - { - errmsg ("vni not specified\n"); - return -99; - } - - M (VXLAN_GPE_IOAM_VNI_ENABLE, vxlan_gpe_ioam_vni_enable); - - - if (ipv6_set) - { - clib_memcpy (&mp->local, &local6, sizeof (local6)); - clib_memcpy (&mp->remote, &remote6, sizeof (remote6)); - } - else - { - clib_memcpy (&mp->local, &local4, sizeof (local4)); - clib_memcpy (&mp->remote, &remote4, sizeof (remote4)); - } - - mp->vni = ntohl (vni); - mp->is_ipv6 = ipv6_set; - - S; - W; - - return (0); -} - -static int -api_vxlan_gpe_ioam_vni_disable (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - - unformat_input_t *line_input = vam->input; - vl_api_vxlan_gpe_ioam_vni_disable_t *mp; - ip4_address_t local4, remote4; - ip6_address_t local6, remote6; - u8 ipv4_set = 0, ipv6_set = 0; - u8 local_set = 0; - u8 remote_set = 0; - u32 vni; - u8 vni_set = 0; - f64 timeout; - - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "local %U", unformat_ip4_address, &local4)) - { - local_set = 1; - ipv4_set = 1; - } - else if (unformat (line_input, "remote %U", - unformat_ip4_address, &remote4)) - { - remote_set = 1; - ipv4_set = 1; - } - else if (unformat (line_input, "local %U", - unformat_ip6_address, &local6)) - { - local_set = 1; - ipv6_set = 1; - } - else if (unformat (line_input, "remote %U", - unformat_ip6_address, &remote6)) - { - remote_set = 1; - ipv6_set = 1; - } - - else if (unformat (line_input, "vni %d", &vni)) - vni_set = 1; - else - { - errmsg ("parse error '%U'\n", format_unformat_error, line_input); - return -99; - } - } - - if (local_set == 0) - { - errmsg ("tunnel local address not specified\n"); - return -99; - } - if (remote_set == 0) - { - errmsg ("tunnel remote address not specified\n"); - return -99; - } - if (ipv4_set && ipv6_set) - { - errmsg ("both IPv4 and IPv6 addresses specified"); - return -99; - } - - if (vni_set == 0) - { - errmsg ("vni not specified\n"); - return -99; - } - - M (VXLAN_GPE_IOAM_VNI_DISABLE, vxlan_gpe_ioam_vni_disable); - - - if (ipv6_set) - { - clib_memcpy (&mp->local, &local6, sizeof (local6)); - clib_memcpy (&mp->remote, &remote6, sizeof (remote6)); - } - else - { - clib_memcpy (&mp->local, &local4, sizeof (local4)); - clib_memcpy (&mp->remote, &remote4, sizeof (remote4)); - } - - mp->vni = ntohl (vni); - mp->is_ipv6 = ipv6_set; - - S; - W; - - return 0; -} - -static int -api_vxlan_gpe_ioam_transit_enable (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - - unformat_input_t *line_input = vam->input; - vl_api_vxlan_gpe_ioam_transit_enable_t *mp; - ip4_address_t local4; - ip6_address_t local6; - u8 ipv4_set = 0, ipv6_set = 0; - u8 local_set = 0; - u32 outer_fib_index = 0; - f64 timeout; - - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "dst-ip %U", unformat_ip4_address, &local4)) - { - local_set = 1; - ipv4_set = 1; - } - else if (unformat (line_input, "dst-ip %U", - unformat_ip6_address, &local6)) - { - local_set = 1; - ipv6_set = 1; - } - - else if (unformat (line_input, "outer-fib-index %d", &outer_fib_index)) - ; - else - { - errmsg ("parse error '%U'\n", format_unformat_error, line_input); - return -99; - } - } - - if (local_set == 0) - { - errmsg ("destination address not specified\n"); - return -99; - } - if (ipv4_set && ipv6_set) - { - errmsg ("both IPv4 and IPv6 addresses specified"); - return -99; - } - - - M (VXLAN_GPE_IOAM_TRANSIT_ENABLE, vxlan_gpe_ioam_transit_enable); - - - if (ipv6_set) - { - errmsg ("IPv6 currently unsupported"); - return -1; - } - else - { - clib_memcpy (&mp->dst_addr, &local4, sizeof (local4)); - } - - mp->outer_fib_index = htonl (outer_fib_index); - mp->is_ipv6 = ipv6_set; - - S; - W; - - return (0); -} - -static int -api_vxlan_gpe_ioam_transit_disable (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - - unformat_input_t *line_input = vam->input; - vl_api_vxlan_gpe_ioam_transit_disable_t *mp; - ip4_address_t local4; - ip6_address_t local6; - u8 ipv4_set = 0, ipv6_set = 0; - u8 local_set = 0; - u32 outer_fib_index; - f64 timeout; - - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "dst-ip %U", unformat_ip4_address, &local4)) - { - local_set = 1; - ipv4_set = 1; - } - else if (unformat (line_input, "dst-ip %U", - unformat_ip6_address, &local6)) - { - local_set = 1; - ipv6_set = 1; - } - - else if (unformat (line_input, "outer-fib-index %d", &outer_fib_index)) - ; - else - { - errmsg ("parse error '%U'\n", format_unformat_error, line_input); - return -99; - } - } - - if (local_set == 0) - { - errmsg ("destination address not specified\n"); - return -99; - } - if (ipv4_set && ipv6_set) - { - errmsg ("both IPv4 and IPv6 addresses specified"); - return -99; - } - - - M (VXLAN_GPE_IOAM_TRANSIT_DISABLE, vxlan_gpe_ioam_transit_disable); - - - if (ipv6_set) - { - return -1; - } - else - { - clib_memcpy (&mp->dst_addr, &local4, sizeof (local4)); - } - - mp->outer_fib_index = htonl (outer_fib_index); - mp->is_ipv6 = ipv6_set; - - S; - W; - - - return (0); -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(vxlan_gpe_ioam_enable, ""\ - "[trace] [pow] [ppc ]") \ -_(vxlan_gpe_ioam_disable, "") \ -_(vxlan_gpe_ioam_vni_enable, ""\ - "local remote vni ") \ -_(vxlan_gpe_ioam_vni_disable, ""\ - "local remote vni ") \ -_(vxlan_gpe_ioam_transit_enable, ""\ - "dst-ip [outer-fib-index ]") \ -_(vxlan_gpe_ioam_transit_disable, ""\ - "dst-ip [outer-fib-index ]") \ - - -void -vat_api_hookup (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * -vat_plugin_register (vat_main_t * vam) -{ - vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; - u8 *name; - - sm->vat_main = vam; - - name = format (0, "ioam_vxlan_gpe_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~ 0) - vat_api_hookup (vam); - - vec_free (name); - - return 0; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/plugins/lb-plugin/Makefile.am b/plugins/lb-plugin/Makefile.am deleted file mode 100644 index 8e360279..00000000 --- a/plugins/lb-plugin/Makefile.am +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2016 Cisco Systems, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AUTOMAKE_OPTIONS = foreign subdir-objects - -AM_CFLAGS = -Wall -AM_LDFLAGS = -module -shared -avoid-version - -vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins -vpppluginsdir = ${libdir}/vpp_plugins - -vppapitestplugins_LTLIBRARIES = lb_test_plugin.la -vppplugins_LTLIBRARIES = lb_plugin.la - -lb_plugin_la_SOURCES = lb/lb.c lb/node.c lb/cli.c lb/util.c lb/refcount.c lb/api.c - -BUILT_SOURCES = \ - lb/lb.api.h \ - lb/lb.api.json - -SUFFIXES = .api.h .api .api.json - -%.api.h: %.api - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --output $@ --show-name $@ - -%.api.json: %.api - @echo " JSON APIGEN " $@ ; \ - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --json $@ - -apidir = $(prefix)/lb/ -api_DATA = lb/lb.api.json - -noinst_HEADERS = lb/lb.h lb/util.h lb/refcount.h lb/lbhash.h lb/lb.api.h - -lb_test_plugin_la_SOURCES = \ - lb/lb_test.c lb/lb_plugin.api.h - -# Remove *.la files -install-data-hook: - @(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) - @(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) diff --git a/plugins/lb-plugin/configure.ac b/plugins/lb-plugin/configure.ac deleted file mode 100644 index 1b02e54f..00000000 --- a/plugins/lb-plugin/configure.ac +++ /dev/null @@ -1,9 +0,0 @@ -AC_INIT(lb_plugin, 1.0) -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) -AC_PREFIX_DEFAULT([/usr]) - -AC_PROG_LIBTOOL -AC_PROG_CC - -AC_OUTPUT([Makefile]) diff --git a/plugins/lb-plugin/lb/api.c b/plugins/lb-plugin/lb/api.c deleted file mode 100644 index 06c53fa1..00000000 --- a/plugins/lb-plugin/lb/api.c +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (lb_main_t * lbm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + lbm->msg_id_base); - foreach_vl_msg_name_crc_lb; -#undef _ -} - -/* Macro to finish up custom dump fns */ -#define FINISH \ - vec_add1 (s, 0); \ - vl_print (handle, (char *)s); \ - vec_free (s); \ - return handle; - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+lbm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -static void -vl_api_lb_conf_t_handler -(vl_api_lb_conf_t * mp) -{ - lb_main_t *lbm = &lb_main; - vl_api_lb_conf_reply_t * rmp; - int rv = 0; - - rv = lb_conf((ip4_address_t *)&mp->ip4_src_address, - (ip6_address_t *)mp->ip6_src_address, - mp->sticky_buckets_per_core, - mp->flow_timeout); - - REPLY_MACRO (VL_API_LB_CONF_REPLY); -} - -static void *vl_api_lb_conf_t_print -(vl_api_lb_conf_t *mp, void * handle) -{ - u8 * s; - s = format (0, "SCRIPT: lb_conf "); - s = format (s, "%U ", format_ip4_address, (ip4_address_t *)&mp->ip4_src_address); - s = format (s, "%U ", format_ip6_address, (ip6_address_t *)mp->ip6_src_address); - s = format (s, "%u ", mp->sticky_buckets_per_core); - s = format (s, "%u ", mp->flow_timeout); - FINISH; -} - - -static void -vl_api_lb_add_del_vip_t_handler -(vl_api_lb_add_del_vip_t * mp) -{ - lb_main_t *lbm = &lb_main; - vl_api_lb_conf_reply_t * rmp; - int rv = 0; - ip46_address_t prefix; - memcpy(&prefix.ip6, mp->ip_prefix, sizeof(prefix.ip6)); - - if (mp->is_del) { - u32 vip_index; - if (!(rv = lb_vip_find_index(&prefix, mp->prefix_length, &vip_index))) - rv = lb_vip_del(vip_index); - } else { - u32 vip_index; - lb_vip_type_t type; - if (ip46_prefix_is_ip4(&prefix, mp->prefix_length)) { - type = mp->is_gre4?LB_VIP_TYPE_IP4_GRE4:LB_VIP_TYPE_IP4_GRE6; - } else { - type = mp->is_gre4?LB_VIP_TYPE_IP6_GRE4:LB_VIP_TYPE_IP6_GRE6; - } - - rv = lb_vip_add(&prefix, mp->prefix_length, type, - mp->new_flows_table_length, &vip_index); - } - REPLY_MACRO (VL_API_LB_CONF_REPLY); -} - -static void *vl_api_lb_add_del_vip_t_print -(vl_api_lb_add_del_vip_t *mp, void * handle) -{ - u8 * s; - s = format (0, "SCRIPT: lb_add_del_vip "); - s = format (s, "%U ", format_ip46_prefix, - (ip46_address_t *)mp->ip_prefix, mp->prefix_length, IP46_TYPE_ANY); - s = format (s, "%s ", mp->is_gre4?"gre4":"gre6"); - s = format (s, "%u ", mp->new_flows_table_length); - s = format (s, "%s ", mp->is_del?"del":"add"); - FINISH; -} - -static void -vl_api_lb_add_del_as_t_handler -(vl_api_lb_add_del_as_t * mp) -{ - lb_main_t *lbm = &lb_main; - vl_api_lb_conf_reply_t * rmp; - int rv = 0; - u32 vip_index; - if ((rv = lb_vip_find_index((ip46_address_t *)mp->vip_ip_prefix, - mp->vip_prefix_length, &vip_index))) - goto done; - - if (mp->is_del) - rv = lb_vip_del_ass(vip_index, (ip46_address_t *)mp->as_address, 1); - else - rv = lb_vip_add_ass(vip_index, (ip46_address_t *)mp->as_address, 1); - -done: - REPLY_MACRO (VL_API_LB_CONF_REPLY); -} - -static void *vl_api_lb_add_del_as_t_print -(vl_api_lb_add_del_as_t *mp, void * handle) -{ - u8 * s; - s = format (0, "SCRIPT: lb_add_del_as "); - s = format (s, "%U ", format_ip46_prefix, - (ip46_address_t *)mp->vip_ip_prefix, mp->vip_prefix_length, IP46_TYPE_ANY); - s = format (s, "%U ", format_ip46_address, - (ip46_address_t *)mp->as_address, IP46_TYPE_ANY); - s = format (s, "%s ", mp->is_del?"del":"add"); - FINISH; -} - -/* List of message types that this plugin understands */ -#define foreach_lb_plugin_api_msg \ -_(LB_CONF, lb_conf) \ -_(LB_ADD_DEL_VIP, lb_add_del_vip) \ -_(LB_ADD_DEL_AS, lb_add_del_as) - -static clib_error_t * lb_api_init (vlib_main_t * vm) -{ - lb_main_t *lbm = &lb_main; - u8 *name = format (0, "lb_%08x%c", api_version, 0); - lbm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + lbm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_lb_plugin_api_msg; -#undef _ - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (lbm, &api_main); - - return 0; -} - -VLIB_INIT_FUNCTION (lb_api_init); diff --git a/plugins/lb-plugin/lb/cli.c b/plugins/lb-plugin/lb/cli.c deleted file mode 100644 index b59c6426..00000000 --- a/plugins/lb-plugin/lb/cli.c +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -static clib_error_t * -lb_vip_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - ip46_address_t prefix; - u8 plen; - u32 new_len = 1024; - u8 del = 0; - int ret; - u32 gre4 = 0; - lb_vip_type_t type; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - if (!unformat(line_input, "%U", unformat_ip46_prefix, &prefix, &plen, IP46_TYPE_ANY, &plen)) - return clib_error_return (0, "invalid vip prefix: '%U'", - format_unformat_error, line_input); - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(line_input, "new_len %d", &new_len)) - ; - else if (unformat(line_input, "del")) - del = 1; - else if (unformat(line_input, "encap gre4")) - gre4 = 1; - else if (unformat(line_input, "encap gre6")) - gre4 = 0; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - - if (ip46_prefix_is_ip4(&prefix, plen)) { - type = (gre4)?LB_VIP_TYPE_IP4_GRE4:LB_VIP_TYPE_IP4_GRE6; - } else { - type = (gre4)?LB_VIP_TYPE_IP6_GRE4:LB_VIP_TYPE_IP6_GRE6; - } - - lb_garbage_collection(); - - u32 index; - if (!del) { - if ((ret = lb_vip_add(&prefix, plen, type, new_len, &index))) { - return clib_error_return (0, "lb_vip_add error %d", ret); - } else { - vlib_cli_output(vm, "lb_vip_add ok %d", index); - } - } else { - if ((ret = lb_vip_find_index(&prefix, plen, &index))) - return clib_error_return (0, "lb_vip_find_index error %d", ret); - else if ((ret = lb_vip_del(index))) - return clib_error_return (0, "lb_vip_del error %d", ret); - } - return NULL; -} - -VLIB_CLI_COMMAND (lb_vip_command, static) = -{ - .path = "lb vip", - .short_help = "lb vip [encap (gre6|gre4)] [new_len ] [del]", - .function = lb_vip_command_fn, -}; - -static clib_error_t * -lb_as_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - ip46_address_t vip_prefix, as_addr; - u8 vip_plen; - ip46_address_t *as_array = 0; - u32 vip_index; - u8 del = 0; - int ret; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - if (!unformat(line_input, "%U", unformat_ip46_prefix, &vip_prefix, &vip_plen, IP46_TYPE_ANY)) - return clib_error_return (0, "invalid as address: '%U'", - format_unformat_error, line_input); - - if ((ret = lb_vip_find_index(&vip_prefix, vip_plen, &vip_index))) - return clib_error_return (0, "lb_vip_find_index error %d", ret); - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(line_input, "%U", unformat_ip46_address, &as_addr, IP46_TYPE_ANY)) { - vec_add1(as_array, as_addr); - } else if (unformat(line_input, "del")) { - del = 1; - } else { - vec_free(as_array); - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - } - - if (!vec_len(as_array)) { - vec_free(as_array); - return clib_error_return (0, "No AS address provided"); - } - - lb_garbage_collection(); - clib_warning("vip index is %d", vip_index); - - if (del) { - if ((ret = lb_vip_del_ass(vip_index, as_array, vec_len(as_array)))) { - vec_free(as_array); - return clib_error_return (0, "lb_vip_del_ass error %d", ret); - } - } else { - if ((ret = lb_vip_add_ass(vip_index, as_array, vec_len(as_array)))) { - vec_free(as_array); - return clib_error_return (0, "lb_vip_add_ass error %d", ret); - } - } - - vec_free(as_array); - return 0; -} - -VLIB_CLI_COMMAND (lb_as_command, static) = -{ - .path = "lb as", - .short_help = "lb as [
[
[...]]] [del]", - .function = lb_as_command_fn, -}; - -static clib_error_t * -lb_conf_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - lb_main_t *lbm = &lb_main; - unformat_input_t _line_input, *line_input = &_line_input; - ip4_address_t ip4 = lbm->ip4_src_address; - ip6_address_t ip6 = lbm->ip6_src_address; - u32 per_cpu_sticky_buckets = lbm->per_cpu_sticky_buckets; - u32 per_cpu_sticky_buckets_log2 = 0; - u32 flow_timeout = lbm->flow_timeout; - int ret; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat(line_input, "ip4-src-address %U", unformat_ip4_address, &ip4)) - ; - else if (unformat(line_input, "ip6-src-address %U", unformat_ip6_address, &ip6)) - ; - else if (unformat(line_input, "buckets %d", &per_cpu_sticky_buckets)) - ; - else if (unformat(line_input, "buckets-log2 %d", &per_cpu_sticky_buckets_log2)) { - if (per_cpu_sticky_buckets_log2 >= 32) - return clib_error_return (0, "buckets-log2 value is too high"); - per_cpu_sticky_buckets = 1 << per_cpu_sticky_buckets_log2; - } else if (unformat(line_input, "timeout %d", &flow_timeout)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - lb_garbage_collection(); - - if ((ret = lb_conf(&ip4, &ip6, per_cpu_sticky_buckets, flow_timeout))) - return clib_error_return (0, "lb_conf error %d", ret); - - return NULL; -} - -VLIB_CLI_COMMAND (lb_conf_command, static) = -{ - .path = "lb conf", - .short_help = "lb conf [ip4-src-address ] [ip6-src-address ] [buckets ] [timeout ]", - .function = lb_conf_command_fn, -}; - -static clib_error_t * -lb_show_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - vlib_cli_output(vm, "%U", format_lb_main); - return NULL; -} - - -VLIB_CLI_COMMAND (lb_show_command, static) = -{ - .path = "show lb", - .short_help = "show lb", - .function = lb_show_command_fn, -}; - -static clib_error_t * -lb_show_vips_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - unformat_input_t line_input; - lb_main_t *lbm = &lb_main; - lb_vip_t *vip; - u8 verbose = 0; - - if (!unformat_user (input, unformat_line_input, &line_input)) - return 0; - - if (unformat(&line_input, "verbose")) - verbose = 1; - - pool_foreach(vip, lbm->vips, { - vlib_cli_output(vm, "%U\n", verbose?format_lb_vip_detailed:format_lb_vip, vip); - }); - - unformat_free (&line_input); - return NULL; -} - -VLIB_CLI_COMMAND (lb_show_vips_command, static) = -{ - .path = "show lb vips", - .short_help = "show lb vips [verbose]", - .function = lb_show_vips_command_fn, -}; diff --git a/plugins/lb-plugin/lb/lb.api b/plugins/lb-plugin/lb/lb.api deleted file mode 100644 index 39ee3c8f..00000000 --- a/plugins/lb-plugin/lb/lb.api +++ /dev/null @@ -1,71 +0,0 @@ -/** \brief Configure Load-Balancer global parameters - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param ip4_src_address - IPv4 address to be used as source for IPv4 GRE traffic. - @param ip6_src_address - IPv6 address to be used as source for IPv6 GRE traffic. - @param n_sticky_buckets - Number of buckets *per worker thread* in the - established flow table (must be power of 2). - @param flow_timeout - Time in seconds after which, if no packet is received - for a given flow, the flow is removed from the established flow table. -*/ -define lb_conf -{ - u32 client_index; - u32 context; - u32 ip4_src_address; - u8 ip6_src_address[16]; - u32 sticky_buckets_per_core; - u32 flow_timeout; -}; - -define lb_conf_reply { - u32 context; - i32 retval; -}; - -/** \brief Add a virtual address (or prefix) - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param ip_prefix - IP address (IPv4 in lower order 32 bits). - @param prefix_length - IP prefix length (96 + 'IPv4 prefix length' for IPv4). - @param is_gre4 - Encap is ip4 GRE (ip6 GRE otherwise). - @param new_flows_table_length - Size of the new connections flow table used - for this VIP (must be power of 2). - @param is_del - The VIP should be removed. -*/ -define lb_add_del_vip { - u32 client_index; - u32 context; - u8 ip_prefix[16]; - u8 prefix_length; - u8 is_gre4; - u32 new_flows_table_length; - u8 is_del; -}; - -define lb_add_del_vip_reply { - u32 context; - i32 retval; -}; - -/** \brief Add an application server for a given VIP - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param vip_ip_prefix - VIP IP address (IPv4 in lower order 32 bits). - @param vip_ip_prefix - VIP IP prefix length (96 + 'IPv4 prefix length' for IPv4). - @param as_address - The application server address (IPv4 in lower order 32 bits). - @param is_del - The AS should be removed. -*/ -define lb_add_del_as { - u32 client_index; - u32 context; - u8 vip_ip_prefix[16]; - u8 vip_prefix_length; - u8 as_address[16]; - u8 is_del; -}; - -define lb_add_del_as_reply { - u32 context; - i32 retval; -}; diff --git a/plugins/lb-plugin/lb/lb.c b/plugins/lb-plugin/lb/lb.c deleted file mode 100644 index 1d9b9870..00000000 --- a/plugins/lb-plugin/lb/lb.c +++ /dev/null @@ -1,844 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -//GC runs at most once every so many seconds -#define LB_GARBAGE_RUN 60 - -//After so many seconds. It is assumed that inter-core race condition will not occur. -#define LB_CONCURRENCY_TIMEOUT 10 - -lb_main_t lb_main; - -#define lb_get_writer_lock() do {} while(__sync_lock_test_and_set (lb_main.writer_lock, 1)) -#define lb_put_writer_lock() lb_main.writer_lock[0] = 0 - -static void lb_as_stack (lb_as_t *as); - - -const static char * const lb_dpo_gre4_ip4[] = { "lb4-gre4" , NULL }; -const static char * const lb_dpo_gre4_ip6[] = { "lb6-gre4" , NULL }; -const static char* const * const lb_dpo_gre4_nodes[DPO_PROTO_NUM] = - { - [DPO_PROTO_IP4] = lb_dpo_gre4_ip4, - [DPO_PROTO_IP6] = lb_dpo_gre4_ip6, - }; - -const static char * const lb_dpo_gre6_ip4[] = { "lb4-gre6" , NULL }; -const static char * const lb_dpo_gre6_ip6[] = { "lb6-gre6" , NULL }; -const static char* const * const lb_dpo_gre6_nodes[DPO_PROTO_NUM] = - { - [DPO_PROTO_IP4] = lb_dpo_gre6_ip4, - [DPO_PROTO_IP6] = lb_dpo_gre6_ip6, - }; - -u32 lb_hash_time_now(vlib_main_t * vm) -{ - return (u32) (vlib_time_now(vm) + 10000); -} - -u8 *format_lb_main (u8 * s, va_list * args) -{ - vlib_thread_main_t *tm = vlib_get_thread_main(); - lb_main_t *lbm = &lb_main; - s = format(s, "lb_main"); - s = format(s, " ip4-src-address: %U \n", format_ip4_address, &lbm->ip4_src_address); - s = format(s, " ip6-src-address: %U \n", format_ip6_address, &lbm->ip6_src_address); - s = format(s, " #vips: %u\n", pool_elts(lbm->vips)); - s = format(s, " #ass: %u\n", pool_elts(lbm->ass) - 1); - - u32 cpu_index; - for(cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++ ) { - lb_hash_t *h = lbm->per_cpu[cpu_index].sticky_ht; - if (h) { - s = format(s, "core %d\n", cpu_index); - s = format(s, " timeout: %ds\n", h->timeout); - s = format(s, " usage: %d / %d\n", lb_hash_elts(h, lb_hash_time_now(vlib_get_main())), lb_hash_size(h)); - } - } - - return s; -} - -static char *lb_vip_type_strings[] = { - [LB_VIP_TYPE_IP6_GRE6] = "ip6-gre6", - [LB_VIP_TYPE_IP6_GRE4] = "ip6-gre4", - [LB_VIP_TYPE_IP4_GRE6] = "ip4-gre6", - [LB_VIP_TYPE_IP4_GRE4] = "ip4-gre4", -}; - -u8 *format_lb_vip_type (u8 * s, va_list * args) -{ - lb_vip_type_t vipt = va_arg (*args, lb_vip_type_t); - u32 i; - for (i=0; itype, - format_ip46_prefix, &vip->prefix, vip->plen, IP46_TYPE_ANY, - vip->new_flow_table_mask + 1, - pool_elts(vip->as_indexes), - (vip->flags & LB_VIP_FLAGS_USED)?"":" removed"); -} - -u8 *format_lb_as (u8 * s, va_list * args) -{ - lb_as_t *as = va_arg (*args, lb_as_t *); - return format(s, "%U %s", format_ip46_address, - &as->address, IP46_TYPE_ANY, - (as->flags & LB_AS_FLAGS_USED)?"used":"removed"); -} - -u8 *format_lb_vip_detailed (u8 * s, va_list * args) -{ - lb_main_t *lbm = &lb_main; - lb_vip_t *vip = va_arg (*args, lb_vip_t *); - uword indent = format_get_indent (s); - - s = format(s, "%U %U [%u] %U%s\n" - "%U new_size:%u\n", - format_white_space, indent, - format_lb_vip_type, vip->type, - vip - lbm->vips, format_ip46_prefix, &vip->prefix, vip->plen, IP46_TYPE_ANY, - (vip->flags & LB_VIP_FLAGS_USED)?"":" removed", - format_white_space, indent, - vip->new_flow_table_mask + 1); - - //Print counters - s = format(s, "%U counters:\n", - format_white_space, indent); - u32 i; - for (i=0; ivip_counters[i].name, - vlib_get_simple_counter(&lbm->vip_counters[i], vip - lbm->vips)); - - - s = format(s, "%U #as:%u\n", - format_white_space, indent, - pool_elts(vip->as_indexes)); - - //Let's count the buckets for each AS - u32 *count = 0; - vec_validate(count, pool_len(lbm->ass)); //Possibly big alloc for not much... - lb_new_flow_entry_t *nfe; - vec_foreach(nfe, vip->new_flow_table) - count[nfe->as_index]++; - - lb_as_t *as; - u32 *as_index; - pool_foreach(as_index, vip->as_indexes, { - as = &lbm->ass[*as_index]; - s = format(s, "%U %U %d buckets %d flows dpo:%u %s\n", - format_white_space, indent, - format_ip46_address, &as->address, IP46_TYPE_ANY, - count[as - lbm->ass], - vlib_refcount_get(&lbm->as_refcount, as - lbm->ass), - as->dpo.dpoi_index, - (as->flags & LB_AS_FLAGS_USED)?"used":" removed"); - }); - - vec_free(count); - - /* - s = format(s, "%U new flows table:\n", format_white_space, indent); - lb_new_flow_entry_t *nfe; - vec_foreach(nfe, vip->new_flow_table) { - s = format(s, "%U %d: %d\n", format_white_space, indent, nfe - vip->new_flow_table, nfe->as_index); - } - */ - return s; -} - -typedef struct { - u32 as_index; - u32 last; - u32 skip; -} lb_pseudorand_t; - -static int lb_pseudorand_compare(void *a, void *b) -{ - lb_as_t *asa, *asb; - lb_main_t *lbm = &lb_main; - asa = &lbm->ass[((lb_pseudorand_t *)a)->as_index]; - asb = &lbm->ass[((lb_pseudorand_t *)b)->as_index]; - return memcmp(&asa->address, &asb->address, sizeof(asb->address)); -} - -static void lb_vip_garbage_collection(lb_vip_t *vip) -{ - lb_main_t *lbm = &lb_main; - ASSERT (lbm->writer_lock[0]); - - u32 now = (u32) vlib_time_now(vlib_get_main()); - if (!clib_u32_loop_gt(now, vip->last_garbage_collection + LB_GARBAGE_RUN)) - return; - - vip->last_garbage_collection = now; - lb_as_t *as; - u32 *as_index; - pool_foreach(as_index, vip->as_indexes, { - as = &lbm->ass[*as_index]; - if (!(as->flags & LB_AS_FLAGS_USED) && //Not used - clib_u32_loop_gt(now, as->last_used + LB_CONCURRENCY_TIMEOUT) && //Not recently used - (vlib_refcount_get(&lbm->as_refcount, as - lbm->ass) == 0)) - { //Not referenced - fib_entry_child_remove(as->next_hop_fib_entry_index, - as->next_hop_child_index); - fib_table_entry_delete_index(as->next_hop_fib_entry_index, - FIB_SOURCE_RR); - as->next_hop_fib_entry_index = FIB_NODE_INDEX_INVALID; - - pool_put(vip->as_indexes, as_index); - pool_put(lbm->ass, as); - } - }); -} - -void lb_garbage_collection() -{ - lb_main_t *lbm = &lb_main; - lb_get_writer_lock(); - lb_vip_t *vip; - u32 *to_be_removed_vips = 0, *i; - pool_foreach(vip, lbm->vips, { - lb_vip_garbage_collection(vip); - - if (!(vip->flags & LB_VIP_FLAGS_USED) && - (pool_elts(vip->as_indexes) == 0)) { - vec_add1(to_be_removed_vips, vip - lbm->vips); - } - }); - - vec_foreach(i, to_be_removed_vips) { - vip = &lbm->vips[*i]; - pool_put(lbm->vips, vip); - pool_free(vip->as_indexes); - } - - vec_free(to_be_removed_vips); - lb_put_writer_lock(); -} - -static void lb_vip_update_new_flow_table(lb_vip_t *vip) -{ - lb_main_t *lbm = &lb_main; - lb_new_flow_entry_t *old_table; - u32 i, *as_index; - lb_new_flow_entry_t *new_flow_table = 0; - lb_as_t *as; - lb_pseudorand_t *pr, *sort_arr = 0; - u32 count; - - ASSERT (lbm->writer_lock[0]); //We must have the lock - - //Check if some AS is configured or not - i = 0; - pool_foreach(as_index, vip->as_indexes, { - as = &lbm->ass[*as_index]; - if (as->flags & LB_AS_FLAGS_USED) { //Not used anymore - i = 1; - goto out; //Not sure 'break' works in this macro-loop - } - }); - -out: - if (i == 0) { - //Only the default. i.e. no AS - vec_validate(new_flow_table, vip->new_flow_table_mask); - for (i=0; ias_indexes)); - - i = 0; - pool_foreach(as_index, vip->as_indexes, { - as = &lbm->ass[*as_index]; - if (!(as->flags & LB_AS_FLAGS_USED)) //Not used anymore - continue; - - sort_arr[i].as_index = as - lbm->ass; - i++; - }); - _vec_len(sort_arr) = i; - - vec_sort_with_function(sort_arr, lb_pseudorand_compare); - - //Now let's pseudo-randomly generate permutations - vec_foreach(pr, sort_arr) { - lb_as_t *as = &lbm->ass[pr->as_index]; - - u64 seed = clib_xxhash(as->address.as_u64[0] ^ - as->address.as_u64[1]); - /* We have 2^n buckets. - * skip must be prime with 2^n. - * So skip must be odd. - * MagLev actually state that M should be prime, - * but this has a big computation cost (% operation). - * Using 2^n is more better (& operation). - */ - pr->skip = ((seed & 0xffffffff) | 1) & vip->new_flow_table_mask; - pr->last = (seed >> 32) & vip->new_flow_table_mask; - } - - //Let's create a new flow table - vec_validate(new_flow_table, vip->new_flow_table_mask); - for (i=0; ilast; - pr->last = (pr->last + pr->skip) & vip->new_flow_table_mask; - if (new_flow_table[last].as_index == ~0) { - new_flow_table[last].as_index = pr->as_index; - break; - } - } - done++; - if (done == vec_len(new_flow_table)) - goto finished; - } - } - - vec_free(sort_arr); - -finished: - -//Count number of changed entries - count = 0; - for (i=0; inew_flow_table == 0 || - new_flow_table[i].as_index != vip->new_flow_table[i].as_index) - count++; - - old_table = vip->new_flow_table; - vip->new_flow_table = new_flow_table; - vec_free(old_table); -} - -int lb_conf(ip4_address_t *ip4_address, ip6_address_t *ip6_address, - u32 per_cpu_sticky_buckets, u32 flow_timeout) -{ - lb_main_t *lbm = &lb_main; - - if (!is_pow2(per_cpu_sticky_buckets)) - return VNET_API_ERROR_INVALID_MEMORY_SIZE; - - lb_get_writer_lock(); //Not exactly necessary but just a reminder that it exists for my future self - lbm->ip4_src_address = *ip4_address; - lbm->ip6_src_address = *ip6_address; - lbm->per_cpu_sticky_buckets = per_cpu_sticky_buckets; - lbm->flow_timeout = flow_timeout; - lb_put_writer_lock(); - return 0; -} - -static -int lb_vip_find_index_with_lock(ip46_address_t *prefix, u8 plen, u32 *vip_index) -{ - lb_main_t *lbm = &lb_main; - lb_vip_t *vip; - ASSERT (lbm->writer_lock[0]); //This must be called with the lock owned - ip46_prefix_normalize(prefix, plen); - pool_foreach(vip, lbm->vips, { - if ((vip->flags & LB_AS_FLAGS_USED) && - vip->plen == plen && - vip->prefix.as_u64[0] == prefix->as_u64[0] && - vip->prefix.as_u64[1] == prefix->as_u64[1]) { - *vip_index = vip - lbm->vips; - return 0; - } - }); - return VNET_API_ERROR_NO_SUCH_ENTRY; -} - -int lb_vip_find_index(ip46_address_t *prefix, u8 plen, u32 *vip_index) -{ - int ret; - lb_get_writer_lock(); - ret = lb_vip_find_index_with_lock(prefix, plen, vip_index); - lb_put_writer_lock(); - return ret; -} - -static int lb_as_find_index_vip(lb_vip_t *vip, ip46_address_t *address, u32 *as_index) -{ - lb_main_t *lbm = &lb_main; - ASSERT (lbm->writer_lock[0]); //This must be called with the lock owned - lb_as_t *as; - u32 *asi; - pool_foreach(asi, vip->as_indexes, { - as = &lbm->ass[*asi]; - if (as->vip_index == (vip - lbm->vips) && - as->address.as_u64[0] == address->as_u64[0] && - as->address.as_u64[1] == address->as_u64[1]) { - *as_index = as - lbm->ass; - return 0; - } - }); - return -1; -} - -int lb_vip_add_ass(u32 vip_index, ip46_address_t *addresses, u32 n) -{ - lb_main_t *lbm = &lb_main; - lb_get_writer_lock(); - lb_vip_t *vip; - if (!(vip = lb_vip_get_by_index(vip_index))) { - lb_put_writer_lock(); - return VNET_API_ERROR_NO_SUCH_ENTRY; - } - - ip46_type_t type = lb_vip_is_gre4(vip)?IP46_TYPE_IP4:IP46_TYPE_IP6; - u32 *to_be_added = 0; - u32 *to_be_updated = 0; - u32 i; - u32 *ip; - - //Sanity check - while (n--) { - - if (!lb_as_find_index_vip(vip, &addresses[n], &i)) { - if (lbm->ass[i].flags & LB_AS_FLAGS_USED) { - vec_free(to_be_added); - vec_free(to_be_updated); - lb_put_writer_lock(); - return VNET_API_ERROR_VALUE_EXIST; - } - vec_add1(to_be_updated, i); - goto next; - } - - if (ip46_address_type(&addresses[n]) != type) { - vec_free(to_be_added); - vec_free(to_be_updated); - lb_put_writer_lock(); - return VNET_API_ERROR_INVALID_ADDRESS_FAMILY; - } - - if (n) { - u32 n2 = n; - while(n2--) //Check for duplicates - if (addresses[n2].as_u64[0] == addresses[n].as_u64[0] && - addresses[n2].as_u64[1] == addresses[n].as_u64[1]) - goto next; - } - - vec_add1(to_be_added, n); - -next: - continue; - } - - //Update reused ASs - vec_foreach(ip, to_be_updated) { - lbm->ass[*ip].flags = LB_AS_FLAGS_USED; - } - vec_free(to_be_updated); - - //Create those who have to be created - vec_foreach(ip, to_be_added) { - lb_as_t *as; - u32 *as_index; - pool_get(lbm->ass, as); - as->address = addresses[*ip]; - as->flags = LB_AS_FLAGS_USED; - as->vip_index = vip_index; - pool_get(vip->as_indexes, as_index); - *as_index = as - lbm->ass; - - /* - * become a child of the FIB entry - * so we are informed when its forwarding changes - */ - fib_prefix_t nh = {}; - if (lb_vip_is_gre4(vip)) { - nh.fp_addr.ip4 = as->address.ip4; - nh.fp_len = 32; - nh.fp_proto = FIB_PROTOCOL_IP4; - } else { - nh.fp_addr.ip6 = as->address.ip6; - nh.fp_len = 128; - nh.fp_proto = FIB_PROTOCOL_IP6; - } - - as->next_hop_fib_entry_index = - fib_table_entry_special_add(0, - &nh, - FIB_SOURCE_RR, - FIB_ENTRY_FLAG_NONE, - ADJ_INDEX_INVALID); - as->next_hop_child_index = - fib_entry_child_add(as->next_hop_fib_entry_index, - lbm->fib_node_type, - as - lbm->ass); - - lb_as_stack(as); - } - vec_free(to_be_added); - - //Recompute flows - lb_vip_update_new_flow_table(vip); - - //Garbage collection maybe - lb_vip_garbage_collection(vip); - - lb_put_writer_lock(); - return 0; -} - -int lb_vip_del_ass_withlock(u32 vip_index, ip46_address_t *addresses, u32 n) -{ - lb_main_t *lbm = &lb_main; - u32 now = (u32) vlib_time_now(vlib_get_main()); - u32 *ip = 0; - - lb_vip_t *vip; - if (!(vip = lb_vip_get_by_index(vip_index))) { - return VNET_API_ERROR_NO_SUCH_ENTRY; - } - - u32 *indexes = NULL; - while (n--) { - u32 i; - if (lb_as_find_index_vip(vip, &addresses[n], &i)) { - vec_free(indexes); - return VNET_API_ERROR_NO_SUCH_ENTRY; - } - - if (n) { //Check for duplicates - u32 n2 = n - 1; - while(n2--) { - if (addresses[n2].as_u64[0] == addresses[n].as_u64[0] && - addresses[n2].as_u64[1] == addresses[n].as_u64[1]) - goto next; - } - } - - vec_add1(indexes, i); -next: - continue; - } - - //Garbage collection maybe - lb_vip_garbage_collection(vip); - - if (indexes != NULL) { - vec_foreach(ip, indexes) { - lbm->ass[*ip].flags &= ~LB_AS_FLAGS_USED; - lbm->ass[*ip].last_used = now; - } - - //Recompute flows - lb_vip_update_new_flow_table(vip); - } - - vec_free(indexes); - return 0; -} - -int lb_vip_del_ass(u32 vip_index, ip46_address_t *addresses, u32 n) -{ - lb_get_writer_lock(); - int ret = lb_vip_del_ass_withlock(vip_index, addresses, n); - lb_put_writer_lock(); - return ret; -} - -/** - * Add the VIP adjacency to the ip4 or ip6 fib - */ -static void lb_vip_add_adjacency(lb_main_t *lbm, lb_vip_t *vip) -{ - dpo_proto_t proto = 0; - dpo_id_t dpo = DPO_INVALID; - fib_prefix_t pfx = {}; - if (lb_vip_is_ip4(vip)) { - pfx.fp_addr.ip4 = vip->prefix.ip4; - pfx.fp_len = vip->plen - 96; - pfx.fp_proto = FIB_PROTOCOL_IP4; - proto = DPO_PROTO_IP4; - } else { - pfx.fp_addr.ip6 = vip->prefix.ip6; - pfx.fp_len = vip->plen; - pfx.fp_proto = FIB_PROTOCOL_IP6; - proto = DPO_PROTO_IP6; - } - dpo_set(&dpo, lb_vip_is_gre4(vip)?lbm->dpo_gre4_type:lbm->dpo_gre6_type, - proto, vip - lbm->vips); - fib_table_entry_special_dpo_add(0, - &pfx, - FIB_SOURCE_PLUGIN_HI, - FIB_ENTRY_FLAG_EXCLUSIVE, - &dpo); - dpo_reset(&dpo); -} - -/** - * Deletes the adjacency associated with the VIP - */ -static void lb_vip_del_adjacency(lb_main_t *lbm, lb_vip_t *vip) -{ - fib_prefix_t pfx = {}; - if (lb_vip_is_ip4(vip)) { - pfx.fp_addr.ip4 = vip->prefix.ip4; - pfx.fp_len = vip->plen - 96; - pfx.fp_proto = FIB_PROTOCOL_IP4; - } else { - pfx.fp_addr.ip6 = vip->prefix.ip6; - pfx.fp_len = vip->plen; - pfx.fp_proto = FIB_PROTOCOL_IP6; - } - fib_table_entry_special_remove(0, &pfx, FIB_SOURCE_PLUGIN_HI); -} - -int lb_vip_add(ip46_address_t *prefix, u8 plen, lb_vip_type_t type, u32 new_length, u32 *vip_index) -{ - lb_main_t *lbm = &lb_main; - lb_vip_t *vip; - lb_get_writer_lock(); - ip46_prefix_normalize(prefix, plen); - - if (!lb_vip_find_index_with_lock(prefix, plen, vip_index)) { - lb_put_writer_lock(); - return VNET_API_ERROR_VALUE_EXIST; - } - - if (!is_pow2(new_length)) { - lb_put_writer_lock(); - return VNET_API_ERROR_INVALID_MEMORY_SIZE; - } - - if (ip46_prefix_is_ip4(prefix, plen) && - (type != LB_VIP_TYPE_IP4_GRE4) && - (type != LB_VIP_TYPE_IP4_GRE6)) - return VNET_API_ERROR_INVALID_ADDRESS_FAMILY; - - - //Allocate - pool_get(lbm->vips, vip); - - //Init - vip->prefix = *prefix; - vip->plen = plen; - vip->last_garbage_collection = (u32) vlib_time_now(vlib_get_main()); - vip->type = type; - vip->flags = LB_VIP_FLAGS_USED; - vip->as_indexes = 0; - - //Validate counters - u32 i; - for (i = 0; i < LB_N_VIP_COUNTERS; i++) { - vlib_validate_simple_counter(&lbm->vip_counters[i], vip - lbm->vips); - vlib_zero_simple_counter(&lbm->vip_counters[i], vip - lbm->vips); - } - - //Configure new flow table - vip->new_flow_table_mask = new_length - 1; - vip->new_flow_table = 0; - - //Create a new flow hash table full of the default entry - lb_vip_update_new_flow_table(vip); - - //Create adjacency to direct traffic - lb_vip_add_adjacency(lbm, vip); - - //Return result - *vip_index = vip - lbm->vips; - - lb_put_writer_lock(); - return 0; -} - -int lb_vip_del(u32 vip_index) -{ - lb_main_t *lbm = &lb_main; - lb_vip_t *vip; - lb_get_writer_lock(); - if (!(vip = lb_vip_get_by_index(vip_index))) { - lb_put_writer_lock(); - return VNET_API_ERROR_NO_SUCH_ENTRY; - } - - //FIXME: This operation is actually not working - //We will need to remove state before performing this. - - { - //Remove all ASs - ip46_address_t *ass = 0; - lb_as_t *as; - u32 *as_index; - pool_foreach(as_index, vip->as_indexes, { - as = &lbm->ass[*as_index]; - vec_add1(ass, as->address); - }); - if (vec_len(ass)) - lb_vip_del_ass_withlock(vip_index, ass, vec_len(ass)); - vec_free(ass); - } - - //Delete adjacency - lb_vip_del_adjacency(lbm, vip); - - //Set the VIP as unused - vip->flags &= ~LB_VIP_FLAGS_USED; - - lb_put_writer_lock(); - return 0; -} - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, - vnet_plugin_handoff_t * h, - int from_early_init) -{ - clib_error_t *error = 0; - return error; -} - - -u8 *format_lb_dpo (u8 * s, va_list * va) -{ - index_t index = va_arg (*va, index_t); - CLIB_UNUSED(u32 indent) = va_arg (*va, u32); - lb_main_t *lbm = &lb_main; - lb_vip_t *vip = pool_elt_at_index (lbm->vips, index); - return format (s, "%U", format_lb_vip, vip); -} - -static void lb_dpo_lock (dpo_id_t *dpo) {} -static void lb_dpo_unlock (dpo_id_t *dpo) {} - -static fib_node_t * -lb_fib_node_get_node (fib_node_index_t index) -{ - lb_main_t *lbm = &lb_main; - lb_as_t *as = pool_elt_at_index (lbm->ass, index); - return (&as->fib_node); -} - -static void -lb_fib_node_last_lock_gone (fib_node_t *node) -{ -} - -static lb_as_t * -lb_as_from_fib_node (fib_node_t *node) -{ - return ((lb_as_t*)(((char*)node) - - STRUCT_OFFSET_OF(lb_as_t, fib_node))); -} - -static void -lb_as_stack (lb_as_t *as) -{ - lb_main_t *lbm = &lb_main; - lb_vip_t *vip = &lbm->vips[as->vip_index]; - dpo_stack(lb_vip_is_gre4(vip)?lbm->dpo_gre4_type:lbm->dpo_gre6_type, - lb_vip_is_ip4(vip)?DPO_PROTO_IP4:DPO_PROTO_IP6, - &as->dpo, - fib_entry_contribute_ip_forwarding( - as->next_hop_fib_entry_index)); -} - -static fib_node_back_walk_rc_t -lb_fib_node_back_walk_notify (fib_node_t *node, - fib_node_back_walk_ctx_t *ctx) -{ - lb_as_stack(lb_as_from_fib_node(node)); - return (FIB_NODE_BACK_WALK_CONTINUE); -} - -clib_error_t * -lb_init (vlib_main_t * vm) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - lb_main_t *lbm = &lb_main; - lb_as_t *default_as; - fib_node_vft_t lb_fib_node_vft = { - .fnv_get = lb_fib_node_get_node, - .fnv_last_lock = lb_fib_node_last_lock_gone, - .fnv_back_walk = lb_fib_node_back_walk_notify, - }; - dpo_vft_t lb_vft = { - .dv_lock = lb_dpo_lock, - .dv_unlock = lb_dpo_unlock, - .dv_format = format_lb_dpo, - }; - - lbm->vips = 0; - lbm->per_cpu = 0; - vec_validate(lbm->per_cpu, tm->n_vlib_mains - 1); - lbm->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); - lbm->writer_lock[0] = 0; - lbm->per_cpu_sticky_buckets = LB_DEFAULT_PER_CPU_STICKY_BUCKETS; - lbm->flow_timeout = LB_DEFAULT_FLOW_TIMEOUT; - lbm->ip4_src_address.as_u32 = 0xffffffff; - lbm->ip6_src_address.as_u64[0] = 0xffffffffffffffffL; - lbm->ip6_src_address.as_u64[1] = 0xffffffffffffffffL; - lbm->dpo_gre4_type = dpo_register_new_type(&lb_vft, lb_dpo_gre4_nodes); - lbm->dpo_gre6_type = dpo_register_new_type(&lb_vft, lb_dpo_gre6_nodes); - lbm->fib_node_type = fib_node_register_new_type(&lb_fib_node_vft); - - //Init AS reference counters - vlib_refcount_init(&lbm->as_refcount); - - //Allocate and init default AS. - lbm->ass = 0; - pool_get(lbm->ass, default_as); - default_as->flags = 0; - default_as->dpo.dpoi_next_node = LB_NEXT_DROP; - default_as->vip_index = ~0; - default_as->address.ip6.as_u64[0] = 0xffffffffffffffffL; - default_as->address.ip6.as_u64[1] = 0xffffffffffffffffL; - -#define _(a,b,c) lbm->vip_counters[c].name = b; - lb_foreach_vip_counter -#undef _ - return NULL; -} - -VLIB_INIT_FUNCTION (lb_init); diff --git a/plugins/lb-plugin/lb/lb.h b/plugins/lb-plugin/lb/lb.h deleted file mode 100644 index 882b9b30..00000000 --- a/plugins/lb-plugin/lb/lb.h +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * lb-plugin implements a MagLev-like load balancer. - * http://research.google.com/pubs/pub44824.html - * - * It hasn't been tested for interoperability with the original MagLev - * but intends to provide similar functionality. - * The load-balancer receives traffic destined to VIP (Virtual IP) - * addresses from one or multiple(ECMP) routers. - * The load-balancer tunnels the traffic toward many application servers - * ensuring session stickyness (i.e. that a single sessions is tunneled - * towards a single application server). - * - */ - -#ifndef LB_PLUGIN_LB_LB_H_ -#define LB_PLUGIN_LB_LB_H_ - -#include -#include - -#include -#include -#include -#include - -#include - -#define LB_DEFAULT_PER_CPU_STICKY_BUCKETS 1 << 10 -#define LB_DEFAULT_FLOW_TIMEOUT 40 - -typedef enum { - LB_NEXT_DROP, - LB_N_NEXT, -} lb_next_t; - -/** - * Each VIP is configured with a set of - * application server. - */ -typedef struct { - /** - * Registration to FIB event. - */ - fib_node_t fib_node; - - /** - * Destination address used to tunnel traffic towards - * that application server. - * The address is also used as ID and pseudo-random - * seed for the load-balancing process. - */ - ip46_address_t address; - - /** - * ASs are indexed by address and VIP Index. - * Which means there will be duplicated if the same server - * address is used for multiple VIPs. - */ - u32 vip_index; - - /** - * Some per-AS flags. - * For now only LB_AS_FLAGS_USED is defined. - */ - u8 flags; - -#define LB_AS_FLAGS_USED 0x1 - - /** - * Rotating timestamp of when LB_AS_FLAGS_USED flag was last set. - * - * AS removal is based on garbage collection and reference counting. - * When an AS is removed, there is a race between configuration core - * and worker cores which may still add a reference while it should not - * be used. This timestamp is used to not remove the AS while a race condition - * may happen. - */ - u32 last_used; - - /** - * The FIB entry index for the next-hop - */ - fib_node_index_t next_hop_fib_entry_index; - - /** - * The child index on the FIB entry - */ - u32 next_hop_child_index; - - /** - * The next DPO in the graph to follow. - */ - dpo_id_t dpo; - -} lb_as_t; - -format_function_t format_lb_as; - -typedef struct { - u32 as_index; -} lb_new_flow_entry_t; - -#define lb_foreach_vip_counter \ - _(NEXT_PACKET, "packet from existing sessions", 0) \ - _(FIRST_PACKET, "first session packet", 1) \ - _(UNTRACKED_PACKET, "untracked packet", 2) \ - _(NO_SERVER, "no server configured", 3) - -typedef enum { -#define _(a,b,c) LB_VIP_COUNTER_##a = c, - lb_foreach_vip_counter -#undef _ - LB_N_VIP_COUNTERS -} lb_vip_counter_t; - -/** - * The load balancer supports IPv4 and IPv6 traffic - * and GRE4 and GRE6 encap. - */ -typedef enum { - LB_VIP_TYPE_IP6_GRE6, - LB_VIP_TYPE_IP6_GRE4, - LB_VIP_TYPE_IP4_GRE6, - LB_VIP_TYPE_IP4_GRE4, - LB_VIP_N_TYPES, -} lb_vip_type_t; - -format_function_t format_lb_vip_type; -unformat_function_t unformat_lb_vip_type; - -/** - * Load balancing service is provided per VIP. - * In this data model, a VIP can be a whole prefix. - * But load balancing only - * occurs on a per-source-address/port basis. Meaning that if a given source - * reuses the same port for multiple destinations within the same VIP, - * they will be considered as a single flow. - */ -typedef struct { - - //Runtime - - /** - * Vector mapping (flow-hash & new_connect_table_mask) to AS index. - * This is used for new flows. - */ - lb_new_flow_entry_t *new_flow_table; - - /** - * New flows table length - 1 - * (length MUST be a power of 2) - */ - u32 new_flow_table_mask; - - /** - * Last time garbage collection was run to free the ASs. - */ - u32 last_garbage_collection; - - //Not runtime - - /** - * A Virtual IP represents a given service delivered - * by a set of application servers. It can be a single - * address or a prefix. - * IPv4 prefixes are encoded using IPv4-in-IPv6 embedded address - * (i.e. ::/96 prefix). - */ - ip46_address_t prefix; - - /** - * The VIP prefix length. - * In case of IPv4, plen = 96 + ip4_plen. - */ - u8 plen; - - /** - * The type of traffic for this. - * LB_TYPE_UNDEFINED if unknown. - */ - lb_vip_type_t type; - - /** - * Flags related to this VIP. - * LB_VIP_FLAGS_USED means the VIP is active. - * When it is not set, the VIP in the process of being removed. - * We cannot immediately remove a VIP because the VIP index still may be stored - * in the adjacency index. - */ - u8 flags; -#define LB_VIP_FLAGS_USED 0x1 - - /** - * Pool of AS indexes used for this VIP. - * This also includes ASs that have been removed (but are still referenced). - */ - u32 *as_indexes; -} lb_vip_t; - -#define lb_vip_is_ip4(vip) ((vip)->type == LB_VIP_TYPE_IP4_GRE6 || (vip)->type == LB_VIP_TYPE_IP4_GRE4) -#define lb_vip_is_gre4(vip) ((vip)->type == LB_VIP_TYPE_IP6_GRE4 || (vip)->type == LB_VIP_TYPE_IP4_GRE4) -format_function_t format_lb_vip; -format_function_t format_lb_vip_detailed; - -typedef struct { - /** - * Each CPU has its own sticky flow hash table. - * One single table is used for all VIPs. - */ - lb_hash_t *sticky_ht; -} lb_per_cpu_t; - -typedef struct { - /** - * Pool of all Virtual IPs - */ - lb_vip_t *vips; - - /** - * Pool of ASs. - * ASs are referenced by address and vip index. - * The first element (index 0) is special and used only to fill - * new_flow_tables when no AS has been configured. - */ - lb_as_t *ass; - - /** - * Each AS has an associated reference counter. - * As ass[0] has a special meaning, its associated counter - * starts at 0 and is decremented instead. i.e. do not use it. - */ - vlib_refcount_t as_refcount; - - /** - * Some global data is per-cpu - */ - lb_per_cpu_t *per_cpu; - - /** - * Node next index for IP adjacencies, for each of the traffic types. - */ - u32 ip_lookup_next_index[LB_VIP_N_TYPES]; - - /** - * Source address used in IPv6 encapsulated traffic - */ - ip6_address_t ip6_src_address; - - /** - * Source address used for IPv4 encapsulated traffic - */ - ip4_address_t ip4_src_address; - - /** - * Number of buckets in the per-cpu sticky hash table. - */ - u32 per_cpu_sticky_buckets; - - /** - * Flow timeout in seconds. - */ - u32 flow_timeout; - - /** - * Per VIP counter - */ - vlib_simple_counter_main_t vip_counters[LB_N_VIP_COUNTERS]; - - /** - * DPO used to send packet from IP4/6 lookup to LB node. - */ - dpo_type_t dpo_gre4_type; - dpo_type_t dpo_gre6_type; - - /** - * Node type for registering to fib changes. - */ - fib_node_type_t fib_node_type; - - /** - * API dynamically registered base ID. - */ - u16 msg_id_base; - - volatile u32 *writer_lock; -} lb_main_t; - -extern lb_main_t lb_main; -extern vlib_node_registration_t lb6_node; -extern vlib_node_registration_t lb4_node; - -/** - * Fix global load-balancer parameters. - * @param ip4_address IPv4 source address used for encapsulated traffic - * @param ip6_address IPv6 source address used for encapsulated traffic - * @return 0 on success. VNET_LB_ERR_XXX on error - */ -int lb_conf(ip4_address_t *ip4_address, ip6_address_t *ip6_address, - u32 sticky_buckets, u32 flow_timeout); - -int lb_vip_add(ip46_address_t *prefix, u8 plen, lb_vip_type_t type, - u32 new_length, u32 *vip_index); -int lb_vip_del(u32 vip_index); - -int lb_vip_find_index(ip46_address_t *prefix, u8 plen, u32 *vip_index); - -#define lb_vip_get_by_index(index) (pool_is_free_index(lb_main.vips, index)?NULL:pool_elt_at_index(lb_main.vips, index)) - -int lb_vip_add_ass(u32 vip_index, ip46_address_t *addresses, u32 n); -int lb_vip_del_ass(u32 vip_index, ip46_address_t *addresses, u32 n); - -u32 lb_hash_time_now(vlib_main_t * vm); - -void lb_garbage_collection(); - -format_function_t format_lb_main; - -#endif /* LB_PLUGIN_LB_LB_H_ */ diff --git a/plugins/lb-plugin/lb/lb_test.c b/plugins/lb-plugin/lb/lb_test.c deleted file mode 100644 index 8c2eaa91..00000000 --- a/plugins/lb-plugin/lb/lb_test.c +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -//TODO: Move that to vat/plugin_api.c -////////////////////////// -uword unformat_ip46_address (unformat_input_t * input, va_list * args) -{ - ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); - ip46_type_t type = va_arg (*args, ip46_type_t); - if ((type != IP46_TYPE_IP6) && - unformat(input, "%U", unformat_ip4_address, &ip46->ip4)) { - ip46_address_mask_ip4(ip46); - return 1; - } else if ((type != IP46_TYPE_IP4) && - unformat(input, "%U", unformat_ip6_address, &ip46->ip6)) { - return 1; - } - return 0; -} -uword unformat_ip46_prefix (unformat_input_t * input, va_list * args) -{ - ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); - u8 *len = va_arg (*args, u8 *); - ip46_type_t type = va_arg (*args, ip46_type_t); - - u32 l; - if ((type != IP46_TYPE_IP6) && unformat(input, "%U/%u", unformat_ip4_address, &ip46->ip4, &l)) { - if (l > 32) - return 0; - *len = l + 96; - ip46->pad[0] = ip46->pad[1] = ip46->pad[2] = 0; - } else if ((type != IP46_TYPE_IP4) && unformat(input, "%U/%u", unformat_ip6_address, &ip46->ip6, &l)) { - if (l > 128) - return 0; - *len = l; - } else { - return 0; - } - return 1; -} -///////////////////////// - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} lb_test_main_t; - -lb_test_main_t lb_test_main; - -#define foreach_standard_reply_retval_handler \ -_(lb_conf_reply) \ -_(lb_add_del_vip_reply) \ -_(lb_add_del_as_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = lb_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ - _(LB_CONF_REPLY, lb_conf_reply) \ - _(LB_ADD_DEL_VIP_REPLY, lb_add_del_vip_reply) \ - _(LB_ADD_DEL_AS_REPLY, lb_add_del_as_reply) - -/* M: construct, but don't yet send a message */ -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memcpy (mp, &mps, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + lbtm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - -static int api_lb_conf (vat_main_t * vam) -{ - lb_test_main_t *lbtm = &lb_test_main; - unformat_input_t *i = vam->input; - f64 timeout; - vl_api_lb_conf_t mps, *mp; - - if (!unformat(i, "%U %U %u %u", - unformat_ip4_address, &mps.ip4_src_address, - unformat_ip6_address, mps.ip6_src_address, - &mps.sticky_buckets_per_core, - &mps.flow_timeout)) { - errmsg ("invalid arguments\n"); - return -99; - } - - M(LB_CONF, lb_conf); S; W; - - /* NOTREACHED */ - return 0; -} - -static int api_lb_add_del_vip (vat_main_t * vam) -{ - lb_test_main_t *lbtm = &lb_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_lb_add_del_vip_t mps, *mp; - mps.is_del = 0; - mps.is_gre4 = 0; - - if (!unformat(i, "%U", - unformat_ip46_prefix, mps.ip_prefix, &mps.prefix_length, IP46_TYPE_ANY)) { - errmsg ("invalid prefix\n"); - return -99; - } - - if (unformat(i, "gre4")) { - mps.is_gre4 = 1; - } else if (unformat(i, "gre6")) { - mps.is_gre4 = 0; - } else { - errmsg ("no encap\n"); - return -99; - } - - if (!unformat(i, "%d", &mps.new_flows_table_length)) { - errmsg ("no table lentgh\n"); - return -99; - } - - if (unformat(i, "del")) { - mps.is_del = 1; - } - - M(LB_ADD_DEL_VIP, lb_add_del_vip); S; W; - /* NOTREACHED */ - return 0; -} - -static int api_lb_add_del_as (vat_main_t * vam) -{ - lb_test_main_t *lbtm = &lb_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_lb_add_del_as_t mps, *mp; - mps.is_del = 0; - - if (!unformat(i, "%U %U", - unformat_ip46_prefix, mps.vip_ip_prefix, &mps.vip_prefix_length, IP46_TYPE_ANY, - unformat_ip46_address, mps.as_address)) { - errmsg ("invalid prefix or address\n"); - return -99; - } - - if (unformat(i, "del")) { - mps.is_del = 1; - } - - M(LB_ADD_DEL_AS, lb_add_del_as); S; W; - /* NOTREACHED */ - return 0; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(lb_conf, " ") \ -_(lb_add_del_vip, " [gre4|gre6] [del]") \ -_(lb_add_del_as, "
[del]") - -void vat_api_hookup (vat_main_t *vam) -{ - lb_test_main_t * lbtm = &lb_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + lbtm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * vat_plugin_register (vat_main_t *vam) -{ - lb_test_main_t * lbtm = &lb_test_main; - - u8 * name; - - lbtm->vat_main = vam; - - /* Ask the vpp engine for the first assigned message-id */ - name = format (0, "lb_%08x%c", api_version, 0); - lbtm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (lbtm->msg_id_base != (u16) ~0) - vat_api_hookup (vam); - - vec_free(name); - - return 0; -} diff --git a/plugins/lb-plugin/lb/lbhash.h b/plugins/lb-plugin/lb/lbhash.h deleted file mode 100644 index ca3cc143..00000000 --- a/plugins/lb-plugin/lb/lbhash.h +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (c) 2012 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * vppinfra already includes tons of different hash tables. - * MagLev flow table is a bit different. It has to be very efficient - * for both writing and reading operations. But it does not need to - * be 100% reliable (write can fail). It also needs to recycle - * old entries in a lazy way. - * - * This hash table is the most dummy hash table you can do. - * Fixed total size, fixed bucket size. - * Advantage is that it could be very efficient (maybe). - * - */ - -#ifndef LB_PLUGIN_LB_LBHASH_H_ -#define LB_PLUGIN_LB_LBHASH_H_ - -#include - -#if defined (__SSE4_2__) -#include -#endif - -/* - * @brief Number of entries per bucket. - */ -#define LBHASH_ENTRY_PER_BUCKET 4 - -#define LB_HASH_DO_NOT_USE_SSE_BUCKETS 0 - -/* - * @brief One bucket contains 4 entries. - * Each bucket takes one 64B cache line in memory. - */ -typedef struct { - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u32 hash[LBHASH_ENTRY_PER_BUCKET]; - u32 timeout[LBHASH_ENTRY_PER_BUCKET]; - u32 vip[LBHASH_ENTRY_PER_BUCKET]; - u32 value[LBHASH_ENTRY_PER_BUCKET]; -} lb_hash_bucket_t; - -typedef struct { - u32 buckets_mask; - u32 timeout; - lb_hash_bucket_t buckets[]; -} lb_hash_t; - -#define lb_hash_nbuckets(h) (((h)->buckets_mask) + 1) -#define lb_hash_size(h) ((h)->buckets_mask + LBHASH_ENTRY_PER_BUCKET) - -#define lb_hash_foreach_bucket(h, bucket) \ - for (bucket = (h)->buckets; \ - bucket < (h)->buckets + lb_hash_nbuckets(h); \ - bucket++) - -#define lb_hash_foreach_entry(h, bucket, i) \ - lb_hash_foreach_bucket(h, bucket) \ - for (i = 0; i < LBHASH_ENTRY_PER_BUCKET; i++) - -#define lb_hash_foreach_valid_entry(h, bucket, i, now) \ - lb_hash_foreach_entry(h, bucket, i) \ - if (!clib_u32_loop_gt((now), bucket->timeout[i])) - -static_always_inline -lb_hash_t *lb_hash_alloc(u32 buckets, u32 timeout) -{ - if (!is_pow2(buckets)) - return NULL; - - // Allocate 1 more bucket for prefetch - u32 size = ((u64)&((lb_hash_t *)(0))->buckets[0]) + - sizeof(lb_hash_bucket_t) * (buckets + 1); - u8 *mem = 0; - lb_hash_t *h; - vec_alloc_aligned(mem, size, CLIB_CACHE_LINE_BYTES); - h = (lb_hash_t *)mem; - h->buckets_mask = (buckets - 1); - h->timeout = timeout; - return h; -} - -static_always_inline -void lb_hash_free(lb_hash_t *h) -{ - u8 *mem = (u8 *)h; - vec_free(mem); -} - -#if __SSE4_2__ -static_always_inline -u32 lb_hash_hash(u64 k0, u64 k1, u64 k2, u64 k3, u64 k4) -{ - u64 val = 0; - val = _mm_crc32_u64(val, k0); - val = _mm_crc32_u64(val, k1); - val = _mm_crc32_u64(val, k2); - val = _mm_crc32_u64(val, k3); - val = _mm_crc32_u64(val, k4); - return (u32) val; -} -#else -static_always_inline -u32 lb_hash_hash(u64 k0, u64 k1, u64 k2, u64 k3, u64 k4) -{ - u64 tmp = k0 ^ k1 ^ k2 ^ k3 ^ k4; - return (u32)clib_xxhash (tmp); -} -#endif - -static_always_inline -void lb_hash_prefetch_bucket(lb_hash_t *ht, u32 hash) -{ - lb_hash_bucket_t *bucket = &ht->buckets[hash & ht->buckets_mask]; - CLIB_PREFETCH(bucket, sizeof(*bucket), READ); -} - -static_always_inline -void lb_hash_get(lb_hash_t *ht, u32 hash, u32 vip, u32 time_now, - u32 *available_index, u32 *found_value) -{ - lb_hash_bucket_t *bucket = &ht->buckets[hash & ht->buckets_mask]; - *found_value = ~0; - *available_index = ~0; -#if __SSE4_2__ && LB_HASH_DO_NOT_USE_SSE_BUCKETS == 0 - u32 bitmask, found_index; - __m128i mask; - - // mask[*] = timeout[*] > now - mask = _mm_cmpgt_epi32(_mm_loadu_si128 ((__m128i *) bucket->timeout), - _mm_set1_epi32 (time_now)); - // bitmask[*] = now <= timeout[*/4] - bitmask = (~_mm_movemask_epi8(mask)) & 0xffff; - // Get first index with now <= timeout[*], if any. - *available_index = (bitmask)?__builtin_ctz(bitmask)/4:*available_index; - - // mask[*] = (timeout[*] > now) && (hash[*] == hash) - mask = _mm_and_si128(mask, - _mm_cmpeq_epi32( - _mm_loadu_si128 ((__m128i *) bucket->hash), - _mm_set1_epi32 (hash))); - - // Load the array of vip values - // mask[*] = (timeout[*] > now) && (hash[*] == hash) && (vip[*] == vip) - mask = _mm_and_si128(mask, - _mm_cmpeq_epi32( - _mm_loadu_si128 ((__m128i *) bucket->vip), - _mm_set1_epi32 (vip))); - - // mask[*] = (timeout[*x4] > now) && (hash[*x4] == hash) && (vip[*x4] == vip) - bitmask = _mm_movemask_epi8(mask); - // Get first index, if any - found_index = (bitmask)?__builtin_ctzll(bitmask)/4:0; - ASSERT(found_index < 4); - *found_value = (bitmask)?bucket->value[found_index]:*found_value; - bucket->timeout[found_index] = - (bitmask)?time_now + ht->timeout:bucket->timeout[found_index]; -#else - u32 i; - for (i = 0; i < LBHASH_ENTRY_PER_BUCKET; i++) { - u8 cmp = (bucket->hash[i] == hash && bucket->vip[i] == vip); - u8 timeouted = clib_u32_loop_gt(time_now, bucket->timeout[i]); - *found_value = (cmp || timeouted)?*found_value:bucket->value[i]; - bucket->timeout[i] = (cmp || timeouted)?time_now + ht->timeout:bucket->timeout[i]; - *available_index = (timeouted && (*available_index == ~0))?i:*available_index; - - if (!cmp) - return; - } -#endif -} - -static_always_inline -u32 lb_hash_available_value(lb_hash_t *h, u32 hash, u32 available_index) -{ - return h->buckets[hash & h->buckets_mask].value[available_index]; -} - -static_always_inline -void lb_hash_put(lb_hash_t *h, u32 hash, u32 value, u32 vip, - u32 available_index, u32 time_now) -{ - lb_hash_bucket_t *bucket = &h->buckets[hash & h->buckets_mask]; - bucket->hash[available_index] = hash; - bucket->value[available_index] = value; - bucket->timeout[available_index] = time_now + h->timeout; - bucket->vip[available_index] = vip; -} - -static_always_inline -u32 lb_hash_elts(lb_hash_t *h, u32 time_now) -{ - u32 tot = 0; - lb_hash_bucket_t *bucket; - u32 i; - lb_hash_foreach_valid_entry(h, bucket, i, time_now) { - tot++; - } - return tot; -} - -#endif /* LB_PLUGIN_LB_LBHASH_H_ */ diff --git a/plugins/lb-plugin/lb/node.c b/plugins/lb-plugin/lb/node.c deleted file mode 100644 index 8b763c53..00000000 --- a/plugins/lb-plugin/lb/node.c +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -#define foreach_lb_error \ - _(NONE, "no error") \ - _(PROTO_NOT_SUPPORTED, "protocol not supported") - -typedef enum { -#define _(sym,str) LB_ERROR_##sym, - foreach_lb_error -#undef _ - LB_N_ERROR, -} lb_error_t; - -static char *lb_error_strings[] = { -#define _(sym,string) string, - foreach_lb_error -#undef _ -}; - -typedef struct { - u32 vip_index; - u32 as_index; -} lb_trace_t; - -u8 * -format_lb_trace (u8 * s, va_list * args) -{ - lb_main_t *lbm = &lb_main; - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - lb_trace_t *t = va_arg (*args, lb_trace_t *); - if (pool_is_free_index(lbm->vips, t->vip_index)) { - s = format(s, "lb vip[%d]: This VIP was freed since capture\n"); - } else { - s = format(s, "lb vip[%d]: %U\n", t->vip_index, format_lb_vip, &lbm->vips[t->vip_index]); - } - if (pool_is_free_index(lbm->ass, t->as_index)) { - s = format(s, "lb as[%d]: This AS was freed since capture\n"); - } else { - s = format(s, "lb as[%d]: %U\n", t->as_index, format_lb_as, &lbm->ass[t->as_index]); - } - return s; -} - -lb_hash_t *lb_get_sticky_table(u32 cpu_index) -{ - lb_main_t *lbm = &lb_main; - lb_hash_t *sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; - //Check if size changed - if (PREDICT_FALSE(sticky_ht && (lbm->per_cpu_sticky_buckets != lb_hash_nbuckets(sticky_ht)))) - { - //Dereference everything in there - lb_hash_bucket_t *b; - u32 i; - lb_hash_foreach_entry(sticky_ht, b, i) { - vlib_refcount_add(&lbm->as_refcount, cpu_index, b->value[i], -1); - vlib_refcount_add(&lbm->as_refcount, cpu_index, 0, 1); - } - - lb_hash_free(sticky_ht); - sticky_ht = NULL; - } - - //Create if necessary - if (PREDICT_FALSE(sticky_ht == NULL)) { - lbm->per_cpu[cpu_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout); - sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; - clib_warning("Regenerated sticky table %p", sticky_ht); - } - - ASSERT(sticky_ht); - - //Update timeout - sticky_ht->timeout = lbm->flow_timeout; - return sticky_ht; -} - -u64 -lb_node_get_other_ports4(ip4_header_t *ip40) -{ - return 0; -} - -u64 -lb_node_get_other_ports6(ip6_header_t *ip60) -{ - return 0; -} - -static_always_inline u32 -lb_node_get_hash(vlib_buffer_t *p, u8 is_input_v4) -{ - u32 hash; - if (is_input_v4) - { - ip4_header_t *ip40; - u64 ports; - ip40 = vlib_buffer_get_current (p); - if (PREDICT_TRUE (ip40->protocol == IP_PROTOCOL_TCP || - ip40->protocol == IP_PROTOCOL_UDP)) - ports = ((u64)((udp_header_t *)(ip40 + 1))->src_port << 16) | - ((u64)((udp_header_t *)(ip40 + 1))->dst_port); - else - ports = lb_node_get_other_ports4(ip40); - - hash = lb_hash_hash(*((u64 *)&ip40->address_pair), ports, - 0, 0, 0); - } - else - { - ip6_header_t *ip60; - ip60 = vlib_buffer_get_current (p); - u64 ports; - if (PREDICT_TRUE (ip60->protocol == IP_PROTOCOL_TCP || - ip60->protocol == IP_PROTOCOL_UDP)) - ports = ((u64)((udp_header_t *)(ip60 + 1))->src_port << 16) | - ((u64)((udp_header_t *)(ip60 + 1))->dst_port); - else - ports = lb_node_get_other_ports6(ip60); - - hash = lb_hash_hash(ip60->src_address.as_u64[0], - ip60->src_address.as_u64[1], - ip60->dst_address.as_u64[0], - ip60->dst_address.as_u64[1], - ports); - } - return hash; -} - -static_always_inline uword -lb_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame, - u8 is_input_v4, //Compile-time parameter stating that is input is v4 (or v6) - u8 is_encap_v4) //Compile-time parameter stating that is GRE encap is v4 (or v6) -{ - lb_main_t *lbm = &lb_main; - u32 n_left_from, *from, next_index, *to_next, n_left_to_next; - u32 cpu_index = os_get_cpu_number(); - u32 lb_time = lb_hash_time_now(vm); - - lb_hash_t *sticky_ht = lb_get_sticky_table(cpu_index); - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - u32 nexthash0 = 0; - if (PREDICT_TRUE(n_left_from > 0)) - nexthash0 = lb_node_get_hash(vlib_get_buffer (vm, from[0]), is_input_v4); - - while (n_left_from > 0) - { - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 pi0; - vlib_buffer_t *p0; - lb_vip_t *vip0; - u32 asindex0; - u16 len0; - u32 available_index0; - u8 counter = 0; - u32 hash0 = nexthash0; - - if (PREDICT_TRUE(n_left_from > 1)) - { - vlib_buffer_t *p1 = vlib_get_buffer (vm, from[1]); - //Compute next hash and prefetch bucket - nexthash0 = lb_node_get_hash(p1, is_input_v4); - lb_hash_prefetch_bucket(sticky_ht, nexthash0); - //Prefetch for encap, next - CLIB_PREFETCH (vlib_buffer_get_current(p1) - 64, 64, STORE); - } - - if (PREDICT_TRUE(n_left_from > 2)) - { - vlib_buffer_t *p2; - p2 = vlib_get_buffer(vm, from[2]); - /* prefetch packet header and data */ - vlib_prefetch_buffer_header(p2, STORE); - CLIB_PREFETCH (vlib_buffer_get_current(p2), 64, STORE); - } - - pi0 = to_next[0] = from[0]; - from += 1; - n_left_from -= 1; - to_next += 1; - n_left_to_next -= 1; - - p0 = vlib_get_buffer (vm, pi0); - vip0 = pool_elt_at_index (lbm->vips, - vnet_buffer (p0)->ip.adj_index[VLIB_TX]); - - if (is_input_v4) - { - ip4_header_t *ip40; - ip40 = vlib_buffer_get_current (p0); - len0 = clib_net_to_host_u16(ip40->length); - } - else - { - ip6_header_t *ip60; - ip60 = vlib_buffer_get_current (p0); - len0 = clib_net_to_host_u16(ip60->payload_length) + sizeof(ip6_header_t); - } - - lb_hash_get(sticky_ht, hash0, vnet_buffer (p0)->ip.adj_index[VLIB_TX], - lb_time, &available_index0, &asindex0); - - if (PREDICT_TRUE(asindex0 != ~0)) - { - //Found an existing entry - counter = LB_VIP_COUNTER_NEXT_PACKET; - } - else if (PREDICT_TRUE(available_index0 != ~0)) - { - //There is an available slot for a new flow - asindex0 = vip0->new_flow_table[hash0 & vip0->new_flow_table_mask].as_index; - counter = LB_VIP_COUNTER_FIRST_PACKET; - counter = (asindex0 == 0)?LB_VIP_COUNTER_NO_SERVER:counter; - - //TODO: There are race conditions with as0 and vip0 manipulation. - //Configuration may be changed, vectors resized, etc... - - //Dereference previously used - vlib_refcount_add(&lbm->as_refcount, cpu_index, - lb_hash_available_value(sticky_ht, hash0, available_index0), -1); - vlib_refcount_add(&lbm->as_refcount, cpu_index, - asindex0, 1); - - //Add sticky entry - //Note that when there is no AS configured, an entry is configured anyway. - //But no configured AS is not something that should happen - lb_hash_put(sticky_ht, hash0, asindex0, - vnet_buffer (p0)->ip.adj_index[VLIB_TX], - available_index0, lb_time); - } - else - { - //Could not store new entry in the table - asindex0 = vip0->new_flow_table[hash0 & vip0->new_flow_table_mask].as_index; - counter = LB_VIP_COUNTER_UNTRACKED_PACKET; - } - - vlib_increment_simple_counter(&lbm->vip_counters[counter], - cpu_index, - vnet_buffer (p0)->ip.adj_index[VLIB_TX], - 1); - - //Now let's encap - { - gre_header_t *gre0; - if (is_encap_v4) - { - ip4_header_t *ip40; - vlib_buffer_advance(p0, - sizeof(ip4_header_t) - sizeof(gre_header_t)); - ip40 = vlib_buffer_get_current(p0); - gre0 = (gre_header_t *)(ip40 + 1); - ip40->src_address = lbm->ip4_src_address; - ip40->dst_address = lbm->ass[asindex0].address.ip4; - ip40->ip_version_and_header_length = 0x45; - ip40->ttl = 128; - ip40->length = clib_host_to_net_u16(len0 + sizeof(gre_header_t) + sizeof(ip4_header_t)); - ip40->protocol = IP_PROTOCOL_GRE; - ip40->checksum = ip4_header_checksum (ip40); - } - else - { - ip6_header_t *ip60; - vlib_buffer_advance(p0, - sizeof(ip6_header_t) - sizeof(gre_header_t)); - ip60 = vlib_buffer_get_current(p0); - gre0 = (gre_header_t *)(ip60 + 1); - ip60->dst_address = lbm->ass[asindex0].address.ip6; - ip60->src_address = lbm->ip6_src_address; - ip60->hop_limit = 128; - ip60->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6<<28); - ip60->payload_length = clib_host_to_net_u16(len0 + sizeof(gre_header_t)); - ip60->protocol = IP_PROTOCOL_GRE; - } - - gre0->flags_and_version = 0; - gre0->protocol = (is_input_v4)? - clib_host_to_net_u16(0x0800): - clib_host_to_net_u16(0x86DD); - } - - if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED)) - { - lb_trace_t *tr = vlib_add_trace (vm, node, p0, sizeof (*tr)); - tr->as_index = asindex0; - tr->vip_index = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; - } - - //Enqueue to next - //Note that this is going to error if asindex0 == 0 - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbm->ass[asindex0].dpo.dpoi_index; - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, pi0, - lbm->ass[asindex0].dpo.dpoi_next_node); - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - return frame->n_vectors; -} - -static uword -lb6_gre6_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return lb_node_fn(vm, node, frame, 0, 0); -} - -static uword -lb6_gre4_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return lb_node_fn(vm, node, frame, 0, 1); -} - -static uword -lb4_gre6_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return lb_node_fn(vm, node, frame, 1, 0); -} - -static uword -lb4_gre4_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return lb_node_fn(vm, node, frame, 1, 1); -} - -VLIB_REGISTER_NODE (lb6_gre6_node) = -{ - .function = lb6_gre6_node_fn, - .name = "lb6-gre6", - .vector_size = sizeof (u32), - .format_trace = format_lb_trace, - - .n_errors = LB_N_ERROR, - .error_strings = lb_error_strings, - - .n_next_nodes = LB_N_NEXT, - .next_nodes = - { - [LB_NEXT_DROP] = "error-drop" - }, -}; - -VLIB_REGISTER_NODE (lb6_gre4_node) = -{ - .function = lb6_gre4_node_fn, - .name = "lb6-gre4", - .vector_size = sizeof (u32), - .format_trace = format_lb_trace, - - .n_errors = LB_N_ERROR, - .error_strings = lb_error_strings, - - .n_next_nodes = LB_N_NEXT, - .next_nodes = - { - [LB_NEXT_DROP] = "error-drop" - }, -}; - -VLIB_REGISTER_NODE (lb4_gre6_node) = -{ - .function = lb4_gre6_node_fn, - .name = "lb4-gre6", - .vector_size = sizeof (u32), - .format_trace = format_lb_trace, - - .n_errors = LB_N_ERROR, - .error_strings = lb_error_strings, - - .n_next_nodes = LB_N_NEXT, - .next_nodes = - { - [LB_NEXT_DROP] = "error-drop" - }, -}; - -VLIB_REGISTER_NODE (lb4_gre4_node) = -{ - .function = lb4_gre4_node_fn, - .name = "lb4-gre4", - .vector_size = sizeof (u32), - .format_trace = format_lb_trace, - - .n_errors = LB_N_ERROR, - .error_strings = lb_error_strings, - - .n_next_nodes = LB_N_NEXT, - .next_nodes = - { - [LB_NEXT_DROP] = "error-drop" - }, -}; - diff --git a/plugins/lb-plugin/lb/refcount.c b/plugins/lb-plugin/lb/refcount.c deleted file mode 100644 index 22415c88..00000000 --- a/plugins/lb-plugin/lb/refcount.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size) -{ - u32 *new_counter = 0, *old_counter; - vec_validate(new_counter, size); - memcpy(new_counter, per_cpu->counters, per_cpu->length); - old_counter = per_cpu->counters; - per_cpu->counters = new_counter; - CLIB_MEMORY_BARRIER(); - per_cpu->length = vec_len(new_counter); - vec_free(old_counter); -} - -u64 vlib_refcount_get(vlib_refcount_t *r, u32 index) -{ - u64 count = 0; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 cpu_index; - for (cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++) { - if (r->per_cpu[cpu_index].length > index) - count += r->per_cpu[cpu_index].counters[index]; - } - return count; -} - diff --git a/plugins/lb-plugin/lb/refcount.h b/plugins/lb-plugin/lb/refcount.h deleted file mode 100644 index 8c26e7be..00000000 --- a/plugins/lb-plugin/lb/refcount.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * vlib provides lock-free counters but those - * - Have 16bits per-CPU counter, which may overflow. - * - Would only increment. - * - * This is very similar to vlib counters, but may be used to count reference. - * Such a counter includes an arbitrary number of counters. Each counter - * is identified by its index. This is used to aggregate per-cpu memory. - * - * Warning: - * This reference counter is lock-free but is not race-condition free. - * The counting result is approximate and another mechanism needs to be used - * in order to ensure that an object may be freed. - * - */ - -#include - -typedef struct { - u32 *counters; - u32 length; - u32 *reader_lengths; - CLIB_CACHE_LINE_ALIGN_MARK(o); -} vlib_refcount_per_cpu_t; - -typedef struct { - vlib_refcount_per_cpu_t *per_cpu; -} vlib_refcount_t; - -void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size); - -static_always_inline -void vlib_refcount_add(vlib_refcount_t *r, u32 cpu_index, u32 counter_index, i32 v) -{ - vlib_refcount_per_cpu_t *per_cpu = &r->per_cpu[cpu_index]; - if (PREDICT_FALSE(counter_index >= per_cpu->length)) - __vlib_refcount_resize(per_cpu, clib_max(counter_index + 16, per_cpu->length * 2)); - - per_cpu->counters[counter_index] += v; -} - -u64 vlib_refcount_get(vlib_refcount_t *r, u32 index); - -static_always_inline -void vlib_refcount_init(vlib_refcount_t *r) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - r->per_cpu = 0; - vec_validate (r->per_cpu, tm->n_vlib_mains - 1); -} - - diff --git a/plugins/lb-plugin/lb/util.c b/plugins/lb-plugin/lb/util.c deleted file mode 100644 index d969d168..00000000 --- a/plugins/lb-plugin/lb/util.c +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -void ip46_prefix_normalize(ip46_address_t *prefix, u8 plen) -{ - if (plen == 0) { - prefix->as_u64[0] = 0; - prefix->as_u64[1] = 0; - } else if (plen <= 64) { - prefix->as_u64[0] &= clib_host_to_net_u64(0xffffffffffffffffL << (64 - plen)); - prefix->as_u64[1] = 0; - } else { - prefix->as_u64[1] &= clib_host_to_net_u64(0xffffffffffffffffL << (128 - plen)); - } - -} - -uword unformat_ip46_prefix (unformat_input_t * input, va_list * args) -{ - ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); - u8 *len = va_arg (*args, u8 *); - ip46_type_t type = va_arg (*args, ip46_type_t); - - u32 l; - if ((type != IP46_TYPE_IP6) && unformat(input, "%U/%u", unformat_ip4_address, &ip46->ip4, &l)) { - if (l > 32) - return 0; - *len = l + 96; - ip46->pad[0] = ip46->pad[1] = ip46->pad[2] = 0; - } else if ((type != IP46_TYPE_IP4) && unformat(input, "%U/%u", unformat_ip6_address, &ip46->ip6, &l)) { - if (l > 128) - return 0; - *len = l; - } else { - return 0; - } - return 1; -} - -u8 *format_ip46_prefix (u8 * s, va_list * args) -{ - ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); - u32 len = va_arg (*args, u32); //va_arg cannot use u8 or u16 - ip46_type_t type = va_arg (*args, ip46_type_t); - - int is_ip4 = 0; - if (type == IP46_TYPE_IP4) - is_ip4 = 1; - else if (type == IP46_TYPE_IP6) - is_ip4 = 0; - else - is_ip4 = (len >= 96) && ip46_address_is_ip4(ip46); - - return is_ip4 ? - format(s, "%U/%d", format_ip4_address, &ip46->ip4, len - 96): - format(s, "%U/%d", format_ip6_address, &ip46->ip6, len); -} - diff --git a/plugins/lb-plugin/lb/util.h b/plugins/lb-plugin/lb/util.h deleted file mode 100644 index 3f082310..00000000 --- a/plugins/lb-plugin/lb/util.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Non-LB specific stuff comes here - */ - -#ifndef LB_PLUGIN_LB_UTIL_H_ -#define LB_PLUGIN_LB_UTIL_H_ - -#include -#include - -#define ip46_address_type(ip46) (ip46_address_is_ip4(ip46)?IP46_TYPE_IP4:IP46_TYPE_IP6) -#define ip46_prefix_is_ip4(ip46, len) ((len) >= 96 && ip46_address_is_ip4(ip46)) -#define ip46_prefix_type(ip46, len) (ip46_prefix_is_ip4(ip46, len)?IP46_TYPE_IP4:IP46_TYPE_IP6) - -void ip46_prefix_normalize(ip46_address_t *prefix, u8 plen); -uword unformat_ip46_prefix (unformat_input_t * input, va_list * args); -u8 *format_ip46_prefix (u8 * s, va_list * args); - -/** - * 32 bits integer comparison for running values. - * 1 > 0 is true. But 1 > 0xffffffff also is. - */ -#define clib_u32_loop_gt(a, b) (((u32)(a)) - ((u32)(b)) < 0x7fffffff) - -#endif /* LB_PLUGIN_LB_UTIL_H_ */ diff --git a/plugins/lb-plugin/lb_plugin_doc.md b/plugins/lb-plugin/lb_plugin_doc.md deleted file mode 100644 index c7885ffb..00000000 --- a/plugins/lb-plugin/lb_plugin_doc.md +++ /dev/null @@ -1,141 +0,0 @@ -# Load Balancer plugin for VPP {#lb_plugin_doc} - -## Version - -The load balancer plugin is currently in *beta* version. -Both CLIs and APIs are subject to *heavy* changes. -Wich also means feedback is really welcome regarding features, apis, etc... - -## Overview - -This plugin provides load balancing for VPP in a way that is largely inspired -from Google's MagLev: http://research.google.com/pubs/pub44824.html - -The load balancer is configured with a set of Virtual IPs (VIP, which can be -prefixes), and for each VIP, with a set of Application Server addresses (ASs). - -Traffic received for a given VIP (or VIP prefix) is tunneled using GRE towards -the different ASs in a way that (tries to) ensure that a given session will -always be tunneled to the same AS. - -Both VIPs or ASs can be IPv4 or IPv6, but for a given VIP, all ASs must be using -the same encap. type (i.e. IPv4+GRE or IPv6+GRE). Meaning that for a given VIP, -all AS addresses must be of the same family. - -## Performances - -The load balancer has been tested up to 1 millions flows and still forwards more -than 3Mpps per core in such circumstances. -Although 3Mpps seems already good, it is likely that performances will be improved -in next versions. - -## Configuration - -### Global LB parameters - -The load balancer needs to be configured with some parameters: - - lb conf [ip4-src-address ] [ip6-src-address ] - [buckets ] [timeout ] - -ip4-src-address: the source address used to send encap. packets using IPv4. - -ip6-src-address: the source address used to send encap. packets using IPv6. - -buckets: the *per-thread* established-connexions-table number of buckets. - -timeout: the number of seconds a connection will remain in the - established-connexions-table while no packet for this flow - is received. - - -### Configure the VIPs - - lb vip [encap (gre6|gre4)] [new_len ] [del] - -new_len is the size of the new-connection-table. It should be 1 or 2 orders of -magnitude bigger than the number of ASs for the VIP in order to ensure a good -load balancing. - -Examples: - - lb vip 2002::/16 encap gre6 new_len 1024 - lb vip 2003::/16 encap gre4 new_len 2048 - lb vip 80.0.0.0/8 encap gre6 new_len 16 - lb vip 90.0.0.0/8 encap gre4 new_len 1024 - -### Configure the ASs (for each VIP) - - lb as [
[
[...]]] [del] - -You can add (or delete) as many ASs at a time (for a single VIP). -Note that the AS address family must correspond to the VIP encap. IP family. - -Examples: - - lb as 2002::/16 2001::2 2001::3 2001::4 - lb as 2003::/16 10.0.0.1 10.0.0.2 - lb as 80.0.0.0/8 2001::2 - lb as 90.0.0.0/8 10.0.0.1 - - - -## Monitoring - -The plugin provides quite a bunch of counters and information. -These are still subject to quite significant changes. - - show lb - show lb vip - show lb vip verbose - - show node counters - - -## Design notes - -### Multi-Threading - -MagLev is a distributed system which pseudo-randomly generates a -new-connections-table based on AS names such that each server configured with -the same set of ASs ends up with the same table. Connection stickyness is then -ensured with an established-connections-table. Using ECMP, it is assumed (but -not relied on) that servers will mostly receive traffic for different flows. - -This implementation pushes the parallelism a little bit further by using -one established-connections table per thread. This is equivalent to assuming -that RSS will make a job similar to ECMP, and is pretty useful as threads don't -need to get a lock in order to write in the table. - -### Hash Table - -A load balancer requires an efficient read and write hash table. The hash table -used by ip6-forward is very read-efficient, but not so much for writing. In -addition, it is not a big deal if writing into the hash table fails (again, -MagLev uses a flow table but does not heaviliy relies on it). - -The plugin therefore uses a very specific (and stupid) hash table. - - Fixed (and power of 2) number of buckets (configured at runtime) - - Fixed (and power of 2) elements per buckets (configured at compilation time) - -### Reference counting - -When an AS is removed, there is two possible ways to react. - - Keep using the AS for established connections - - Change AS for established connections (likely to cause error for TCP) - -In the first case, although an AS is removed from the configuration, its -associated state needs to stay around as long as it is used by at least one -thread. - -In order to avoid locks, a specific reference counter is used. The design is quite -similar to clib counters but: - - It is possible to decrease the value - - Summing will not zero the per-thread counters - - Only the thread can reallocate its own counters vector (to avoid concurrency issues) - -This reference counter is lock free, but reading a count of 0 does not mean -the value can be freed unless it is ensured by *other* means that no other thread -is concurrently referencing the object. In the case of this plugin, it is assumed -that no concurrent event will take place after a few seconds. - diff --git a/plugins/sample-plugin/Makefile.am b/plugins/sample-plugin/Makefile.am deleted file mode 100644 index e221f8c1..00000000 --- a/plugins/sample-plugin/Makefile.am +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2015 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AUTOMAKE_OPTIONS = foreign subdir-objects - -AM_CFLAGS = -Wall -AM_LDFLAGS = -module -shared -avoid-version - -vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins -vpppluginsdir = ${libdir}/vpp_plugins - -vppapitestplugins_LTLIBRARIES = sample_test_plugin.la -vppplugins_LTLIBRARIES = sample_plugin.la - -sample_plugin_la_SOURCES = sample/sample.c sample/node.c \ - sample/sample_plugin.api.h - -BUILT_SOURCES = sample/sample.api.h sample/sample.api.json - -SUFFIXES = .api.h .api - -%.api.h: %.api - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --output $@ --show-name $@ - -%.api.json: %.api - @echo " JSON APIGEN " $@ ; \ - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --json $@ - -apidir = $(prefix)/sample/ -api_DATA = sample.api.json - -noinst_HEADERS = \ - sample/sample_all_api_h.h \ - sample/sample_msg_enum.h \ - sample/sample.api.h - -sample_test_plugin_la_SOURCES = sample/sample_test.c sample/sample_plugin.api.h - -# Remove *.la files -install-data-hook: - @(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) - @(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) diff --git a/plugins/sample-plugin/configure.ac b/plugins/sample-plugin/configure.ac deleted file mode 100644 index 43642732..00000000 --- a/plugins/sample-plugin/configure.ac +++ /dev/null @@ -1,9 +0,0 @@ -AC_INIT(sample_plugin, 1.0) -LT_INIT -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) -AC_PREFIX_DEFAULT([/usr]) - -AC_PROG_CC - -AC_OUTPUT([Makefile]) diff --git a/plugins/sample-plugin/sample/node.c b/plugins/sample-plugin/sample/node.c deleted file mode 100644 index 94c1706b..00000000 --- a/plugins/sample-plugin/sample/node.c +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include - -typedef struct { - u32 next_index; - u32 sw_if_index; - u8 new_src_mac[6]; - u8 new_dst_mac[6]; -} sample_trace_t; - -static u8 * -format_mac_address (u8 * s, va_list * args) -{ - u8 *a = va_arg (*args, u8 *); - return format (s, "%02x:%02x:%02x:%02x:%02x:%02x", - a[0], a[1], a[2], a[3], a[4], a[5]); -} - -/* packet trace format function */ -static u8 * format_sample_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - sample_trace_t * t = va_arg (*args, sample_trace_t *); - - s = format (s, "SAMPLE: sw_if_index %d, next index %d\n", - t->sw_if_index, t->next_index); - s = format (s, " new src %U -> new dst %U", - format_mac_address, t->new_src_mac, - format_mac_address, t->new_dst_mac); - - return s; -} - -vlib_node_registration_t sample_node; - -#define foreach_sample_error \ -_(SWAPPED, "Mac swap packets processed") - -typedef enum { -#define _(sym,str) SAMPLE_ERROR_##sym, - foreach_sample_error -#undef _ - SAMPLE_N_ERROR, -} sample_error_t; - -static char * sample_error_strings[] = { -#define _(sym,string) string, - foreach_sample_error -#undef _ -}; - -typedef enum { - SAMPLE_NEXT_INTERFACE_OUTPUT, - SAMPLE_N_NEXT, -} sample_next_t; - -#define foreach_mac_address_offset \ -_(0) \ -_(1) \ -_(2) \ -_(3) \ -_(4) \ -_(5) - -static uword -sample_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - u32 n_left_from, * from, * to_next; - sample_next_t next_index; - u32 pkts_swapped = 0; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, - to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 next0 = SAMPLE_NEXT_INTERFACE_OUTPUT; - u32 next1 = SAMPLE_NEXT_INTERFACE_OUTPUT; - u32 sw_if_index0, sw_if_index1; - u8 tmp0[6], tmp1[6]; - ethernet_header_t *en0, *en1; - u32 bi0, bi1; - vlib_buffer_t * b0, * b1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t * p2, * p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); - } - - /* speculatively enqueue b0 and b1 to the current next frame */ - to_next[0] = bi0 = from[0]; - to_next[1] = bi1 = from[1]; - from += 2; - to_next += 2; - n_left_from -= 2; - n_left_to_next -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - ASSERT (b0->current_data == 0); - ASSERT (b1->current_data == 0); - - en0 = vlib_buffer_get_current (b0); - en1 = vlib_buffer_get_current (b1); - - /* This is not the fastest way to swap src + dst mac addresses */ -#define _(a) tmp0[a] = en0->src_address[a]; - foreach_mac_address_offset; -#undef _ -#define _(a) en0->src_address[a] = en0->dst_address[a]; - foreach_mac_address_offset; -#undef _ -#define _(a) en0->dst_address[a] = tmp0[a]; - foreach_mac_address_offset; -#undef _ - -#define _(a) tmp1[a] = en1->src_address[a]; - foreach_mac_address_offset; -#undef _ -#define _(a) en1->src_address[a] = en1->dst_address[a]; - foreach_mac_address_offset; -#undef _ -#define _(a) en1->dst_address[a] = tmp1[a]; - foreach_mac_address_offset; -#undef _ - - - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX]; - - /* Send pkt back out the RX interface */ - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sw_if_index0; - vnet_buffer(b1)->sw_if_index[VLIB_TX] = sw_if_index1; - - pkts_swapped += 2; - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE))) - { - if (b0->flags & VLIB_BUFFER_IS_TRACED) - { - sample_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - clib_memcpy (t->new_src_mac, en0->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en0->dst_address, - sizeof (t->new_dst_mac)); - - } - if (b1->flags & VLIB_BUFFER_IS_TRACED) - { - sample_trace_t *t = - vlib_add_trace (vm, node, b1, sizeof (*t)); - t->sw_if_index = sw_if_index1; - t->next_index = next1; - clib_memcpy (t->new_src_mac, en1->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en1->dst_address, - sizeof (t->new_dst_mac)); - } - } - - /* verify speculative enqueues, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, - to_next, n_left_to_next, - bi0, bi1, next0, next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t * b0; - u32 next0 = SAMPLE_NEXT_INTERFACE_OUTPUT; - u32 sw_if_index0; - u8 tmp0[6]; - ethernet_header_t *en0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - /* - * Direct from the driver, we should be at offset 0 - * aka at &b0->data[0] - */ - ASSERT (b0->current_data == 0); - - en0 = vlib_buffer_get_current (b0); - - /* This is not the fastest way to swap src + dst mac addresses */ -#define _(a) tmp0[a] = en0->src_address[a]; - foreach_mac_address_offset; -#undef _ -#define _(a) en0->src_address[a] = en0->dst_address[a]; - foreach_mac_address_offset; -#undef _ -#define _(a) en0->dst_address[a] = tmp0[a]; - foreach_mac_address_offset; -#undef _ - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - - /* Send pkt back out the RX interface */ - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sw_if_index0; - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) { - sample_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - clib_memcpy (t->new_src_mac, en0->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en0->dst_address, - sizeof (t->new_dst_mac)); - } - - pkts_swapped += 1; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, sample_node.index, - SAMPLE_ERROR_SWAPPED, pkts_swapped); - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (sample_node) = { - .function = sample_node_fn, - .name = "sample", - .vector_size = sizeof (u32), - .format_trace = format_sample_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(sample_error_strings), - .error_strings = sample_error_strings, - - .n_next_nodes = SAMPLE_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [SAMPLE_NEXT_INTERFACE_OUTPUT] = "interface-output", - }, -}; diff --git a/plugins/sample-plugin/sample/sample.api b/plugins/sample-plugin/sample/sample.api deleted file mode 100644 index f99cdb38..00000000 --- a/plugins/sample-plugin/sample/sample.api +++ /dev/null @@ -1,39 +0,0 @@ -/* Hey Emacs use -*- mode: C -*- */ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Define a simple binary API to control the feature */ - -define sample_macswap_enable_disable { - /* Client identifier, set from api_main.my_client_index */ - u32 client_index; - - /* Arbitrary context, so client can match reply to request */ - u32 context; - - /* Enable / disable the feature */ - u8 enable_disable; - - /* Interface handle */ - u32 sw_if_index; -}; - -define sample_macswap_enable_disable_reply { - /* From the request */ - u32 context; - - /* Return value, zero means all OK */ - i32 retval; -}; diff --git a/plugins/sample-plugin/sample/sample.c b/plugins/sample-plugin/sample/sample.c deleted file mode 100644 index 603cb2d0..00000000 --- a/plugins/sample-plugin/sample/sample.c +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * sample.c - simple MAC-swap API / debug CLI handling - *------------------------------------------------------------------ - */ - -#include -#include -#include - -#include -#include -#include - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - - -/* List of message types that this plugin understands */ - -#define foreach_sample_plugin_api_msg \ -_(SAMPLE_MACSWAP_ENABLE_DISABLE, sample_macswap_enable_disable) - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - sample_main_t * sm = &sample_main; - clib_error_t * error = 0; - - sm->vlib_main = vm; - sm->vnet_main = h->vnet_main; - sm->ethernet_main = h->ethernet_main; - - return error; -} - -/* Action function shared between message handler and debug CLI */ - -int sample_macswap_enable_disable (sample_main_t * sm, u32 sw_if_index, - int enable_disable) -{ - vnet_sw_interface_t * sw; - int rv = 0; - - /* Utterly wrong? */ - if (pool_is_free_index (sm->vnet_main->interface_main.sw_interfaces, - sw_if_index)) - return VNET_API_ERROR_INVALID_SW_IF_INDEX; - - /* Not a physical port? */ - sw = vnet_get_sw_interface (sm->vnet_main, sw_if_index); - if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE) - return VNET_API_ERROR_INVALID_SW_IF_INDEX; - - vnet_feature_enable_disable ("device-input", "sample", - sw_if_index, enable_disable, 0, 0); - - return rv; -} - -static clib_error_t * -macswap_enable_disable_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - sample_main_t * sm = &sample_main; - u32 sw_if_index = ~0; - int enable_disable = 1; - - int rv; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "disable")) - enable_disable = 0; - else if (unformat (input, "%U", unformat_vnet_sw_interface, - sm->vnet_main, &sw_if_index)) - ; - else - break; - } - - if (sw_if_index == ~0) - return clib_error_return (0, "Please specify an interface..."); - - rv = sample_macswap_enable_disable (sm, sw_if_index, enable_disable); - - switch(rv) { - case 0: - break; - - case VNET_API_ERROR_INVALID_SW_IF_INDEX: - return clib_error_return - (0, "Invalid interface, only works on physical ports"); - break; - - case VNET_API_ERROR_UNIMPLEMENTED: - return clib_error_return (0, "Device driver doesn't support redirection"); - break; - - default: - return clib_error_return (0, "sample_macswap_enable_disable returned %d", - rv); - } - return 0; -} - -VLIB_CLI_COMMAND (sr_content_command, static) = { - .path = "sample macswap", - .short_help = - "sample macswap [disable]", - .function = macswap_enable_disable_command_fn, -}; - -/* API message handler */ -static void vl_api_sample_macswap_enable_disable_t_handler -(vl_api_sample_macswap_enable_disable_t * mp) -{ - vl_api_sample_macswap_enable_disable_reply_t * rmp; - sample_main_t * sm = &sample_main; - int rv; - - rv = sample_macswap_enable_disable (sm, ntohl(mp->sw_if_index), - (int) (mp->enable_disable)); - - REPLY_MACRO(VL_API_SAMPLE_MACSWAP_ENABLE_DISABLE_REPLY); -} - -/* Set up the API message handling tables */ -static clib_error_t * -sample_plugin_api_hookup (vlib_main_t *vm) -{ - sample_main_t * sm = &sample_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_sample_plugin_api_msg; -#undef _ - - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (sample_main_t * sm, api_main_t *am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); - foreach_vl_msg_name_crc_sample; -#undef _ -} - -static clib_error_t * sample_init (vlib_main_t * vm) -{ - sample_main_t * sm = &sample_main; - clib_error_t * error = 0; - u8 * name; - - name = format (0, "sample_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - sm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - - error = sample_plugin_api_hookup (vm); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (sm, &api_main); - - vec_free(name); - - return error; -} - -VLIB_INIT_FUNCTION (sample_init); - -VNET_FEATURE_INIT (sample, static) = -{ - .arc_name = "device-input", - .node_name = "sample", - .runs_before = VNET_FEATURES ("ethernet-input"), -}; diff --git a/plugins/sample-plugin/sample/sample.h b/plugins/sample-plugin/sample/sample.h deleted file mode 100644 index d268d482..00000000 --- a/plugins/sample-plugin/sample/sample.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_sample_h__ -#define __included_sample_h__ - -#include -#include -#include - -#include -#include -#include - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - - /* convenience */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; - ethernet_main_t * ethernet_main; -} sample_main_t; - -sample_main_t sample_main; - -vlib_node_registration_t sample_node; - -#endif /* __included_sample_h__ */ diff --git a/plugins/sample-plugin/sample/sample_all_api_h.h b/plugins/sample-plugin/sample/sample_all_api_h.h deleted file mode 100644 index 774d782f..00000000 --- a/plugins/sample-plugin/sample/sample_all_api_h.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/sample-plugin/sample/sample_msg_enum.h b/plugins/sample-plugin/sample/sample_msg_enum.h deleted file mode 100644 index af4172f7..00000000 --- a/plugins/sample-plugin/sample/sample_msg_enum.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_sample_msg_enum_h -#define included_sample_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_sample_msg_enum_h */ diff --git a/plugins/sample-plugin/sample/sample_test.c b/plugins/sample-plugin/sample/sample_test.c deleted file mode 100644 index dd1b0215..00000000 --- a/plugins/sample-plugin/sample/sample_test.c +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - *------------------------------------------------------------------ - * sample_test.c - test harness plugin - *------------------------------------------------------------------ - */ - -#include -#include -#include -#include -#include - -uword unformat_sw_if_index (unformat_input_t * input, va_list * args); - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} sample_test_main_t; - -sample_test_main_t sample_test_main; - -#define foreach_standard_reply_retval_handler \ -_(sample_macswap_enable_disable_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = sample_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(SAMPLE_MACSWAP_ENABLE_DISABLE_REPLY, sample_macswap_enable_disable_reply) - - -/* M: construct, but don't yet send a message */ - -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - -static int api_sample_macswap_enable_disable (vat_main_t * vam) -{ - sample_test_main_t * sm = &sample_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - int enable_disable = 1; - u32 sw_if_index = ~0; - vl_api_sample_macswap_enable_disable_t * mp; - - /* Parse args required to build the message */ - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { - if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) - ; - else if (unformat (i, "sw_if_index %d", &sw_if_index)) - ; - else if (unformat (i, "disable")) - enable_disable = 0; - else - break; - } - - if (sw_if_index == ~0) { - errmsg ("missing interface name / explicit sw_if_index number \n"); - return -99; - } - - /* Construct the API message */ - M(SAMPLE_MACSWAP_ENABLE_DISABLE, sample_macswap_enable_disable); - mp->sw_if_index = ntohl (sw_if_index); - mp->enable_disable = enable_disable; - - /* send it... */ - S; - - /* Wait for a reply... */ - W; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(sample_macswap_enable_disable, " [disable]") - -void vat_api_hookup (vat_main_t *vam) -{ - sample_test_main_t * sm = &sample_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * vat_plugin_register (vat_main_t *vam) -{ - sample_test_main_t * sm = &sample_test_main; - u8 * name; - - sm->vat_main = vam; - - name = format (0, "sample_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~0) - vat_api_hookup (vam); - - vec_free(name); - - return 0; -} diff --git a/plugins/snat-plugin/Makefile.am b/plugins/snat-plugin/Makefile.am deleted file mode 100644 index e1922eb0..00000000 --- a/plugins/snat-plugin/Makefile.am +++ /dev/null @@ -1,113 +0,0 @@ - -# Copyright (c) -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -AUTOMAKE_OPTIONS = foreign subdir-objects - -AM_CFLAGS = -Wall @DPDK@ -AM_LDFLAGS = -module -shared -avoid-version - -vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins -vpppluginsdir = ${libdir}/vpp_plugins - -vppapitestplugins_LTLIBRARIES = snat_test_plugin.la -vppplugins_LTLIBRARIES = snat_plugin.la - -snat_plugin_la_SOURCES = snat/snat.c \ - snat/in2out.c \ - snat/out2in.c \ - snat/snat_plugin.api.h - -BUILT_SOURCES = snat/snat.api.h snat/snat.api.json - -SUFFIXES = .api.h .api - -%.api.h: %.api - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --output $@ --show-name $@ - -%.api.json: %.api - @echo " JSON APIGEN " $@ ; \ - mkdir -p `dirname $@` ; \ - $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --json $@ - -apidir = $(prefix)/snat/ -api_DATA = snat.api.json - -noinst_HEADERS = \ - snat/snat_all_api_h.h \ - snat/snat_msg_enum.h \ - snat/snat.api.h - -snat_test_plugin_la_SOURCES = \ - snat/snat_test.c snat/snat_plugin.api.h - -# Remove *.la files -install-data-hook: - @(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) - @(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) - -# -# Java code generation -# -jvpp_registry_root = ../../vpp-api/java -jvpp_registry_version = 17.04 -jsnat_jarfile = jvpp-snat-$(PACKAGE_VERSION).jar -jvpp_package_dir = io/fd/vpp/jvpp/snat -jvpp_root = snat/jvpp -jvpp_target_dir = target -jvpp_target = $(jvpp_root)/$(jvpp_target_dir) -api_file=$(srcdir)/snat/snat.api - -lib_LTLIBRARIES = libjvpp_snat.la -libjvpp_snat_la_SOURCES = snat/snat.api.h snat/jvpp_snat.c snat/jvpp/io_fd_vpp_jvpp_snat_JVppSnatImpl.h -libjvpp_snat_la_LIBADD = -lvlibmemoryclient -lvlibapi -lvppinfra \ - -lpthread -lm -lrt -L$(jvpp_registry_root)/.libs -ljvpp_common -libjvpp_snat_la_LDFLAGS = -module -libjvpp_snat_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -I../ -I$(srcdir)/../ - -BUILT_SOURCES += $(jvpp_root)/io_fd_vpp_jvpp_snat_JVppSnatImpl.h - -$(jvpp_root)/io_fd_vpp_jvpp_snat_JVppSnatImpl.h: snat.api.json - dir=`pwd`; \ - mkdir -p $(jvpp_target); \ - mkdir -p $(jvpp_root)/$(jvpp_package_dir); \ - cd $(jvpp_root)/$(jvpp_package_dir); \ - mkdir -p dto future callfacade callback notification test; \ - @srcdir@/$(jvpp_registry_root)/jvpp/gen/jvpp_gen.py -i $${dir}/snat.api.json --plugin_name snat; \ - cd -; \ - mv -f $(jvpp_root)/$(jvpp_package_dir)/jvpp_snat_gen.h $(jvpp_root)/jvpp_snat_gen.h; \ - cp $(srcdir)/$(jvpp_root)/$(jvpp_package_dir)/test/*.java $(jvpp_root)/$(jvpp_package_dir)/test/; \ - cd $(jvpp_root); \ - $(JAVAC) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d $(jvpp_target_dir) $(jvpp_package_dir)/*.java \ - $(jvpp_package_dir)/dto/*.java \ - $(jvpp_package_dir)/callback/*.java \ - $(jvpp_package_dir)/notification/*.java \ - $(jvpp_package_dir)/future/*.java \ - $(jvpp_package_dir)/callfacade/*.java \ - $(jvpp_package_dir)/test/*.java \ - || (echo "snat jvpp compilation failed: $$?"; exit 1); \ - $(JAVAH) -classpath .:$(jvpp_target_dir):../../$(jvpp_registry_root)/jvpp-registry-$(jvpp_registry_version).jar -d . io.fd.vpp.jvpp.snat.JVppSnatImpl ; - -$(jsnat_jarfile): libjvpp_snat.la - cp .libs/libjvpp_snat.so.0.0.0 $(jvpp_target); \ - cd $(jvpp_target); \ - $(JAR) cfv $(JARFLAGS) ../../../$@ libjvpp_snat.so.0.0.0 $(jvpp_package_dir)/* ; cd ..; - -snat.api.json: - @echo " jSnat_sfc API"; \ - vppapigen --input $(api_file) --json snat.api.json; - -all-local: $(jsnat_jarfile) diff --git a/plugins/snat-plugin/configure.ac b/plugins/snat-plugin/configure.ac deleted file mode 100644 index 011246cd..00000000 --- a/plugins/snat-plugin/configure.ac +++ /dev/null @@ -1,32 +0,0 @@ -AC_INIT(snat_plugin, 1.0) -AC_CONFIG_MACRO_DIR([../../vpp-api/java/m4]) -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) -AC_PREFIX_DEFAULT([/usr]) - -AC_PROG_LIBTOOL -AC_PROG_CC - -if test -f /usr/bin/lsb_release && test `lsb_release -si` == "Ubuntu" && test `lsb_release -sr` == "14.04" && test -d /usr/lib/jvm/java-8-openjdk-amd64/ ; then - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ - JAVAC=${JAVA_HOME}/bin/javac - PATH=${JAVA_HOME}/bin/:${PATH} - break -fi - -AX_CHECK_JAVA_HOME -AX_PROG_JAVAC -AX_PROG_JAVAH -AX_PROG_JAR -AX_PROG_JAVADOC -AX_PROG_JAVA - -AC_ARG_WITH(dpdk, - AC_HELP_STRING([--with-dpdk],[Use DPDK]), - [with_dpdk=1], - [with_dpdk=0]) - -AM_CONDITIONAL(WITH_DPDK, test "$with_dpdk" = "1") -AC_SUBST(DPDK,["-DDPDK=${with_dpdk}"]) - -AC_OUTPUT([Makefile]) diff --git a/plugins/snat-plugin/snat/in2out.c b/plugins/snat-plugin/snat/in2out.c deleted file mode 100644 index c78fdd76..00000000 --- a/plugins/snat-plugin/snat/in2out.c +++ /dev/null @@ -1,1597 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -typedef struct { - u32 sw_if_index; - u32 next_index; - u32 session_index; - u32 is_slow_path; -} snat_in2out_trace_t; - -typedef struct { - u32 next_worker_index; - u8 do_handoff; -} snat_in2out_worker_handoff_trace_t; - -/* packet trace format function */ -static u8 * format_snat_in2out_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *); - char * tag; - - tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH"; - - s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag, - t->sw_if_index, t->next_index, t->session_index); - - return s; -} - -static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *); - - s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d", - t->sw_if_index, t->next_index); - - return s; -} - -static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - snat_in2out_worker_handoff_trace_t * t = - va_arg (*args, snat_in2out_worker_handoff_trace_t *); - char * m; - - m = t->do_handoff ? "next worker" : "same worker"; - s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index); - - return s; -} - -vlib_node_registration_t snat_in2out_node; -vlib_node_registration_t snat_in2out_slowpath_node; -vlib_node_registration_t snat_in2out_fast_node; -vlib_node_registration_t snat_in2out_worker_handoff_node; - -#define foreach_snat_in2out_error \ -_(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \ -_(IN2OUT_PACKETS, "Good in2out packets processed") \ -_(OUT_OF_PORTS, "Out of ports") \ -_(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \ -_(BAD_ICMP_TYPE, "icmp type not echo-request") \ -_(NO_TRANSLATION, "No translation") - -typedef enum { -#define _(sym,str) SNAT_IN2OUT_ERROR_##sym, - foreach_snat_in2out_error -#undef _ - SNAT_IN2OUT_N_ERROR, -} snat_in2out_error_t; - -static char * snat_in2out_error_strings[] = { -#define _(sym,string) string, - foreach_snat_in2out_error -#undef _ -}; - -typedef enum { - SNAT_IN2OUT_NEXT_LOOKUP, - SNAT_IN2OUT_NEXT_DROP, - SNAT_IN2OUT_NEXT_SLOW_PATH, - SNAT_IN2OUT_N_NEXT, -} snat_in2out_next_t; - -static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, - ip4_header_t * ip0, - u32 rx_fib_index0, - snat_session_key_t * key0, - snat_session_t ** sessionp, - vlib_node_runtime_t * node, - u32 next0, - u32 cpu_index) -{ - snat_user_t *u; - snat_user_key_t user_key; - snat_session_t *s; - clib_bihash_kv_8_8_t kv0, value0; - u32 oldest_per_user_translation_list_index; - dlist_elt_t * oldest_per_user_translation_list_elt; - dlist_elt_t * per_user_translation_list_elt; - dlist_elt_t * per_user_list_head_elt; - u32 session_index; - snat_session_key_t key1; - u32 address_index = ~0; - u32 outside_fib_index; - uword * p; - snat_static_mapping_key_t worker_by_out_key; - - p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id); - if (! p) - { - b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB]; - return SNAT_IN2OUT_NEXT_DROP; - } - outside_fib_index = p[0]; - - user_key.addr = ip0->src_address; - user_key.fib_index = rx_fib_index0; - kv0.key = user_key.as_u64; - - /* Ever heard of the "user" = src ip4 address before? */ - if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) - { - /* no, make a new one */ - pool_get (sm->per_thread_data[cpu_index].users, u); - memset (u, 0, sizeof (*u)); - u->addr = ip0->src_address; - - pool_get (sm->per_thread_data[cpu_index].list_pool, per_user_list_head_elt); - - u->sessions_per_user_list_head_index = per_user_list_head_elt - - sm->per_thread_data[cpu_index].list_pool; - - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, - u->sessions_per_user_list_head_index); - - kv0.value = u - sm->per_thread_data[cpu_index].users; - - /* add user */ - clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); - } - else - { - u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, - value0.value); - } - - /* Over quota? Recycle the least recently used dynamic translation */ - if (u->nsessions >= sm->max_translations_per_user) - { - /* Remove the oldest dynamic translation */ - do { - oldest_per_user_translation_list_index = - clib_dlist_remove_head (sm->per_thread_data[cpu_index].list_pool, - u->sessions_per_user_list_head_index); - - ASSERT (oldest_per_user_translation_list_index != ~0); - - /* add it back to the end of the LRU list */ - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - u->sessions_per_user_list_head_index, - oldest_per_user_translation_list_index); - /* Get the list element */ - oldest_per_user_translation_list_elt = - pool_elt_at_index (sm->per_thread_data[cpu_index].list_pool, - oldest_per_user_translation_list_index); - - /* Get the session index from the list element */ - session_index = oldest_per_user_translation_list_elt->value; - - /* Get the session */ - s = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - session_index); - } while (snat_is_session_static (s)); - - /* Remove in2out, out2in keys */ - kv0.key = s->in2out.as_u64; - if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */)) - clib_warning ("in2out key delete failed"); - kv0.key = s->out2in.as_u64; - if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */)) - clib_warning ("out2in key delete failed"); - - snat_free_outside_address_and_port - (sm, &s->out2in, s->outside_address_index); - s->outside_address_index = ~0; - - if (snat_alloc_outside_address_and_port (sm, &key1, &address_index)) - { - ASSERT(0); - - b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS]; - return SNAT_IN2OUT_NEXT_DROP; - } - s->outside_address_index = address_index; - } - else - { - u8 static_mapping = 1; - - /* First try to match static mapping by local address and port */ - if (snat_static_mapping_match (sm, *key0, &key1, 0)) - { - static_mapping = 0; - /* Try to create dynamic translation */ - if (snat_alloc_outside_address_and_port (sm, &key1, &address_index)) - { - b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS]; - return SNAT_IN2OUT_NEXT_DROP; - } - } - - /* Create a new session */ - pool_get (sm->per_thread_data[cpu_index].sessions, s); - memset (s, 0, sizeof (*s)); - - s->outside_address_index = address_index; - - if (static_mapping) - { - u->nstaticsessions++; - s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING; - } - else - { - u->nsessions++; - } - - /* Create list elts */ - pool_get (sm->per_thread_data[cpu_index].list_pool, - per_user_translation_list_elt); - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, - per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); - - per_user_translation_list_elt->value = - s - sm->per_thread_data[cpu_index].sessions; - s->per_user_index = per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool; - s->per_user_list_head_index = u->sessions_per_user_list_head_index; - - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s->per_user_list_head_index, - per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); - } - - s->in2out = *key0; - s->out2in = key1; - s->out2in.protocol = key0->protocol; - s->out2in.fib_index = outside_fib_index; - *sessionp = s; - - /* Add to translation hashes */ - kv0.key = s->in2out.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; - if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) - clib_warning ("in2out key add failed"); - - kv0.key = s->out2in.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; - - if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) - clib_warning ("out2in key add failed"); - - /* Add to translated packets worker lookup */ - worker_by_out_key.addr = s->out2in.addr; - worker_by_out_key.port = s->out2in.port; - worker_by_out_key.fib_index = s->out2in.fib_index; - kv0.key = worker_by_out_key.as_u64; - kv0.value = cpu_index; - clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1); - return next0; -} - -static inline u32 icmp_in2out_slow_path (snat_main_t *sm, - vlib_buffer_t * b0, - ip4_header_t * ip0, - icmp46_header_t * icmp0, - u32 sw_if_index0, - u32 rx_fib_index0, - vlib_node_runtime_t * node, - u32 next0, - f64 now, - u32 cpu_index) -{ - snat_session_key_t key0; - icmp_echo_header_t *echo0; - clib_bihash_kv_8_8_t kv0, value0; - snat_session_t * s0; - u32 new_addr0, old_addr0; - u16 old_id0, new_id0; - ip_csum_t sum0; - snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; - - if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request)) - { - b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE]; - return SNAT_IN2OUT_NEXT_DROP; - } - - echo0 = (icmp_echo_header_t *)(icmp0+1); - - key0.addr = ip0->src_address; - key0.port = echo0->identifier; - key0.protocol = SNAT_PROTOCOL_ICMP; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0)) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - return next0; - - next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); - - if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) - return next0; - } - else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value0.value); - - old_addr0 = ip0->src_address.as_u32; - ip0->src_address = s0->out2in.addr; - new_addr0 = ip0->src_address.as_u32; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - src_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - old_id0 = echo0->identifier; - new_id0 = s0->out2in.port; - echo0->identifier = new_id0; - - sum0 = icmp0->checksum; - sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, - identifier); - icmp0->checksum = ip_csum_fold (sum0); - - /* Accounting */ - s0->last_heard = now; - s0->total_pkts++; - s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0); - /* Per-user LRU list maintenance for dynamic translations */ - if (!snat_is_session_static (s0)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_list_head_index, - s0->per_user_index); - } - - return next0; -} - -/** - * @brief Hairpinning - * - * Hairpinning allows two endpoints on the internal side of the NAT to - * communicate even if they only use each other's external IP addresses - * and ports. - * - * @param sm SNAT main. - * @param b0 Vlib buffer. - * @param ip0 IP header. - * @param udp0 UDP header. - * @param tcp0 TCP header. - * @param proto0 SNAT protocol. - */ -static inline void -snat_hairpinning (snat_main_t *sm, - vlib_buffer_t * b0, - ip4_header_t * ip0, - udp_header_t * udp0, - tcp_header_t * tcp0, - u32 proto0) -{ - snat_session_key_t key0, sm0; - snat_static_mapping_key_t k0; - snat_session_t * s0; - clib_bihash_kv_8_8_t kv0, value0; - ip_csum_t sum0; - u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si; - u16 new_dst_port0, old_dst_port0; - - key0.addr = ip0->dst_address; - key0.port = udp0->dst_port; - key0.protocol = proto0; - key0.fib_index = sm->outside_fib_index; - kv0.key = key0.as_u64; - - /* Check if destination is in active sessions */ - if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) - { - /* or static mappings */ - if (!snat_static_mapping_match(sm, key0, &sm0, 1)) - { - new_dst_addr0 = sm0.addr.as_u32; - new_dst_port0 = sm0.port; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; - } - } - else - { - si = value0.value; - if (sm->num_workers > 1) - { - k0.addr = ip0->dst_address; - k0.port = udp0->dst_port; - k0.fib_index = sm->outside_fib_index; - kv0.key = k0.as_u64; - if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0)) - ASSERT(0); - else - ti = value0.value; - } - else - ti = sm->num_workers; - - s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si); - new_dst_addr0 = s0->in2out.addr.as_u32; - new_dst_port0 = s0->in2out.port; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; - } - - /* Destination is behind the same NAT, use internal address and port */ - if (new_dst_addr0) - { - old_dst_addr0 = ip0->dst_address.as_u32; - ip0->dst_address.as_u32 = new_dst_addr0; - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, - ip4_header_t, dst_address); - ip0->checksum = ip_csum_fold (sum0); - - old_dst_port0 = tcp0->ports.dst; - if (PREDICT_TRUE(new_dst_port0 != old_dst_port0)) - { - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - tcp0->ports.dst = new_dst_port0; - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, - ip4_header_t, dst_address); - sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0, - ip4_header_t /* cheat */, length); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - udp0->dst_port = new_dst_port0; - udp0->checksum = 0; - } - } - } -} - -static inline uword -snat_in2out_node_fn_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, int is_slow_path) -{ - u32 n_left_from, * from, * to_next; - snat_in2out_next_t next_index; - u32 pkts_processed = 0; - snat_main_t * sm = &snat_main; - snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; - f64 now = vlib_time_now (vm); - u32 stats_node_index; - u32 cpu_index = os_get_cpu_number (); - - stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index : - snat_in2out_node.index; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, - to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 bi0, bi1; - vlib_buffer_t * b0, * b1; - u32 next0, next1; - u32 sw_if_index0, sw_if_index1; - ip4_header_t * ip0, * ip1; - ip_csum_t sum0, sum1; - u32 new_addr0, old_addr0, new_addr1, old_addr1; - u16 old_port0, new_port0, old_port1, new_port1; - udp_header_t * udp0, * udp1; - tcp_header_t * tcp0, * tcp1; - icmp46_header_t * icmp0, * icmp1; - snat_session_key_t key0, key1; - u32 rx_fib_index0, rx_fib_index1; - u32 proto0, proto1; - snat_session_t * s0 = 0, * s1 = 0; - clib_bihash_kv_8_8_t kv0, value0, kv1, value1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t * p2, * p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); - } - - /* speculatively enqueue b0 and b1 to the current next frame */ - to_next[0] = bi0 = from[0]; - to_next[1] = bi1 = from[1]; - from += 2; - to_next += 2; - n_left_from -= 2; - n_left_to_next -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - tcp0 = (tcp_header_t *) udp0; - icmp0 = (icmp46_header_t *) udp0; - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, - sw_if_index0); - - next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP; - - proto0 = ~0; - proto0 = (ip0->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto0; - - /* Next configured feature, probably ip4-lookup */ - if (is_slow_path) - { - if (PREDICT_FALSE (proto0 == ~0)) - goto trace00; - - if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = icmp_in2out_slow_path - (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, - node, next0, now, cpu_index); - goto trace00; - } - } - else - { - if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; - goto trace00; - } - } - - key0.addr = ip0->src_address; - key0.port = udp0->src_port; - key0.protocol = proto0; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0)) - { - if (is_slow_path) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - goto trace00; - - next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); - if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) - goto trace00; - } - else - { - next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; - goto trace00; - } - } - else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value0.value); - - old_addr0 = ip0->src_address.as_u32; - ip0->src_address = s0->out2in.addr; - new_addr0 = ip0->src_address.as_u32; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - src_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - old_port0 = tcp0->ports.src; - tcp0->ports.src = s0->out2in.port; - new_port0 = tcp0->ports.src; - - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - sum0 = ip_csum_update (sum0, old_port0, new_port0, - ip4_header_t /* cheat */, - length /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - old_port0 = udp0->src_port; - udp0->src_port = s0->out2in.port; - udp0->checksum = 0; - } - - /* Hairpinning */ - snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0); - - /* Accounting */ - s0->last_heard = now; - s0->total_pkts++; - s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s0)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_list_head_index, - s0->per_user_index); - } - trace00: - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_in2out_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->is_slow_path = is_slow_path; - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->session_index = ~0; - if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; - } - - pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; - - ip1 = vlib_buffer_get_current (b1); - udp1 = ip4_next_header (ip1); - tcp1 = (tcp_header_t *) udp1; - icmp1 = (icmp46_header_t *) udp1; - - sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX]; - rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, - sw_if_index1); - - proto1 = ~0; - proto1 = (ip1->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto1; - proto1 = (ip1->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto1; - proto1 = (ip1->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto1; - - /* Next configured feature, probably ip4-lookup */ - if (is_slow_path) - { - if (PREDICT_FALSE (proto1 == ~0)) - goto trace01; - - if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP)) - { - next1 = icmp_in2out_slow_path - (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, - next1, now, cpu_index); - goto trace01; - } - } - else - { - if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP)) - { - next1 = SNAT_IN2OUT_NEXT_SLOW_PATH; - goto trace01; - } - } - - key1.addr = ip1->src_address; - key1.port = udp1->src_port; - key1.protocol = proto1; - key1.fib_index = rx_fib_index1; - - kv1.key = key1.as_u64; - - if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0)) - { - if (is_slow_path) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index1)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index1, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index1; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip1->dst_address.as_u32 == - rt->cached_ip4_address)) - goto trace01; - - next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1, - &s1, node, next1, cpu_index); - if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP)) - goto trace01; - } - else - { - next1 = SNAT_IN2OUT_NEXT_SLOW_PATH; - goto trace01; - } - } - else - s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value1.value); - - old_addr1 = ip1->src_address.as_u32; - ip1->src_address = s1->out2in.addr; - new_addr1 = ip1->src_address.as_u32; - vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index; - - sum1 = ip1->checksum; - sum1 = ip_csum_update (sum1, old_addr1, new_addr1, - ip4_header_t, - src_address /* changed member */); - ip1->checksum = ip_csum_fold (sum1); - - if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP)) - { - old_port1 = tcp1->ports.src; - tcp1->ports.src = s1->out2in.port; - new_port1 = tcp1->ports.src; - - sum1 = tcp1->checksum; - sum1 = ip_csum_update (sum1, old_addr1, new_addr1, - ip4_header_t, - dst_address /* changed member */); - sum1 = ip_csum_update (sum1, old_port1, new_port1, - ip4_header_t /* cheat */, - length /* changed member */); - tcp1->checksum = ip_csum_fold(sum1); - } - else - { - old_port1 = udp1->src_port; - udp1->src_port = s1->out2in.port; - udp1->checksum = 0; - } - - /* Hairpinning */ - snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1); - - /* Accounting */ - s1->last_heard = now; - s1->total_pkts++; - s1->total_bytes += vlib_buffer_length_in_chain (vm, b1); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s1)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s1->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s1->per_user_list_head_index, - s1->per_user_index); - } - trace01: - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b1->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_in2out_trace_t *t = - vlib_add_trace (vm, node, b1, sizeof (*t)); - t->sw_if_index = sw_if_index1; - t->next_index = next1; - t->session_index = ~0; - if (s1) - t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; - } - - pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP; - - /* verify speculative enqueues, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, - to_next, n_left_to_next, - bi0, bi1, next0, next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t * b0; - u32 next0; - u32 sw_if_index0; - ip4_header_t * ip0; - ip_csum_t sum0; - u32 new_addr0, old_addr0; - u16 old_port0, new_port0; - udp_header_t * udp0; - tcp_header_t * tcp0; - icmp46_header_t * icmp0; - snat_session_key_t key0; - u32 rx_fib_index0; - u32 proto0; - snat_session_t * s0 = 0; - clib_bihash_kv_8_8_t kv0, value0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - next0 = SNAT_IN2OUT_NEXT_LOOKUP; - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - tcp0 = (tcp_header_t *) udp0; - icmp0 = (icmp46_header_t *) udp0; - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, - sw_if_index0); - - proto0 = ~0; - proto0 = (ip0->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto0; - - /* Next configured feature, probably ip4-lookup */ - if (is_slow_path) - { - if (PREDICT_FALSE (proto0 == ~0)) - goto trace0; - - if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = icmp_in2out_slow_path - (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index); - goto trace0; - } - } - else - { - if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; - goto trace0; - } - } - - key0.addr = ip0->src_address; - key0.port = udp0->src_port; - key0.protocol = proto0; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0)) - { - if (is_slow_path) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - goto trace0; - - next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); - if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) - goto trace0; - } - else - { - next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; - goto trace0; - } - } - else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value0.value); - - old_addr0 = ip0->src_address.as_u32; - ip0->src_address = s0->out2in.addr; - new_addr0 = ip0->src_address.as_u32; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - src_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - old_port0 = tcp0->ports.src; - tcp0->ports.src = s0->out2in.port; - new_port0 = tcp0->ports.src; - - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - sum0 = ip_csum_update (sum0, old_port0, new_port0, - ip4_header_t /* cheat */, - length /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - old_port0 = udp0->src_port; - udp0->src_port = s0->out2in.port; - udp0->checksum = 0; - } - - /* Hairpinning */ - snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0); - - /* Accounting */ - s0->last_heard = now; - s0->total_pkts++; - s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s0)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_list_head_index, - s0->per_user_index); - } - - trace0: - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_in2out_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->is_slow_path = is_slow_path; - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->session_index = ~0; - if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; - } - - pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, stats_node_index, - SNAT_IN2OUT_ERROR_IN2OUT_PACKETS, - pkts_processed); - return frame->n_vectors; -} - -static uword -snat_in2out_fast_path_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */); -} - -VLIB_REGISTER_NODE (snat_in2out_node) = { - .function = snat_in2out_fast_path_fn, - .name = "snat-in2out", - .vector_size = sizeof (u32), - .format_trace = format_snat_in2out_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(snat_in2out_error_strings), - .error_strings = snat_in2out_error_strings, - - .runtime_data_bytes = sizeof (snat_runtime_t), - - .n_next_nodes = SNAT_IN2OUT_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [SNAT_IN2OUT_NEXT_DROP] = "error-drop", - [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup", - [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath", - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn); - -static uword -snat_in2out_slow_path_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */); -} - -VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = { - .function = snat_in2out_slow_path_fn, - .name = "snat-in2out-slowpath", - .vector_size = sizeof (u32), - .format_trace = format_snat_in2out_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(snat_in2out_error_strings), - .error_strings = snat_in2out_error_strings, - - .runtime_data_bytes = sizeof (snat_runtime_t), - - .n_next_nodes = SNAT_IN2OUT_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [SNAT_IN2OUT_NEXT_DROP] = "error-drop", - [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup", - [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath", - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn); - -static uword -snat_in2out_worker_handoff_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - snat_main_t *sm = &snat_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 n_left_from, *from, *to_next = 0; - static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index; - static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index - = 0; - vlib_frame_queue_elt_t *hf = 0; - vlib_frame_t *f = 0; - int i; - u32 n_left_to_next_worker = 0, *to_next_worker = 0; - u32 next_worker_index = 0; - u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number (); - - ASSERT (vec_len (sm->workers)); - - if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0)) - { - vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1); - - vec_validate_init_empty (congested_handoff_queue_by_worker_index, - sm->first_worker_index + sm->num_workers - 1, - (vlib_frame_queue_t *) (~0)); - } - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - - while (n_left_from > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 sw_if_index0; - u32 rx_fib_index0; - ip4_header_t * ip0; - snat_user_key_t key0; - clib_bihash_kv_8_8_t kv0, value0; - u8 do_handoff; - - bi0 = from[0]; - from += 1; - n_left_from -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); - - ip0 = vlib_buffer_get_current (b0); - - key0.addr = ip0->src_address; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - /* Ever heard of of the "user" before? */ - if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv0, &value0)) - { - /* No, assign next available worker (RR) */ - next_worker_index = sm->first_worker_index + - sm->workers[sm->next_worker++ % vec_len (sm->workers)]; - - /* add non-traslated packets worker lookup */ - kv0.value = next_worker_index; - clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1); - } - else - next_worker_index = value0.value; - - if (PREDICT_FALSE (next_worker_index != cpu_index)) - { - do_handoff = 1; - - if (next_worker_index != current_worker_index) - { - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index, - next_worker_index, - handoff_queue_elt_by_worker_index); - - n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors; - to_next_worker = &hf->buffer_index[hf->n_vectors]; - current_worker_index = next_worker_index; - } - - /* enqueue to correct worker thread */ - to_next_worker[0] = bi0; - to_next_worker++; - n_left_to_next_worker--; - - if (n_left_to_next_worker == 0) - { - hf->n_vectors = VLIB_FRAME_SIZE; - vlib_put_frame_queue_elt (hf); - current_worker_index = ~0; - handoff_queue_elt_by_worker_index[next_worker_index] = 0; - hf = 0; - } - } - else - { - do_handoff = 0; - /* if this is 1st frame */ - if (!f) - { - f = vlib_get_frame_to_node (vm, snat_in2out_node.index); - to_next = vlib_frame_vector_args (f); - } - - to_next[0] = bi0; - to_next += 1; - f->n_vectors++; - } - - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_in2out_worker_handoff_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->next_worker_index = next_worker_index; - t->do_handoff = do_handoff; - } - } - - if (f) - vlib_put_frame_to_node (vm, snat_in2out_node.index, f); - - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - /* Ship frames to the worker nodes */ - for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++) - { - if (handoff_queue_elt_by_worker_index[i]) - { - hf = handoff_queue_elt_by_worker_index[i]; - /* - * It works better to let the handoff node - * rate-adapt, always ship the handoff queue element. - */ - if (1 || hf->n_vectors == hf->last_n_vectors) - { - vlib_put_frame_queue_elt (hf); - handoff_queue_elt_by_worker_index[i] = 0; - } - else - hf->last_n_vectors = hf->n_vectors; - } - congested_handoff_queue_by_worker_index[i] = - (vlib_frame_queue_t *) (~0); - } - hf = 0; - current_worker_index = ~0; - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = { - .function = snat_in2out_worker_handoff_fn, - .name = "snat-in2out-worker-handoff", - .vector_size = sizeof (u32), - .format_trace = format_snat_in2out_worker_handoff_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_next_nodes = 1, - - .next_nodes = { - [0] = "error-drop", - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn); - -static inline u32 icmp_in2out_static_map (snat_main_t *sm, - vlib_buffer_t * b0, - ip4_header_t * ip0, - icmp46_header_t * icmp0, - u32 sw_if_index0, - vlib_node_runtime_t * node, - u32 next0, - u32 rx_fib_index0) -{ - snat_session_key_t key0, sm0; - icmp_echo_header_t *echo0; - u32 new_addr0, old_addr0; - u16 old_id0, new_id0; - ip_csum_t sum0; - snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; - - echo0 = (icmp_echo_header_t *)(icmp0+1); - - key0.addr = ip0->src_address; - key0.port = echo0->identifier; - key0.fib_index = rx_fib_index0; - - if (snat_static_mapping_match(sm, key0, &sm0, 0)) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - return next0; - - b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION]; - return SNAT_IN2OUT_NEXT_DROP; - } - - new_addr0 = sm0.addr.as_u32; - new_id0 = sm0.port; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; - old_addr0 = ip0->src_address.as_u32; - ip0->src_address.as_u32 = new_addr0; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - src_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_FALSE(new_id0 != echo0->identifier)) - { - old_id0 = echo0->identifier; - echo0->identifier = new_id0; - - sum0 = icmp0->checksum; - sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, - identifier); - icmp0->checksum = ip_csum_fold (sum0); - } - - return next0; -} - -static uword -snat_in2out_fast_static_map_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - u32 n_left_from, * from, * to_next; - snat_in2out_next_t next_index; - u32 pkts_processed = 0; - snat_main_t * sm = &snat_main; - snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; - u32 stats_node_index; - - stats_node_index = snat_in2out_fast_node.index; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, - to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t * b0; - u32 next0; - u32 sw_if_index0; - ip4_header_t * ip0; - ip_csum_t sum0; - u32 new_addr0, old_addr0; - u16 old_port0, new_port0; - udp_header_t * udp0; - tcp_header_t * tcp0; - icmp46_header_t * icmp0; - snat_session_key_t key0, sm0; - u32 proto0; - u32 rx_fib_index0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - next0 = SNAT_IN2OUT_NEXT_LOOKUP; - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - tcp0 = (tcp_header_t *) udp0; - icmp0 = (icmp46_header_t *) udp0; - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); - - proto0 = ~0; - proto0 = (ip0->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto0; - - if (PREDICT_FALSE (proto0 == ~0)) - goto trace0; - - if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - goto trace0; - - next0 = icmp_in2out_static_map - (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0); - goto trace0; - } - - key0.addr = ip0->src_address; - key0.port = udp0->src_port; - key0.fib_index = rx_fib_index0; - - if (snat_static_mapping_match(sm, key0, &sm0, 0)) - { - b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION]; - next0= SNAT_IN2OUT_NEXT_DROP; - goto trace0; - } - - new_addr0 = sm0.addr.as_u32; - new_port0 = sm0.port; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; - old_addr0 = ip0->src_address.as_u32; - ip0->src_address.as_u32 = new_addr0; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - src_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_FALSE(new_port0 != udp0->dst_port)) - { - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - old_port0 = tcp0->ports.src; - tcp0->ports.src = new_port0; - - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - sum0 = ip_csum_update (sum0, old_port0, new_port0, - ip4_header_t /* cheat */, - length /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - old_port0 = udp0->src_port; - udp0->src_port = new_port0; - udp0->checksum = 0; - } - } - else - { - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - } - - /* Hairpinning */ - snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0); - - trace0: - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_in2out_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - } - - pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, stats_node_index, - SNAT_IN2OUT_ERROR_IN2OUT_PACKETS, - pkts_processed); - return frame->n_vectors; -} - - -VLIB_REGISTER_NODE (snat_in2out_fast_node) = { - .function = snat_in2out_fast_static_map_fn, - .name = "snat-in2out-fast", - .vector_size = sizeof (u32), - .format_trace = format_snat_in2out_fast_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(snat_in2out_error_strings), - .error_strings = snat_in2out_error_strings, - - .runtime_data_bytes = sizeof (snat_runtime_t), - - .n_next_nodes = SNAT_IN2OUT_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [SNAT_IN2OUT_NEXT_DROP] = "error-drop", - [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup", - [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath", - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn); diff --git a/plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java b/plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java deleted file mode 100644 index 32165d96..00000000 --- a/plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.fd.vpp.jvpp.snat.test; - -import io.fd.vpp.jvpp.JVpp; -import io.fd.vpp.jvpp.JVppRegistry; -import io.fd.vpp.jvpp.JVppRegistryImpl; -import io.fd.vpp.jvpp.VppCallbackException; -import io.fd.vpp.jvpp.snat.JVppSnatImpl; -import io.fd.vpp.jvpp.snat.callback.SnatInterfaceAddDelFeatureCallback; -import io.fd.vpp.jvpp.snat.dto.SnatInterfaceAddDelFeature; -import io.fd.vpp.jvpp.snat.dto.SnatInterfaceAddDelFeatureReply; - -public class CallbackApiTest { - - static class TestCallback implements SnatInterfaceAddDelFeatureCallback { - - @Override - public void onSnatInterfaceAddDelFeatureReply(final SnatInterfaceAddDelFeatureReply msg) { - System.out.printf("Received SnatInterfaceAddDelFeatureReply: context=%d%n", - msg.context); - } - - @Override - public void onError(VppCallbackException ex) { - System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(), - ex.getCtxId(), ex.getErrorCode()); - } - } - - public static void main(String[] args) throws Exception { - testCallbackApi(); - } - - private static void testCallbackApi() throws Exception { - System.out.println("Testing Java callback API for snat plugin"); - try (final JVppRegistry registry = new JVppRegistryImpl("SnatCallbackApiTest"); - final JVpp jvpp = new JVppSnatImpl()) { - registry.register(jvpp, new TestCallback()); - - System.out.println("Sending SnatInterfaceAddDelFeature request..."); - SnatInterfaceAddDelFeature request = new SnatInterfaceAddDelFeature(); - request.isAdd = 1; - request.isInside = 1; - request.swIfIndex = 1; - final int result = jvpp.send(request); - System.out.printf("SnatInterfaceAddDelFeature send result = %d%n", result); - - Thread.sleep(1000); - - System.out.println("Disconnecting..."); - } - } -} diff --git a/plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/Readme.txt b/plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/Readme.txt deleted file mode 100644 index a2b0c41f..00000000 --- a/plugins/snat-plugin/snat/jvpp/io/fd/vpp/jvpp/snat/test/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/plugins/snat-plugin/jvpp-snat-1.0.jar io.fd.vpp.jvpp.snat.test.CallbackApiTest diff --git a/plugins/snat-plugin/snat/jvpp_snat.c b/plugins/snat-plugin/snat/jvpp_snat.c deleted file mode 100644 index fd72ddb1..00000000 --- a/plugins/snat-plugin/snat/jvpp_snat.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -#include -#include -#include - -#if VPPJNI_DEBUG == 1 - #define DEBUG_LOG(...) clib_warning(__VA_ARGS__) -#else - #define DEBUG_LOG(...) -#endif - -#include - -#include "snat/jvpp/io_fd_vpp_jvpp_snat_JVppSnatImpl.h" -#include "jvpp_snat.h" -#include "snat/jvpp/jvpp_snat_gen.h" - -/* - * Class: io_fd_vpp_jvpp_snat_JVppsnatImpl - * Method: init0 - * Signature: (JI)V - */ -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_init0 - (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) { - snat_main_t * plugin_main = &snat_main; - u8 * name; - clib_warning ("Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_init0"); - - plugin_main->my_client_index = my_client_index; - plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address; - - name = format (0, "snat_%08x%c", api_version, 0); - plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (plugin_main->msg_id_base == (u16) ~0) { - jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException"); - (*env)->ThrowNew(env, exClass, "snat plugin is not loaded in VPP"); - } else { - plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback); - plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback)); - - #define _(N,n) \ - vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_api_reply_handler; - #undef _ - } -} - -JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_close0 -(JNIEnv *env, jclass clazz) { - snat_main_t * plugin_main = &snat_main; - - // cleanup: - (*env)->DeleteGlobalRef(env, plugin_main->callbackClass); - (*env)->DeleteGlobalRef(env, plugin_main->callbackObject); - - plugin_main->callbackClass = NULL; - plugin_main->callbackObject = NULL; -} - -/* Attach thread to JVM and cache class references when initiating JVPP SNAT */ -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv* env; - - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return JNI_EVERSION; - } - - if (cache_class_references(env) != 0) { - clib_warning ("Failed to cache class references\n"); - return JNI_ERR; - } - - return JNI_VERSION_1_8; -} - -/* Clean up cached references when disposing JVPP SNAT */ -void JNI_OnUnload(JavaVM *vm, void *reserved) { - JNIEnv* env; - if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { - return; - } - delete_class_references(env); -} diff --git a/plugins/snat-plugin/snat/jvpp_snat.h b/plugins/snat-plugin/snat/jvpp_snat.h deleted file mode 100644 index 6426bda8..00000000 --- a/plugins/snat-plugin/snat/jvpp_snat.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_jvpp_snat_h__ -#define __included_jvpp_snat_h__ - -#include -#include -#include -#include -#include -#include - -/* Global state for JVPP-SNAT */ -typedef struct { - /* Base message index for the nsh plugin */ - u16 msg_id_base; - - /* Pointer to shared memory queue */ - unix_shared_memory_queue_t * vl_input_queue; - - /* VPP api client index */ - u32 my_client_index; - - /* Callback object and class references enabling asynchronous Java calls */ - jobject callbackObject; - jclass callbackClass; - -} snat_main_t; - -snat_main_t snat_main __attribute__((aligned (64))); - - -#endif /* __included_jvpp_snat_h__ */ diff --git a/plugins/snat-plugin/snat/out2in.c b/plugins/snat-plugin/snat/out2in.c deleted file mode 100644 index f1f4159c..00000000 --- a/plugins/snat-plugin/snat/out2in.c +++ /dev/null @@ -1,1261 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -typedef struct { - u32 sw_if_index; - u32 next_index; - u32 session_index; -} snat_out2in_trace_t; - -typedef struct { - u32 next_worker_index; - u8 do_handoff; -} snat_out2in_worker_handoff_trace_t; - -/* packet trace format function */ -static u8 * format_snat_out2in_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *); - - s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d", - t->sw_if_index, t->next_index, t->session_index); - return s; -} - -static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *); - - s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d", - t->sw_if_index, t->next_index); - return s; -} - -static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - snat_out2in_worker_handoff_trace_t * t = - va_arg (*args, snat_out2in_worker_handoff_trace_t *); - char * m; - - m = t->do_handoff ? "next worker" : "same worker"; - s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index); - - return s; -} - -vlib_node_registration_t snat_out2in_node; -vlib_node_registration_t snat_out2in_fast_node; -vlib_node_registration_t snat_out2in_worker_handoff_node; - -#define foreach_snat_out2in_error \ -_(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \ -_(OUT2IN_PACKETS, "Good out2in packets processed") \ -_(BAD_ICMP_TYPE, "icmp type not echo-reply") \ -_(NO_TRANSLATION, "No translation") - -typedef enum { -#define _(sym,str) SNAT_OUT2IN_ERROR_##sym, - foreach_snat_out2in_error -#undef _ - SNAT_OUT2IN_N_ERROR, -} snat_out2in_error_t; - -static char * snat_out2in_error_strings[] = { -#define _(sym,string) string, - foreach_snat_out2in_error -#undef _ -}; - -typedef enum { - SNAT_OUT2IN_NEXT_DROP, - SNAT_OUT2IN_NEXT_LOOKUP, - SNAT_OUT2IN_N_NEXT, -} snat_out2in_next_t; - -/** - * @brief Create session for static mapping. - * - * Create NAT session initiated by host from external network with static - * mapping. - * - * @param sm SNAT main. - * @param b0 Vlib buffer. - * @param in2out In2out SNAT session key. - * @param out2in Out2in SNAT session key. - * @param node Vlib node. - * - * @returns SNAT session if successfully created otherwise 0. - */ -static inline snat_session_t * -create_session_for_static_mapping (snat_main_t *sm, - vlib_buffer_t *b0, - snat_session_key_t in2out, - snat_session_key_t out2in, - vlib_node_runtime_t * node, - u32 cpu_index) -{ - snat_user_t *u; - snat_user_key_t user_key; - snat_session_t *s; - clib_bihash_kv_8_8_t kv0, value0; - dlist_elt_t * per_user_translation_list_elt; - dlist_elt_t * per_user_list_head_elt; - - user_key.addr = in2out.addr; - user_key.fib_index = in2out.fib_index; - kv0.key = user_key.as_u64; - - /* Ever heard of the "user" = inside ip4 address before? */ - if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) - { - /* no, make a new one */ - pool_get (sm->per_thread_data[cpu_index].users, u); - memset (u, 0, sizeof (*u)); - u->addr = in2out.addr; - - pool_get (sm->per_thread_data[cpu_index].list_pool, - per_user_list_head_elt); - - u->sessions_per_user_list_head_index = per_user_list_head_elt - - sm->per_thread_data[cpu_index].list_pool; - - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, - u->sessions_per_user_list_head_index); - - kv0.value = u - sm->per_thread_data[cpu_index].users; - - /* add user */ - clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); - - /* add non-traslated packets worker lookup */ - kv0.value = cpu_index; - clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1); - } - else - { - u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, - value0.value); - } - - pool_get (sm->per_thread_data[cpu_index].sessions, s); - memset (s, 0, sizeof (*s)); - - s->outside_address_index = ~0; - s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING; - u->nstaticsessions++; - - /* Create list elts */ - pool_get (sm->per_thread_data[cpu_index].list_pool, - per_user_translation_list_elt); - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, - per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); - - per_user_translation_list_elt->value = - s - sm->per_thread_data[cpu_index].sessions; - s->per_user_index = - per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool; - s->per_user_list_head_index = u->sessions_per_user_list_head_index; - - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s->per_user_list_head_index, - per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); - - s->in2out = in2out; - s->out2in = out2in; - s->in2out.protocol = out2in.protocol; - - /* Add to translation hashes */ - kv0.key = s->in2out.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; - if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) - clib_warning ("in2out key add failed"); - - kv0.key = s->out2in.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; - - if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) - clib_warning ("out2in key add failed"); - - return s; -} - -static inline u32 icmp_out2in_slow_path (snat_main_t *sm, - vlib_buffer_t * b0, - ip4_header_t * ip0, - icmp46_header_t * icmp0, - u32 sw_if_index0, - u32 rx_fib_index0, - vlib_node_runtime_t * node, - u32 next0, f64 now, - u32 cpu_index) -{ - snat_session_key_t key0, sm0; - icmp_echo_header_t *echo0; - clib_bihash_kv_8_8_t kv0, value0; - snat_session_t * s0; - u32 new_addr0, old_addr0; - u16 old_id0, new_id0; - ip_csum_t sum0; - snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; - - echo0 = (icmp_echo_header_t *)(icmp0+1); - - key0.addr = ip0->dst_address; - key0.port = echo0->identifier; - key0.protocol = SNAT_PROTOCOL_ICMP; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) - { - /* Try to match static mapping by external address and port, - destination address and port in packet */ - if (snat_static_mapping_match(sm, key0, &sm0, 1)) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - return next0; - - b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; - return SNAT_OUT2IN_NEXT_DROP; - } - - /* Create session initiated by host from external network */ - s0 = create_session_for_static_mapping(sm, b0, sm0, key0, - node, cpu_index); - if (!s0) - return SNAT_OUT2IN_NEXT_DROP; - } - else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value0.value); - - old_addr0 = ip0->dst_address.as_u32; - ip0->dst_address = s0->in2out.addr; - new_addr0 = ip0->dst_address.as_u32; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - old_id0 = echo0->identifier; - new_id0 = s0->in2out.port; - echo0->identifier = new_id0; - - sum0 = icmp0->checksum; - sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, - identifier); - icmp0->checksum = ip_csum_fold (sum0); - - /* Accounting */ - s0->last_heard = now; - s0->total_pkts++; - s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s0)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_list_head_index, - s0->per_user_index); - } - - return next0; -} - -static uword -snat_out2in_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - u32 n_left_from, * from, * to_next; - snat_out2in_next_t next_index; - u32 pkts_processed = 0; - snat_main_t * sm = &snat_main; - f64 now = vlib_time_now (vm); - u32 cpu_index = os_get_cpu_number (); - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, - to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - u32 bi0, bi1; - vlib_buffer_t * b0, * b1; - u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP; - u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP; - u32 sw_if_index0, sw_if_index1; - ip4_header_t * ip0, *ip1; - ip_csum_t sum0, sum1; - u32 new_addr0, old_addr0; - u16 new_port0, old_port0; - u32 new_addr1, old_addr1; - u16 new_port1, old_port1; - udp_header_t * udp0, * udp1; - tcp_header_t * tcp0, * tcp1; - icmp46_header_t * icmp0, * icmp1; - snat_session_key_t key0, key1, sm0, sm1; - u32 rx_fib_index0, rx_fib_index1; - u32 proto0, proto1; - snat_session_t * s0 = 0, * s1 = 0; - clib_bihash_kv_8_8_t kv0, kv1, value0, value1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t * p2, * p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); - } - - /* speculatively enqueue b0 and b1 to the current next frame */ - to_next[0] = bi0 = from[0]; - to_next[1] = bi1 = from[1]; - from += 2; - to_next += 2; - n_left_from -= 2; - n_left_to_next -= 2; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - tcp0 = (tcp_header_t *) udp0; - icmp0 = (icmp46_header_t *) udp0; - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, - sw_if_index0); - - proto0 = ~0; - proto0 = (ip0->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto0; - - if (PREDICT_FALSE (proto0 == ~0)) - goto trace0; - - if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = icmp_out2in_slow_path - (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index); - goto trace0; - } - - key0.addr = ip0->dst_address; - key0.port = udp0->dst_port; - key0.protocol = proto0; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) - { - /* Try to match static mapping by external address and port, - destination address and port in packet */ - if (snat_static_mapping_match(sm, key0, &sm0, 1)) - { - b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; - goto trace0; - } - - /* Create session initiated by host from external network */ - s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, - cpu_index); - if (!s0) - goto trace0; - } - else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value0.value); - - old_addr0 = ip0->dst_address.as_u32; - ip0->dst_address = s0->in2out.addr; - new_addr0 = ip0->dst_address.as_u32; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - old_port0 = tcp0->ports.dst; - tcp0->ports.dst = s0->in2out.port; - new_port0 = tcp0->ports.dst; - - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - - sum0 = ip_csum_update (sum0, old_port0, new_port0, - ip4_header_t /* cheat */, - length /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - old_port0 = udp0->dst_port; - udp0->dst_port = s0->in2out.port; - udp0->checksum = 0; - } - - /* Accounting */ - s0->last_heard = now; - s0->total_pkts++; - s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s0)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_list_head_index, - s0->per_user_index); - } - trace0: - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_out2in_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->session_index = ~0; - if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; - } - - pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; - - - ip1 = vlib_buffer_get_current (b1); - udp1 = ip4_next_header (ip1); - tcp1 = (tcp_header_t *) udp1; - icmp1 = (icmp46_header_t *) udp1; - - sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX]; - rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, - sw_if_index1); - - proto1 = ~0; - proto1 = (ip1->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto1; - proto1 = (ip1->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto1; - proto1 = (ip1->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto1; - - if (PREDICT_FALSE (proto1 == ~0)) - goto trace1; - - if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP)) - { - next1 = icmp_out2in_slow_path - (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, - next1, now, cpu_index); - goto trace1; - } - - key1.addr = ip1->dst_address; - key1.port = udp1->dst_port; - key1.protocol = proto1; - key1.fib_index = rx_fib_index1; - - kv1.key = key1.as_u64; - - if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1)) - { - /* Try to match static mapping by external address and port, - destination address and port in packet */ - if (snat_static_mapping_match(sm, key1, &sm1, 1)) - { - b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; - goto trace1; - } - - /* Create session initiated by host from external network */ - s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node, - cpu_index); - if (!s1) - goto trace1; - } - else - s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value1.value); - - old_addr1 = ip1->dst_address.as_u32; - ip1->dst_address = s1->in2out.addr; - new_addr1 = ip1->dst_address.as_u32; - vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index; - - sum1 = ip1->checksum; - sum1 = ip_csum_update (sum1, old_addr1, new_addr1, - ip4_header_t, - dst_address /* changed member */); - ip1->checksum = ip_csum_fold (sum1); - - if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP)) - { - old_port1 = tcp1->ports.dst; - tcp1->ports.dst = s1->in2out.port; - new_port1 = tcp1->ports.dst; - - sum1 = tcp1->checksum; - sum1 = ip_csum_update (sum1, old_addr1, new_addr1, - ip4_header_t, - dst_address /* changed member */); - - sum1 = ip_csum_update (sum1, old_port1, new_port1, - ip4_header_t /* cheat */, - length /* changed member */); - tcp1->checksum = ip_csum_fold(sum1); - } - else - { - old_port1 = udp1->dst_port; - udp1->dst_port = s1->in2out.port; - udp1->checksum = 0; - } - - /* Accounting */ - s1->last_heard = now; - s1->total_pkts++; - s1->total_bytes += vlib_buffer_length_in_chain (vm, b1); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s1)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s1->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s1->per_user_list_head_index, - s1->per_user_index); - } - trace1: - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b1->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_out2in_trace_t *t = - vlib_add_trace (vm, node, b1, sizeof (*t)); - t->sw_if_index = sw_if_index1; - t->next_index = next1; - t->session_index = ~0; - if (s1) - t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; - } - - pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP; - - /* verify speculative enqueues, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, - to_next, n_left_to_next, - bi0, bi1, next0, next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t * b0; - u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP; - u32 sw_if_index0; - ip4_header_t * ip0; - ip_csum_t sum0; - u32 new_addr0, old_addr0; - u16 new_port0, old_port0; - udp_header_t * udp0; - tcp_header_t * tcp0; - icmp46_header_t * icmp0; - snat_session_key_t key0, sm0; - u32 rx_fib_index0; - u32 proto0; - snat_session_t * s0 = 0; - clib_bihash_kv_8_8_t kv0, value0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - tcp0 = (tcp_header_t *) udp0; - icmp0 = (icmp46_header_t *) udp0; - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, - sw_if_index0); - - proto0 = ~0; - proto0 = (ip0->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto0; - - if (PREDICT_FALSE (proto0 == ~0)) - goto trace00; - - if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = icmp_out2in_slow_path - (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index); - goto trace00; - } - - key0.addr = ip0->dst_address; - key0.port = udp0->dst_port; - key0.protocol = proto0; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) - { - /* Try to match static mapping by external address and port, - destination address and port in packet */ - if (snat_static_mapping_match(sm, key0, &sm0, 1)) - { - b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; - goto trace00; - } - - /* Create session initiated by host from external network */ - s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, - cpu_index); - if (!s0) - goto trace00; - } - else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, - value0.value); - - old_addr0 = ip0->dst_address.as_u32; - ip0->dst_address = s0->in2out.addr; - new_addr0 = ip0->dst_address.as_u32; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - old_port0 = tcp0->ports.dst; - tcp0->ports.dst = s0->in2out.port; - new_port0 = tcp0->ports.dst; - - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - - sum0 = ip_csum_update (sum0, old_port0, new_port0, - ip4_header_t /* cheat */, - length /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - old_port0 = udp0->dst_port; - udp0->dst_port = s0->in2out.port; - udp0->checksum = 0; - } - - /* Accounting */ - s0->last_heard = now; - s0->total_pkts++; - s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); - /* Per-user LRU list maintenance for dynamic translation */ - if (!snat_is_session_static (s0)) - { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, - s0->per_user_list_head_index, - s0->per_user_index); - } - trace00: - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_out2in_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - t->session_index = ~0; - if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; - } - - pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, snat_out2in_node.index, - SNAT_OUT2IN_ERROR_OUT2IN_PACKETS, - pkts_processed); - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (snat_out2in_node) = { - .function = snat_out2in_node_fn, - .name = "snat-out2in", - .vector_size = sizeof (u32), - .format_trace = format_snat_out2in_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(snat_out2in_error_strings), - .error_strings = snat_out2in_error_strings, - - .runtime_data_bytes = sizeof (snat_runtime_t), - - .n_next_nodes = SNAT_OUT2IN_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [SNAT_OUT2IN_NEXT_DROP] = "error-drop", - [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", - }, -}; -VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn); - -static uword -snat_out2in_worker_handoff_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - snat_main_t *sm = &snat_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 n_left_from, *from, *to_next = 0; - static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index; - static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index - = 0; - vlib_frame_queue_elt_t *hf = 0; - vlib_frame_t *f = 0; - int i; - u32 n_left_to_next_worker = 0, *to_next_worker = 0; - u32 next_worker_index = 0; - u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number (); - - ASSERT (vec_len (sm->workers)); - - if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0)) - { - vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1); - - vec_validate_init_empty (congested_handoff_queue_by_worker_index, - sm->first_worker_index + sm->num_workers - 1, - (vlib_frame_queue_t *) (~0)); - } - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - - while (n_left_from > 0) - { - u32 bi0; - vlib_buffer_t *b0; - u32 sw_if_index0; - u32 rx_fib_index0; - ip4_header_t * ip0; - udp_header_t * udp0; - snat_static_mapping_key_t key0; - clib_bihash_kv_8_8_t kv0, value0; - u8 do_handoff; - - bi0 = from[0]; - from += 1; - n_left_from -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - - key0.addr = ip0->dst_address; - key0.port = udp0->dst_port; - key0.fib_index = rx_fib_index0; - - kv0.key = key0.as_u64; - - /* Ever heard of of the "user" before? */ - if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0)) - { - key0.port = 0; - kv0.key = key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0)) - { - /* No, assign next available worker (RR) */ - next_worker_index = sm->first_worker_index + - sm->workers[sm->next_worker++ % vec_len (sm->workers)]; - } - else - { - /* Static mapping without port */ - next_worker_index = value0.value; - } - - /* Add to translated packets worker lookup */ - kv0.value = next_worker_index; - clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1); - } - else - next_worker_index = value0.value; - - if (PREDICT_FALSE (next_worker_index != cpu_index)) - { - do_handoff = 1; - - if (next_worker_index != current_worker_index) - { - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index, - next_worker_index, - handoff_queue_elt_by_worker_index); - - n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors; - to_next_worker = &hf->buffer_index[hf->n_vectors]; - current_worker_index = next_worker_index; - } - - /* enqueue to correct worker thread */ - to_next_worker[0] = bi0; - to_next_worker++; - n_left_to_next_worker--; - - if (n_left_to_next_worker == 0) - { - hf->n_vectors = VLIB_FRAME_SIZE; - vlib_put_frame_queue_elt (hf); - current_worker_index = ~0; - handoff_queue_elt_by_worker_index[next_worker_index] = 0; - hf = 0; - } - } - else - { - do_handoff = 0; - /* if this is 1st frame */ - if (!f) - { - f = vlib_get_frame_to_node (vm, snat_out2in_node.index); - to_next = vlib_frame_vector_args (f); - } - - to_next[0] = bi0; - to_next += 1; - f->n_vectors++; - } - - if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_out2in_worker_handoff_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->next_worker_index = next_worker_index; - t->do_handoff = do_handoff; - } - } - - if (f) - vlib_put_frame_to_node (vm, snat_out2in_node.index, f); - - if (hf) - hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; - - /* Ship frames to the worker nodes */ - for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++) - { - if (handoff_queue_elt_by_worker_index[i]) - { - hf = handoff_queue_elt_by_worker_index[i]; - /* - * It works better to let the handoff node - * rate-adapt, always ship the handoff queue element. - */ - if (1 || hf->n_vectors == hf->last_n_vectors) - { - vlib_put_frame_queue_elt (hf); - handoff_queue_elt_by_worker_index[i] = 0; - } - else - hf->last_n_vectors = hf->n_vectors; - } - congested_handoff_queue_by_worker_index[i] = - (vlib_frame_queue_t *) (~0); - } - hf = 0; - current_worker_index = ~0; - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = { - .function = snat_out2in_worker_handoff_fn, - .name = "snat-out2in-worker-handoff", - .vector_size = sizeof (u32), - .format_trace = format_snat_out2in_worker_handoff_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_next_nodes = 1, - - .next_nodes = { - [0] = "error-drop", - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn); - -static inline u32 icmp_out2in_fast (snat_main_t *sm, - vlib_buffer_t * b0, - ip4_header_t * ip0, - icmp46_header_t * icmp0, - u32 sw_if_index0, - vlib_node_runtime_t * node, - u32 next0, - u32 rx_fib_index0) -{ - snat_session_key_t key0, sm0; - icmp_echo_header_t *echo0; - u32 new_addr0, old_addr0; - u16 old_id0, new_id0; - ip_csum_t sum0; - snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; - - echo0 = (icmp_echo_header_t *)(icmp0+1); - - key0.addr = ip0->dst_address; - key0.port = echo0->identifier; - key0.fib_index = rx_fib_index0; - - if (snat_static_mapping_match(sm, key0, &sm0, 1)) - { - ip4_address_t * first_int_addr; - - if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) - { - first_int_addr = - ip4_interface_first_address (sm->ip4_main, sw_if_index0, - 0 /* just want the address */); - rt->cached_sw_if_index = sw_if_index0; - rt->cached_ip4_address = first_int_addr->as_u32; - } - - /* Don't NAT packet aimed at the intfc address */ - if (PREDICT_FALSE(ip0->dst_address.as_u32 == - rt->cached_ip4_address)) - return next0; - - b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; - return SNAT_OUT2IN_NEXT_DROP; - } - - new_addr0 = sm0.addr.as_u32; - new_id0 = sm0.port; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; - - old_addr0 = ip0->dst_address.as_u32; - ip0->dst_address.as_u32 = new_addr0; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_FALSE(new_id0 != echo0->identifier)) - { - old_id0 = echo0->identifier; - echo0->identifier = new_id0; - - sum0 = icmp0->checksum; - sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, - identifier); - icmp0->checksum = ip_csum_fold (sum0); - } - - return next0; -} - -static uword -snat_out2in_fast_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - u32 n_left_from, * from, * to_next; - snat_out2in_next_t next_index; - u32 pkts_processed = 0; - snat_main_t * sm = &snat_main; - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, - to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0; - vlib_buffer_t * b0; - u32 next0 = SNAT_OUT2IN_NEXT_DROP; - u32 sw_if_index0; - ip4_header_t * ip0; - ip_csum_t sum0; - u32 new_addr0, old_addr0; - u16 new_port0, old_port0; - udp_header_t * udp0; - tcp_header_t * tcp0; - icmp46_header_t * icmp0; - snat_session_key_t key0, sm0; - u32 proto0; - u32 rx_fib_index0; - - /* speculatively enqueue b0 to the current next frame */ - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - ip0 = vlib_buffer_get_current (b0); - udp0 = ip4_next_header (ip0); - tcp0 = (tcp_header_t *) udp0; - icmp0 = (icmp46_header_t *) udp0; - - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; - rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); - - vnet_feature_next (sw_if_index0, &next0, b0); - - proto0 = ~0; - proto0 = (ip0->protocol == IP_PROTOCOL_UDP) - ? SNAT_PROTOCOL_UDP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_TCP) - ? SNAT_PROTOCOL_TCP : proto0; - proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) - ? SNAT_PROTOCOL_ICMP : proto0; - - if (PREDICT_FALSE (proto0 == ~0)) - goto trace00; - - if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) - { - next0 = icmp_out2in_fast - (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0); - goto trace00; - } - - key0.addr = ip0->dst_address; - key0.port = udp0->dst_port; - key0.fib_index = rx_fib_index0; - - if (snat_static_mapping_match(sm, key0, &sm0, 1)) - { - b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; - goto trace00; - } - - new_addr0 = sm0.addr.as_u32; - new_port0 = sm0.port; - vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; - old_addr0 = ip0->dst_address.as_u32; - ip0->dst_address.as_u32 = new_addr0; - - sum0 = ip0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - ip0->checksum = ip_csum_fold (sum0); - - if (PREDICT_FALSE(new_port0 != udp0->dst_port)) - { - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - old_port0 = tcp0->ports.dst; - tcp0->ports.dst = new_port0; - - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - - sum0 = ip_csum_update (sum0, old_port0, new_port0, - ip4_header_t /* cheat */, - length /* changed member */); - tcp0->checksum = ip_csum_fold(sum0); - } - else - { - old_port0 = udp0->dst_port; - udp0->dst_port = new_port0; - udp0->checksum = 0; - } - } - else - { - if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) - { - sum0 = tcp0->checksum; - sum0 = ip_csum_update (sum0, old_addr0, new_addr0, - ip4_header_t, - dst_address /* changed member */); - - tcp0->checksum = ip_csum_fold(sum0); - } - } - - trace00: - - if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) - && (b0->flags & VLIB_BUFFER_IS_TRACED))) - { - snat_out2in_trace_t *t = - vlib_add_trace (vm, node, b0, sizeof (*t)); - t->sw_if_index = sw_if_index0; - t->next_index = next0; - } - - pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; - - /* verify speculative enqueue, maybe switch current next frame */ - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } - - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, snat_out2in_fast_node.index, - SNAT_OUT2IN_ERROR_OUT2IN_PACKETS, - pkts_processed); - return frame->n_vectors; -} - -VLIB_REGISTER_NODE (snat_out2in_fast_node) = { - .function = snat_out2in_fast_node_fn, - .name = "snat-out2in-fast", - .vector_size = sizeof (u32), - .format_trace = format_snat_out2in_fast_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(snat_out2in_error_strings), - .error_strings = snat_out2in_error_strings, - - .runtime_data_bytes = sizeof (snat_runtime_t), - - .n_next_nodes = SNAT_OUT2IN_N_NEXT, - - /* edit / add dispositions here */ - .next_nodes = { - [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", - [SNAT_OUT2IN_NEXT_DROP] = "error-drop", - }, -}; -VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn); diff --git a/plugins/snat-plugin/snat/snat.api b/plugins/snat-plugin/snat/snat.api deleted file mode 100644 index a191eed5..00000000 --- a/plugins/snat-plugin/snat/snat.api +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * @file snat.api - * @brief VPP control-plane API messages. - * - * This file defines VPP control-plane API messages which are generally - * called through a shared memory interface. - */ - -/** \brief Add/del S-NAT address range - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param is_ip4 - 1 if address type is IPv4 - @first_ip_address - first IP address - @last_ip_address - last IP address - @is_add - 1 if add, 0 if delete -*/ -define snat_add_address_range { - u32 client_index; - u32 context; - u8 is_ip4; - u8 first_ip_address[16]; - u8 last_ip_address[16]; - u8 is_add; -}; - -/** \brief Add S-NAT address range reply - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return code -*/ -define snat_add_address_range_reply { - u32 context; - i32 retval; -}; - -/** \brief Dump S-NAT addresses - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define snat_address_dump { - u32 client_index; - u32 context; -}; - -/** \brief S-NAT address details response - @param context - sender context, to match reply w/ request - @param is_ip4 - 1 if address type is IPv4 - @param ip_address - IP address -*/ -define snat_address_details { - u32 context; - u8 is_ip4; - u8 ip_address[16]; -}; - -/** \brief Enable/disable S-NAT feature on the interface - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param is_add - 1 if add, 0 if delete - @param is_inside - 1 if inside, 0 if outside - @param sw_if_index - software index of the interface -*/ -define snat_interface_add_del_feature { - u32 client_index; - u32 context; - u8 is_add; - u8 is_inside; - u32 sw_if_index; -}; - -/** \brief Enable/disable S-NAT feature on the interface reply - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return code -*/ -define snat_interface_add_del_feature_reply { - u32 context; - i32 retval; -}; - -/** \brief Dump interfaces with S-NAT feature - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define snat_interface_dump { - u32 client_index; - u32 context; -}; - -/** \brief S-NAT interface details response - @param context - sender context, to match reply w/ request - @param is_inside - 1 if inside, 0 if outside - @param sw_if_index - software index of the interface -*/ -define snat_interface_details { - u32 context; - u8 is_inside; - u32 sw_if_index; -}; - -/** \brief Add/delete S-NAT static mapping - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param is_add - 1 if add, 0 if delete - @param is_ip4 - 1 if address type is IPv4 - @param addr_only - 1 if address only mapping - @param local_ip_address - local IP address - @param external_ip_address - external IP address - @param local_port - local port number - @param external_port - external port number - @param vfr_id - VRF ID -*/ -define snat_add_static_mapping { - u32 client_index; - u32 context; - u8 is_add; - u8 is_ip4; - u8 addr_only; - u8 local_ip_address[16]; - u8 external_ip_address[16]; - u16 local_port; - u16 external_port; - u32 vrf_id; -}; - -/** \brief Add/delete S-NAT static mapping reply - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return code -*/ -define snat_add_static_mapping_reply { - u32 context; - i32 retval; -}; - -/** \brief Dump S-NAT static mappings - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define snat_static_mapping_dump { - u32 client_index; - u32 context; -}; - -/** \brief S-NAT static mapping details response - @param context - sender context, to match reply w/ request - @param is_ip4 - 1 if address type is IPv4 - @param addr_only - 1 if address only mapping - @param local_ip_address - local IP address - @param external_ip_address - external IP address - @param local_port - local port number - @param external_port - external port number - @param vfr_id - VRF ID -*/ -define snat_static_mapping_details { - u32 context; - u8 is_ip4; - u8 addr_only; - u8 local_ip_address[16]; - u8 external_ip_address[16]; - u16 local_port; - u16 external_port; - u32 vrf_id; -}; - -/** \brief Control ping from client to api server request - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define snat_control_ping -{ - u32 client_index; - u32 context; -}; - -/** \brief Control ping from the client to the server response - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return code for the request - @param vpe_pid - the pid of the vpe, returned by the server -*/ -define snat_control_ping_reply -{ - u32 context; - i32 retval; - u32 client_index; - u32 vpe_pid; -}; - -/** \brief Show S-NAT plugin startup config - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define snat_show_config -{ - u32 client_index; - u32 context; -}; - -/** \brief Show S-NAT plugin startup config reply - @param context - sender context, to match reply w/ request - @param retval - return code for the request - @param static_mapping_only - if 1 dynamic translations disabled - @param static_mapping_connection_tracking - if 1 create session data - @param translation_buckets - number of translation hash buckets - @param translation_memory_size - translation hash memory size - @param user_buckets - number of user hash buckets - @param user_memory_size - user hash memory size - @param max_translations_per_user - maximum number of translations per user - @param outside_vrf_id - outside VRF id - @param inside_vrf_id - default inside VRF id -*/ -define snat_show_config_reply -{ - u32 context; - i32 retval; - u8 static_mapping_only; - u8 static_mapping_connection_tracking; - u32 translation_buckets; - u32 translation_memory_size; - u32 user_buckets; - u32 user_memory_size; - u32 max_translations_per_user; - u32 outside_vrf_id; - u32 inside_vrf_id; -}; - -/** \brief Set S-NAT workers - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param worker_mask - S-NAT workers mask -*/ -define snat_set_workers { - u32 client_index; - u32 context; - u64 worker_mask; -}; - -/** \brief Set S-NAT workers reply - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param retval - return code -*/ -define snat_set_workers_reply { - u32 context; - i32 retval; -}; - -/** \brief Dump S-NAT workers - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request -*/ -define snat_worker_dump { - u32 client_index; - u32 context; -}; - -/** \brief S-NAT workers details response - @param context - sender context, to match reply w/ request - @param worker_index - worker index - @param lcore_id - lcore ID - @param name - worker name -*/ -define snat_worker_details { - u32 context; - u32 worker_index; - u32 lcore_id; - u8 name[64]; -}; diff --git a/plugins/snat-plugin/snat/snat.c b/plugins/snat-plugin/snat/snat.c deleted file mode 100644 index bc995684..00000000 --- a/plugins/snat-plugin/snat/snat.c +++ /dev/null @@ -1,1957 +0,0 @@ -/* - * snat.c - simple nat plugin - * - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include -#include - -snat_main_t snat_main; - -/* define message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* define generated endian-swappers */ -#define vl_endianfun -#include -#undef vl_endianfun - -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) - -/* Get the API version number */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* Macro to finish up custom dump fns */ -#define FINISH \ - vec_add1 (s, 0); \ - vl_print (handle, (char *)s); \ - vec_free (s); \ - return handle; - -/* - * A handy macro to set up a message reply. - * Assumes that the following variables are available: - * mp - pointer to request message - * rmp - pointer to reply message type - * rv - return value - */ - -#define REPLY_MACRO(t) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - -#define REPLY_MACRO2(t, body) \ -do { \ - unix_shared_memory_queue_t * q = \ - vl_api_client_index_to_input_queue (mp->client_index); \ - if (!q) \ - return; \ - \ - rmp = vl_msg_api_alloc (sizeof (*rmp)); \ - rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ - rmp->context = mp->context; \ - rmp->retval = ntohl(rv); \ - do {body;} while (0); \ - vl_msg_api_send_shmem (q, (u8 *)&rmp); \ -} while(0); - - -/* Hook up input features */ -VNET_FEATURE_INIT (ip4_snat_in2out, static) = { - .arc_name = "ip4-unicast", - .node_name = "snat-in2out", - .runs_before = VNET_FEATURES ("snat-out2in"), -}; -VNET_FEATURE_INIT (ip4_snat_out2in, static) = { - .arc_name = "ip4-unicast", - .node_name = "snat-out2in", - .runs_before = VNET_FEATURES ("ip4-lookup"), -}; -VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = { - .arc_name = "ip4-unicast", - .node_name = "snat-in2out-worker-handoff", - .runs_before = VNET_FEATURES ("snat-out2in-worker-handoff"), -}; -VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = { - .arc_name = "ip4-unicast", - .node_name = "snat-out2in-worker-handoff", - .runs_before = VNET_FEATURES ("ip4-lookup"), -}; -VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = { - .arc_name = "ip4-unicast", - .node_name = "snat-in2out-fast", - .runs_before = VNET_FEATURES ("snat-out2in-fast"), -}; -VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = { - .arc_name = "ip4-unicast", - .node_name = "snat-out2in-fast", - .runs_before = VNET_FEATURES ("ip4-lookup"), -}; - - -/* - * This routine exists to convince the vlib plugin framework that - * we haven't accidentally copied a random .dll into the plugin directory. - * - * Also collects global variable pointers passed from the vpp engine - */ - -clib_error_t * -vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, - int from_early_init) -{ - snat_main_t * sm = &snat_main; - clib_error_t * error = 0; - - sm->vlib_main = vm; - sm->vnet_main = h->vnet_main; - sm->ethernet_main = h->ethernet_main; - - return error; -} - -/*$$$$$ move to an installed header file */ -#if (1 || CLIB_DEBUG > 0) /* "trust, but verify" */ - -#define VALIDATE_SW_IF_INDEX(mp) \ - do { u32 __sw_if_index = ntohl(mp->sw_if_index); \ - vnet_main_t *__vnm = vnet_get_main(); \ - if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ - __sw_if_index)) { \ - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ - goto bad_sw_if_index; \ - } \ -} while(0); - -#define BAD_SW_IF_INDEX_LABEL \ -do { \ -bad_sw_if_index: \ - ; \ -} while (0); - -#define VALIDATE_RX_SW_IF_INDEX(mp) \ - do { u32 __rx_sw_if_index = ntohl(mp->rx_sw_if_index); \ - vnet_main_t *__vnm = vnet_get_main(); \ - if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ - __rx_sw_if_index)) { \ - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ - goto bad_rx_sw_if_index; \ - } \ -} while(0); - -#define BAD_RX_SW_IF_INDEX_LABEL \ -do { \ -bad_rx_sw_if_index: \ - ; \ -} while (0); - -#define VALIDATE_TX_SW_IF_INDEX(mp) \ - do { u32 __tx_sw_if_index = ntohl(mp->tx_sw_if_index); \ - vnet_main_t *__vnm = vnet_get_main(); \ - if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ - __tx_sw_if_index)) { \ - rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ - goto bad_tx_sw_if_index; \ - } \ -} while(0); - -#define BAD_TX_SW_IF_INDEX_LABEL \ -do { \ -bad_tx_sw_if_index: \ - ; \ -} while (0); - -#else - -#define VALIDATE_SW_IF_INDEX(mp) -#define BAD_SW_IF_INDEX_LABEL -#define VALIDATE_RX_SW_IF_INDEX(mp) -#define BAD_RX_SW_IF_INDEX_LABEL -#define VALIDATE_TX_SW_IF_INDEX(mp) -#define BAD_TX_SW_IF_INDEX_LABEL - -#endif /* CLIB_DEBUG > 0 */ - -void snat_add_address (snat_main_t *sm, ip4_address_t *addr) -{ - snat_address_t * ap; - - /* Check if address already exists */ - vec_foreach (ap, sm->addresses) - { - if (ap->addr.as_u32 == addr->as_u32) - return; - } - - vec_add2 (sm->addresses, ap, 1); - ap->addr = *addr; - clib_bitmap_alloc (ap->busy_port_bitmap, 65535); -} - -static int is_snat_address_used_in_static_mapping (snat_main_t *sm, - ip4_address_t addr) -{ - snat_static_mapping_t *m; - pool_foreach (m, sm->static_mappings, - ({ - if (m->external_addr.as_u32 == addr.as_u32) - return 1; - })); - - return 0; -} - -int snat_del_address (snat_main_t *sm, ip4_address_t addr) -{ - snat_address_t *a = 0; - snat_session_t *ses; - u32 *ses_to_be_removed = 0, *ses_index; - clib_bihash_kv_8_8_t kv, value; - snat_user_key_t user_key; - snat_user_t *u; - snat_main_per_thread_data_t *tsm; - - int i; - - /* Find SNAT address */ - for (i=0; i < vec_len (sm->addresses); i++) - { - if (sm->addresses[i].addr.as_u32 == addr.as_u32) - { - a = sm->addresses + i; - break; - } - } - if (!a) - return VNET_API_ERROR_NO_SUCH_ENTRY; - - /* Check if address is used in some static mapping */ - if (is_snat_address_used_in_static_mapping(sm, addr)) - { - clib_warning ("address used in static mapping"); - return VNET_API_ERROR_UNSPECIFIED; - } - - /* Delete sessions using address */ - if (a->busy_ports) - { - vec_foreach (tsm, sm->per_thread_data) - { - pool_foreach (ses, tsm->sessions, ({ - if (ses->out2in.addr.as_u32 == addr.as_u32) - { - vec_add1 (ses_to_be_removed, ses - tsm->sessions); - kv.key = ses->in2out.as_u64; - clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0); - kv.key = ses->out2in.as_u64; - clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0); - clib_dlist_remove (tsm->list_pool, ses->per_user_index); - user_key.addr = ses->in2out.addr; - user_key.fib_index = ses->in2out.fib_index; - kv.key = user_key.as_u64; - if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value)) - { - u = pool_elt_at_index (tsm->users, value.value); - u->nsessions--; - } - } - })); - - vec_foreach (ses_index, ses_to_be_removed) - pool_put_index (tsm->sessions, ses_index[0]); - - vec_free (ses_to_be_removed); - } - } - - vec_del1 (sm->addresses, i); - - return 0; -} - -static void increment_v4_address (ip4_address_t * a) -{ - u32 v; - - v = clib_net_to_host_u32(a->as_u32) + 1; - a->as_u32 = clib_host_to_net_u32(v); -} - -/** - * @brief Add static mapping. - * - * Create static mapping between local addr+port and external addr+port. - * - * @param l_addr Local IPv4 address. - * @param e_addr External IPv4 address. - * @param l_port Local port number. - * @param e_port External port number. - * @param vrf_id VRF ID. - * @param addr_only If 0 address port and pair mapping, otherwise address only. - * @param is_add If 0 delete static mapping, otherwise add. - * - * @returns - */ -int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr, - u16 l_port, u16 e_port, u32 vrf_id, int addr_only, - int is_add) -{ - snat_main_t * sm = &snat_main; - snat_static_mapping_t *m; - snat_static_mapping_key_t m_key; - clib_bihash_kv_8_8_t kv, value; - snat_address_t *a = 0; - u32 fib_index = ~0; - uword * p; - int i; - - /* If outside FIB index is not resolved yet */ - if (sm->outside_fib_index == ~0) - { - p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id); - if (!p) - return VNET_API_ERROR_NO_SUCH_FIB; - sm->outside_fib_index = p[0]; - } - - m_key.addr = e_addr; - m_key.port = addr_only ? 0 : e_port; - m_key.fib_index = sm->outside_fib_index; - kv.key = m_key.as_u64; - if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value)) - m = 0; - else - m = pool_elt_at_index (sm->static_mappings, value.value); - - if (is_add) - { - if (m) - return VNET_API_ERROR_VALUE_EXIST; - - /* Convert VRF id to FIB index */ - if (vrf_id != ~0) - { - p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id); - if (!p) - return VNET_API_ERROR_NO_SUCH_FIB; - fib_index = p[0]; - } - /* If not specified use inside VRF id from SNAT plugin startup config */ - else - { - if (sm->inside_fib_index == ~0) - { - p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->inside_vrf_id); - if (!p) - return VNET_API_ERROR_NO_SUCH_FIB; - fib_index = p[0]; - sm->inside_fib_index = fib_index; - } - else - fib_index = sm->inside_fib_index; - - vrf_id = sm->inside_vrf_id; - } - - /* Find external address in allocated addresses and reserve port for - address and port pair mapping when dynamic translations enabled */ - if (!addr_only && !(sm->static_mapping_only)) - { - for (i = 0; i < vec_len (sm->addresses); i++) - { - if (sm->addresses[i].addr.as_u32 == e_addr.as_u32) - { - a = sm->addresses + i; - /* External port must be unused */ - if (clib_bitmap_get_no_check (a->busy_port_bitmap, e_port)) - return VNET_API_ERROR_INVALID_VALUE; - clib_bitmap_set_no_check (a->busy_port_bitmap, e_port, 1); - if (e_port > 1024) - a->busy_ports++; - - break; - } - } - /* External address must be allocated */ - if (!a) - return VNET_API_ERROR_NO_SUCH_ENTRY; - } - - pool_get (sm->static_mappings, m); - memset (m, 0, sizeof (*m)); - m->local_addr = l_addr; - m->external_addr = e_addr; - m->addr_only = addr_only; - m->vrf_id = vrf_id; - m->fib_index = fib_index; - if (!addr_only) - { - m->local_port = l_port; - m->external_port = e_port; - } - - m_key.addr = m->local_addr; - m_key.port = m->local_port; - m_key.fib_index = m->fib_index; - kv.key = m_key.as_u64; - kv.value = m - sm->static_mappings; - clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1); - - m_key.addr = m->external_addr; - m_key.port = m->external_port; - m_key.fib_index = sm->outside_fib_index; - kv.key = m_key.as_u64; - kv.value = m - sm->static_mappings; - clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1); - - /* Assign worker */ - if (sm->workers) - { - snat_user_key_t w_key0; - snat_static_mapping_key_t w_key1; - - w_key0.addr = m->local_addr; - w_key0.fib_index = m->fib_index; - kv.key = w_key0.as_u64; - - if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value)) - { - kv.value = sm->first_worker_index + - sm->workers[sm->next_worker++ % vec_len (sm->workers)]; - - clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv, 1); - } - else - { - kv.value = value.value; - } - - w_key1.addr = m->external_addr; - w_key1.port = clib_host_to_net_u16 (m->external_port); - w_key1.fib_index = sm->outside_fib_index; - kv.key = w_key1.as_u64; - clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv, 1); - } - } - else - { - if (!m) - return VNET_API_ERROR_NO_SUCH_ENTRY; - - /* Free external address port */ - if (!addr_only && !(sm->static_mapping_only)) - { - for (i = 0; i < vec_len (sm->addresses); i++) - { - if (sm->addresses[i].addr.as_u32 == e_addr.as_u32) - { - a = sm->addresses + i; - clib_bitmap_set_no_check (a->busy_port_bitmap, e_port, 0); - a->busy_ports--; - - break; - } - } - } - - m_key.addr = m->local_addr; - m_key.port = m->local_port; - m_key.fib_index = m->fib_index; - kv.key = m_key.as_u64; - clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0); - - m_key.addr = m->external_addr; - m_key.port = m->external_port; - m_key.fib_index = sm->outside_fib_index; - kv.key = m_key.as_u64; - clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0); - - /* Delete session(s) for static mapping if exist */ - if (!(sm->static_mapping_only) || - (sm->static_mapping_only && sm->static_mapping_connection_tracking)) - { - snat_user_key_t u_key; - snat_user_t *u; - dlist_elt_t * head, * elt; - u32 elt_index, head_index, del_elt_index; - u32 ses_index; - u64 user_index; - snat_session_t * s; - snat_main_per_thread_data_t *tsm; - - u_key.addr = m->local_addr; - u_key.fib_index = m->fib_index; - kv.key = u_key.as_u64; - if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value)) - { - user_index = value.value; - if (!clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value)) - tsm = vec_elt_at_index (sm->per_thread_data, value.value); - else - tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers); - u = pool_elt_at_index (tsm->users, user_index); - if (u->nstaticsessions) - { - head_index = u->sessions_per_user_list_head_index; - head = pool_elt_at_index (tsm->list_pool, head_index); - elt_index = head->next; - elt = pool_elt_at_index (tsm->list_pool, elt_index); - ses_index = elt->value; - while (ses_index != ~0) - { - s = pool_elt_at_index (tsm->sessions, ses_index); - del_elt_index = elt_index; - elt_index = elt->next; - elt = pool_elt_at_index (tsm->list_pool, elt_index); - ses_index = elt->value; - - if (!addr_only) - { - if ((s->out2in.addr.as_u32 != e_addr.as_u32) && - (clib_net_to_host_u16 (s->out2in.port) != e_port)) - continue; - } - - value.key = s->in2out.as_u64; - clib_bihash_add_del_8_8 (&sm->in2out, &value, 0); - value.key = s->out2in.as_u64; - clib_bihash_add_del_8_8 (&sm->out2in, &value, 0); - pool_put (tsm->sessions, s); - - clib_dlist_remove (tsm->list_pool, del_elt_index); - pool_put_index (tsm->list_pool, del_elt_index); - u->nstaticsessions--; - - if (!addr_only) - break; - } - if (addr_only) - { - pool_put (tsm->users, u); - clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 0); - } - } - } - } - - /* Delete static mapping from pool */ - pool_put (sm->static_mappings, m); - } - - return 0; -} - -static int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del) -{ - snat_main_t *sm = &snat_main; - snat_interface_t *i; - const char * feature_name; - - if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)) - feature_name = is_inside ? "snat-in2out-fast" : "snat-out2in-fast"; - else - { - if (sm->num_workers > 1) - feature_name = is_inside ? "snat-in2out-worker-handoff" : "snat-out2in-worker-handoff"; - else - feature_name = is_inside ? "snat-in2out" : "snat-out2in"; - } - - vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, - !is_del, 0, 0); - - if (sm->fq_in2out_index == ~0) - sm->fq_in2out_index = vlib_frame_queue_main_init (snat_in2out_node.index, 0); - - if (sm->fq_out2in_index == ~0) - sm->fq_out2in_index = vlib_frame_queue_main_init (snat_out2in_node.index, 0); - - pool_foreach (i, sm->interfaces, - ({ - if (i->sw_if_index == sw_if_index) - { - if (is_del) - pool_put (sm->interfaces, i); - else - return VNET_API_ERROR_VALUE_EXIST; - - return 0; - } - })); - - if (is_del) - return VNET_API_ERROR_NO_SUCH_ENTRY; - - pool_get (sm->interfaces, i); - i->sw_if_index = sw_if_index; - i->is_inside = is_inside; - - return 0; -} - -static int snat_set_workers (uword * bitmap) -{ - snat_main_t *sm = &snat_main; - int i; - - if (sm->num_workers < 2) - return VNET_API_ERROR_FEATURE_DISABLED; - - if (clib_bitmap_last_set (bitmap) >= sm->num_workers) - return VNET_API_ERROR_INVALID_WORKER; - - vec_free (sm->workers); - clib_bitmap_foreach (i, bitmap, - ({ - vec_add1(sm->workers, i); - })); - - return 0; -} - -static void -vl_api_snat_add_address_range_t_handler -(vl_api_snat_add_address_range_t * mp) -{ - snat_main_t * sm = &snat_main; - vl_api_snat_add_address_range_reply_t * rmp; - ip4_address_t this_addr; - u32 start_host_order, end_host_order; - int i, count; - int rv = 0; - u32 * tmp; - - if (mp->is_ip4 != 1) - { - rv = VNET_API_ERROR_UNIMPLEMENTED; - goto send_reply; - } - - if (sm->static_mapping_only) - { - rv = VNET_API_ERROR_FEATURE_DISABLED; - goto send_reply; - } - - tmp = (u32 *) mp->first_ip_address; - start_host_order = clib_host_to_net_u32 (tmp[0]); - tmp = (u32 *) mp->last_ip_address; - end_host_order = clib_host_to_net_u32 (tmp[0]); - - count = (end_host_order - start_host_order) + 1; - - if (count > 1024) - clib_warning ("%U - %U, %d addresses...", - format_ip4_address, mp->first_ip_address, - format_ip4_address, mp->last_ip_address, - count); - - memcpy (&this_addr.as_u8, mp->first_ip_address, 4); - - for (i = 0; i < count; i++) - { - if (mp->is_add) - snat_add_address (sm, &this_addr); - else - rv = snat_del_address (sm, this_addr); - - if (rv) - goto send_reply; - - increment_v4_address (&this_addr); - } - - send_reply: - REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY); -} - -static void *vl_api_snat_add_address_range_t_print -(vl_api_snat_add_address_range_t *mp, void * handle) -{ - u8 * s; - - s = format (0, "SCRIPT: snat_add_address_range "); - s = format (s, "%U ", format_ip4_address, mp->first_ip_address); - if (memcmp (mp->first_ip_address, mp->last_ip_address, 4)) - { - s = format (s, " - %U ", format_ip4_address, mp->last_ip_address); - } - FINISH; -} - -static void -send_snat_address_details -(snat_address_t * a, unix_shared_memory_queue_t * q, u32 context) -{ - vl_api_snat_address_details_t *rmp; - snat_main_t * sm = &snat_main; - - rmp = vl_msg_api_alloc (sizeof (*rmp)); - memset (rmp, 0, sizeof (*rmp)); - rmp->_vl_msg_id = ntohs (VL_API_SNAT_ADDRESS_DETAILS+sm->msg_id_base); - rmp->is_ip4 = 1; - clib_memcpy (rmp->ip_address, &(a->addr), 4); - rmp->context = context; - - vl_msg_api_send_shmem (q, (u8 *) & rmp); -} - -static void -vl_api_snat_address_dump_t_handler -(vl_api_snat_address_dump_t * mp) -{ - unix_shared_memory_queue_t *q; - snat_main_t * sm = &snat_main; - snat_address_t * a; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - return; - - vec_foreach (a, sm->addresses) - send_snat_address_details (a, q, mp->context); -} - -static void *vl_api_snat_address_dump_t_print -(vl_api_snat_address_dump_t *mp, void * handle) -{ - u8 *s; - - s = format (0, "SCRIPT: snat_address_dump "); - - FINISH; -} - -static void -vl_api_snat_interface_add_del_feature_t_handler -(vl_api_snat_interface_add_del_feature_t * mp) -{ - snat_main_t * sm = &snat_main; - vl_api_snat_interface_add_del_feature_reply_t * rmp; - u8 is_del = mp->is_add == 0; - u32 sw_if_index = ntohl(mp->sw_if_index); - int rv = 0; - - VALIDATE_SW_IF_INDEX(mp); - - rv = snat_interface_add_del (sw_if_index, mp->is_inside, is_del); - - BAD_SW_IF_INDEX_LABEL; - - REPLY_MACRO(VL_API_SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY); -} - -static void *vl_api_snat_interface_add_del_feature_t_print -(vl_api_snat_interface_add_del_feature_t * mp, void *handle) -{ - u8 * s; - - s = format (0, "SCRIPT: snat_interface_add_del_feature "); - s = format (s, "sw_if_index %d %s %s", - clib_host_to_net_u32(mp->sw_if_index), - mp->is_inside ? "in":"out", - mp->is_add ? "" : "del"); - - FINISH; -} - -static void -send_snat_interface_details -(snat_interface_t * i, unix_shared_memory_queue_t * q, u32 context) -{ - vl_api_snat_interface_details_t *rmp; - snat_main_t * sm = &snat_main; - - rmp = vl_msg_api_alloc (sizeof (*rmp)); - memset (rmp, 0, sizeof (*rmp)); - rmp->_vl_msg_id = ntohs (VL_API_SNAT_INTERFACE_DETAILS+sm->msg_id_base); - rmp->sw_if_index = ntohl (i->sw_if_index); - rmp->is_inside = i->is_inside; - rmp->context = context; - - vl_msg_api_send_shmem (q, (u8 *) & rmp); -} - -static void -vl_api_snat_interface_dump_t_handler -(vl_api_snat_interface_dump_t * mp) -{ - unix_shared_memory_queue_t *q; - snat_main_t * sm = &snat_main; - snat_interface_t * i; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - return; - - pool_foreach (i, sm->interfaces, - ({ - send_snat_interface_details(i, q, mp->context); - })); -} - -static void *vl_api_snat_interface_dump_t_print -(vl_api_snat_interface_dump_t *mp, void * handle) -{ - u8 *s; - - s = format (0, "SCRIPT: snat_interface_dump "); - - FINISH; -}static void - -vl_api_snat_add_static_mapping_t_handler -(vl_api_snat_add_static_mapping_t * mp) -{ - snat_main_t * sm = &snat_main; - vl_api_snat_add_static_mapping_reply_t * rmp; - ip4_address_t local_addr, external_addr; - u16 local_port = 0, external_port = 0; - u32 vrf_id; - int rv = 0; - - if (mp->is_ip4 != 1) - { - rv = VNET_API_ERROR_UNIMPLEMENTED; - goto send_reply; - } - - memcpy (&local_addr.as_u8, mp->local_ip_address, 4); - memcpy (&external_addr.as_u8, mp->external_ip_address, 4); - if (mp->addr_only == 0) - { - local_port = clib_net_to_host_u16 (mp->local_port); - external_port = clib_net_to_host_u16 (mp->external_port); - } - vrf_id = clib_net_to_host_u32 (mp->vrf_id); - - rv = snat_add_static_mapping(local_addr, external_addr, local_port, - external_port, vrf_id, mp->addr_only, - mp->is_add); - - send_reply: - REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY); -} - -static void *vl_api_snat_add_static_mapping_t_print -(vl_api_snat_add_static_mapping_t *mp, void * handle) -{ - u8 * s; - - s = format (0, "SCRIPT: snat_add_static_mapping "); - s = format (s, "local_addr %U external_addr %U ", - format_ip4_address, mp->local_ip_address, - format_ip4_address, mp->external_ip_address); - - if (mp->addr_only == 0) - s = format (s, "local_port %d external_port %d ", - clib_net_to_host_u16 (mp->local_port), - clib_net_to_host_u16 (mp->external_port)); - - if (mp->vrf_id != ~0) - s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id)); - - FINISH; -} - -static void -send_snat_static_mapping_details -(snat_static_mapping_t * m, unix_shared_memory_queue_t * q, u32 context) -{ - vl_api_snat_static_mapping_details_t *rmp; - snat_main_t * sm = &snat_main; - - rmp = vl_msg_api_alloc (sizeof (*rmp)); - memset (rmp, 0, sizeof (*rmp)); - rmp->_vl_msg_id = ntohs (VL_API_SNAT_STATIC_MAPPING_DETAILS+sm->msg_id_base); - rmp->is_ip4 = 1; - rmp->addr_only = m->addr_only; - clib_memcpy (rmp->local_ip_address, &(m->local_addr), 4); - clib_memcpy (rmp->external_ip_address, &(m->external_addr), 4); - rmp->local_port = htons (m->local_port); - rmp->external_port = htons (m->external_port); - rmp->vrf_id = htonl (m->vrf_id); - rmp->context = context; - - vl_msg_api_send_shmem (q, (u8 *) & rmp); -} - -static void -vl_api_snat_static_mapping_dump_t_handler -(vl_api_snat_static_mapping_dump_t * mp) -{ - unix_shared_memory_queue_t *q; - snat_main_t * sm = &snat_main; - snat_static_mapping_t * m; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - return; - - pool_foreach (m, sm->static_mappings, - ({ - send_snat_static_mapping_details (m, q, mp->context); - })); -} - -static void *vl_api_snat_static_mapping_dump_t_print -(vl_api_snat_static_mapping_dump_t *mp, void * handle) -{ - u8 *s; - - s = format (0, "SCRIPT: snat_static_mapping_dump "); - - FINISH; -} - -static void -vl_api_snat_control_ping_t_handler -(vl_api_snat_control_ping_t * mp) -{ - vl_api_snat_control_ping_reply_t *rmp; - snat_main_t * sm = &snat_main; - int rv = 0; - - REPLY_MACRO2(VL_API_SNAT_CONTROL_PING_REPLY, - ({ - rmp->vpe_pid = ntohl (getpid()); - })); -} - -static void *vl_api_snat_control_ping_t_print -(vl_api_snat_control_ping_t *mp, void * handle) -{ - u8 *s; - - s = format (0, "SCRIPT: snat_control_ping "); - - FINISH; -} - -static void -vl_api_snat_show_config_t_handler -(vl_api_snat_show_config_t * mp) -{ - vl_api_snat_show_config_reply_t *rmp; - snat_main_t * sm = &snat_main; - int rv = 0; - - REPLY_MACRO2(VL_API_SNAT_SHOW_CONFIG_REPLY, - ({ - rmp->translation_buckets = htonl (sm->translation_buckets); - rmp->translation_memory_size = htonl (sm->translation_memory_size); - rmp->user_buckets = htonl (sm->user_buckets); - rmp->user_memory_size = htonl (sm->user_memory_size); - rmp->max_translations_per_user = htonl (sm->max_translations_per_user); - rmp->outside_vrf_id = htonl (sm->outside_vrf_id); - rmp->inside_vrf_id = htonl (sm->inside_vrf_id); - rmp->static_mapping_only = sm->static_mapping_only; - rmp->static_mapping_connection_tracking = - sm->static_mapping_connection_tracking; - })); -} - -static void *vl_api_snat_show_config_t_print -(vl_api_snat_show_config_t *mp, void * handle) -{ - u8 *s; - - s = format (0, "SCRIPT: snat_show_config "); - - FINISH; -} - -static void -vl_api_snat_set_workers_t_handler -(vl_api_snat_set_workers_t * mp) -{ - snat_main_t * sm = &snat_main; - vl_api_snat_set_workers_reply_t * rmp; - int rv = 0; - uword *bitmap = 0; - u64 mask = clib_net_to_host_u64 (mp->worker_mask); - - if (sm->num_workers < 2) - { - rv = VNET_API_ERROR_FEATURE_DISABLED; - goto send_reply; - } - - bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask)); - rv = snat_set_workers(bitmap); - clib_bitmap_free (bitmap); - - send_reply: - REPLY_MACRO (VL_API_SNAT_SET_WORKERS_REPLY); -} - -static void *vl_api_snat_set_workers_t_print -(vl_api_snat_set_workers_t *mp, void * handle) -{ - u8 * s; - uword *bitmap = 0; - u8 first = 1; - int i; - u64 mask = clib_net_to_host_u64 (mp->worker_mask); - - s = format (0, "SCRIPT: snat_set_workers "); - bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask)); - clib_bitmap_foreach (i, bitmap, - ({ - if (first) - s = format (s, "%d", i); - else - s = format (s, ",%d", i); - first = 0; - })); - clib_bitmap_free (bitmap); - FINISH; -} - -static void -send_snat_worker_details -(u32 worker_index, unix_shared_memory_queue_t * q, u32 context) -{ - vl_api_snat_worker_details_t *rmp; - snat_main_t * sm = &snat_main; - vlib_worker_thread_t *w = - vlib_worker_threads + worker_index + sm->first_worker_index; - - rmp = vl_msg_api_alloc (sizeof (*rmp)); - memset (rmp, 0, sizeof (*rmp)); - rmp->_vl_msg_id = ntohs (VL_API_SNAT_WORKER_DETAILS+sm->msg_id_base); - rmp->context = context; - rmp->worker_index = htonl (worker_index); - rmp->lcore_id = htonl (w->lcore_id); - strncpy ((char *) rmp->name, (char *) w->name, ARRAY_LEN (rmp->name) - 1); - - vl_msg_api_send_shmem (q, (u8 *) & rmp); -} - -static void -vl_api_snat_worker_dump_t_handler -(vl_api_snat_worker_dump_t * mp) -{ - unix_shared_memory_queue_t *q; - snat_main_t * sm = &snat_main; - u32 * worker_index; - - q = vl_api_client_index_to_input_queue (mp->client_index); - if (q == 0) - return; - - vec_foreach (worker_index, sm->workers) - { - send_snat_worker_details(*worker_index, q, mp->context); - } -} - -static void *vl_api_snat_worker_dump_t_print -(vl_api_snat_worker_dump_t *mp, void * handle) -{ - u8 *s; - - s = format (0, "SCRIPT: snat_worker_dump "); - - FINISH; -} - -/* List of message types that this plugin understands */ -#define foreach_snat_plugin_api_msg \ -_(SNAT_ADD_ADDRESS_RANGE, snat_add_address_range) \ -_(SNAT_INTERFACE_ADD_DEL_FEATURE, snat_interface_add_del_feature) \ -_(SNAT_ADD_STATIC_MAPPING, snat_add_static_mapping) \ -_(SNAT_CONTROL_PING, snat_control_ping) \ -_(SNAT_STATIC_MAPPING_DUMP, snat_static_mapping_dump) \ -_(SNAT_SHOW_CONFIG, snat_show_config) \ -_(SNAT_ADDRESS_DUMP, snat_address_dump) \ -_(SNAT_INTERFACE_DUMP, snat_interface_dump) \ -_(SNAT_SET_WORKERS, snat_set_workers) \ -_(SNAT_WORKER_DUMP, snat_worker_dump) - -/* Set up the API message handling tables */ -static clib_error_t * -snat_plugin_api_hookup (vlib_main_t *vm) -{ - snat_main_t * sm __attribute__ ((unused)) = &snat_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_snat_plugin_api_msg; -#undef _ - - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (snat_main_t * sm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); - foreach_vl_msg_name_crc_snat; -#undef _ -} - -static void plugin_custom_dump_configure (snat_main_t * sm) -{ -#define _(n,f) sm->api_main->msg_print_handlers \ - [VL_API_##n + sm->msg_id_base] \ - = (void *) vl_api_##f##_t_print; - foreach_snat_plugin_api_msg; -#undef _ -} - -static clib_error_t * snat_init (vlib_main_t * vm) -{ - snat_main_t * sm = &snat_main; - clib_error_t * error = 0; - ip4_main_t * im = &ip4_main; - ip_lookup_main_t * lm = &im->lookup_main; - u8 * name; - uword *p; - vlib_thread_registration_t *tr; - vlib_thread_main_t *tm = vlib_get_thread_main (); - uword *bitmap = 0; - u32 i; - - name = format (0, "snat_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - sm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - - sm->vlib_main = vm; - sm->vnet_main = vnet_get_main(); - sm->ip4_main = im; - sm->ip4_lookup_main = lm; - sm->api_main = &api_main; - sm->first_worker_index = 0; - sm->next_worker = 0; - sm->num_workers = 0; - sm->workers = 0; - sm->fq_in2out_index = ~0; - sm->fq_out2in_index = ~0; - - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - if (p) - { - tr = (vlib_thread_registration_t *) p[0]; - if (tr) - { - sm->num_workers = tr->count; - sm->first_worker_index = tr->first_index; - } - } - - /* Use all available workers by default */ - if (sm->num_workers > 1) - { - for (i=0; i < sm->num_workers; i++) - bitmap = clib_bitmap_set (bitmap, i, 1); - snat_set_workers(bitmap); - clib_bitmap_free (bitmap); - } - - error = snat_plugin_api_hookup (vm); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (sm, &api_main); - - plugin_custom_dump_configure (sm); - vec_free(name); - - return error; -} - -VLIB_INIT_FUNCTION (snat_init); - -void snat_free_outside_address_and_port (snat_main_t * sm, - snat_session_key_t * k, - u32 address_index) -{ - snat_address_t *a; - u16 port_host_byte_order = clib_net_to_host_u16 (k->port); - - ASSERT (address_index < vec_len (sm->addresses)); - - a = sm->addresses + address_index; - - ASSERT (clib_bitmap_get_no_check (a->busy_port_bitmap, - port_host_byte_order) == 1); - - clib_bitmap_set_no_check (a->busy_port_bitmap, port_host_byte_order, 0); - a->busy_ports--; -} - -/** - * @brief Match SNAT static mapping. - * - * @param sm SNAT main. - * @param match Address and port to match. - * @param mapping External or local address and port of the matched mapping. - * @param by_external If 0 match by local address otherwise match by external - * address. - * - * @returns 0 if match found otherwise 1. - */ -int snat_static_mapping_match (snat_main_t * sm, - snat_session_key_t match, - snat_session_key_t * mapping, - u8 by_external) -{ - clib_bihash_kv_8_8_t kv, value; - snat_static_mapping_t *m; - snat_static_mapping_key_t m_key; - clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local; - - if (by_external) - mapping_hash = &sm->static_mapping_by_external; - - m_key.addr = match.addr; - m_key.port = clib_net_to_host_u16 (match.port); - m_key.fib_index = match.fib_index; - - kv.key = m_key.as_u64; - - if (clib_bihash_search_8_8 (mapping_hash, &kv, &value)) - { - /* Try address only mapping */ - m_key.port = 0; - kv.key = m_key.as_u64; - if (clib_bihash_search_8_8 (mapping_hash, &kv, &value)) - return 1; - } - - m = pool_elt_at_index (sm->static_mappings, value.value); - - if (by_external) - { - mapping->addr = m->local_addr; - /* Address only mapping doesn't change port */ - mapping->port = m->addr_only ? match.port - : clib_host_to_net_u16 (m->local_port); - mapping->fib_index = m->fib_index; - } - else - { - mapping->addr = m->external_addr; - /* Address only mapping doesn't change port */ - mapping->port = m->addr_only ? match.port - : clib_host_to_net_u16 (m->external_port); - mapping->fib_index = sm->outside_fib_index; - } - - return 0; -} - -int snat_alloc_outside_address_and_port (snat_main_t * sm, - snat_session_key_t * k, - u32 * address_indexp) -{ - int i; - snat_address_t *a; - u32 portnum; - - for (i = 0; i < vec_len (sm->addresses); i++) - { - if (sm->addresses[i].busy_ports < (65535-1024)) - { - a = sm->addresses + i; - - while (1) - { - portnum = random_u32 (&sm->random_seed); - portnum &= 0xFFFF; - if (portnum < 1024) - continue; - if (clib_bitmap_get_no_check (a->busy_port_bitmap, portnum)) - continue; - clib_bitmap_set_no_check (a->busy_port_bitmap, portnum, 1); - a->busy_ports++; - /* Caller sets protocol and fib index */ - k->addr = a->addr; - k->port = clib_host_to_net_u16(portnum); - *address_indexp = i; - return 0; - } - } - } - /* Totally out of translations to use... */ - return 1; -} - - -static clib_error_t * -add_address_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - snat_main_t * sm = &snat_main; - ip4_address_t start_addr, end_addr, this_addr; - u32 start_host_order, end_host_order; - int i, count; - int is_add = 1; - int rv = 0; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "%U - %U", - unformat_ip4_address, &start_addr, - unformat_ip4_address, &end_addr)) - ; - else if (unformat (line_input, "%U", unformat_ip4_address, &start_addr)) - end_addr = start_addr; - else if (unformat (line_input, "del")) - is_add = 0; - else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); - } - unformat_free (line_input); - - if (sm->static_mapping_only) - return clib_error_return (0, "static mapping only mode"); - - start_host_order = clib_host_to_net_u32 (start_addr.as_u32); - end_host_order = clib_host_to_net_u32 (end_addr.as_u32); - - if (end_host_order < start_host_order) - return clib_error_return (0, "end address less than start address"); - - count = (end_host_order - start_host_order) + 1; - - if (count > 1024) - clib_warning ("%U - %U, %d addresses...", - format_ip4_address, &start_addr, - format_ip4_address, &end_addr, - count); - - this_addr = start_addr; - - for (i = 0; i < count; i++) - { - if (is_add) - snat_add_address (sm, &this_addr); - else - rv = snat_del_address (sm, this_addr); - - switch (rv) - { - case VNET_API_ERROR_NO_SUCH_ENTRY: - return clib_error_return (0, "S-NAT address not exist."); - break; - case VNET_API_ERROR_UNSPECIFIED: - return clib_error_return (0, "S-NAT address used in static mapping."); - break; - default: - break; - } - - increment_v4_address (&this_addr); - } - - return 0; -} - -VLIB_CLI_COMMAND (add_address_command, static) = { - .path = "snat add address", - .short_help = "snat add addresses [- ] [del]", - .function = add_address_command_fn, -}; - -static clib_error_t * -snat_feature_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - vnet_main_t * vnm = vnet_get_main(); - clib_error_t * error = 0; - u32 sw_if_index; - u32 * inside_sw_if_indices = 0; - u32 * outside_sw_if_indices = 0; - int is_del = 0; - int i; - - sw_if_index = ~0; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "in %U", unformat_vnet_sw_interface, - vnm, &sw_if_index)) - vec_add1 (inside_sw_if_indices, sw_if_index); - else if (unformat (line_input, "out %U", unformat_vnet_sw_interface, - vnm, &sw_if_index)) - vec_add1 (outside_sw_if_indices, sw_if_index); - else if (unformat (line_input, "del")) - is_del = 1; - else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); - } - unformat_free (line_input); - - if (vec_len (inside_sw_if_indices)) - { - for (i = 0; i < vec_len(inside_sw_if_indices); i++) - { - sw_if_index = inside_sw_if_indices[i]; - snat_interface_add_del (sw_if_index, 1, is_del); - } - } - - if (vec_len (outside_sw_if_indices)) - { - for (i = 0; i < vec_len(outside_sw_if_indices); i++) - { - sw_if_index = outside_sw_if_indices[i]; - snat_interface_add_del (sw_if_index, 0, is_del); - } - } - - vec_free (inside_sw_if_indices); - vec_free (outside_sw_if_indices); - - return error; -} - -VLIB_CLI_COMMAND (set_interface_snat_command, static) = { - .path = "set interface snat", - .function = snat_feature_command_fn, - .short_help = "set interface snat in out [del]", -}; - -static clib_error_t * -add_static_mapping_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - clib_error_t * error = 0; - ip4_address_t l_addr, e_addr; - u32 l_port = 0, e_port = 0, vrf_id = ~0; - int is_add = 1; - int addr_only = 1; - int rv; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "local %U %u", unformat_ip4_address, &l_addr, - &l_port)) - addr_only = 0; - else if (unformat (line_input, "local %U", unformat_ip4_address, &l_addr)) - ; - else if (unformat (line_input, "external %U %u", unformat_ip4_address, - &e_addr, &e_port)) - addr_only = 0; - else if (unformat (line_input, "external %U", unformat_ip4_address, - &e_addr)) - ; - else if (unformat (line_input, "vrf %u", &vrf_id)) - ; - else if (unformat (line_input, "del")) - is_add = 0; - else - return clib_error_return (0, "unknown input: '%U'", - format_unformat_error, line_input); - } - unformat_free (line_input); - - rv = snat_add_static_mapping(l_addr, e_addr, (u16) l_port, (u16) e_port, - vrf_id, addr_only, is_add); - - switch (rv) - { - case VNET_API_ERROR_INVALID_VALUE: - return clib_error_return (0, "External port already in use."); - break; - case VNET_API_ERROR_NO_SUCH_ENTRY: - if (is_add) - return clib_error_return (0, "External addres must be allocated."); - else - return clib_error_return (0, "Mapping not exist."); - break; - case VNET_API_ERROR_NO_SUCH_FIB: - return clib_error_return (0, "No such VRF id."); - case VNET_API_ERROR_VALUE_EXIST: - return clib_error_return (0, "Mapping already exist."); - default: - break; - } - - return error; -} - -/*? - * @cliexpar - * @cliexstart{snat add static mapping} - * Static mapping allows hosts on the external network to initiate connection - * to to the local network host. - * To create static mapping between local host address 10.0.0.3 port 6303 and - * external address 4.4.4.4 port 3606 use: - * vpp# snat add static mapping local 10.0.0.3 6303 external 4.4.4.4 3606 - * If not runnig "static mapping only" S-NAT plugin mode use before: - * vpp# snat add address 4.4.4.4 - * To create static mapping between local and external address use: - * vpp# snat add static mapping local 10.0.0.3 external 4.4.4.4 - * @cliexend -?*/ -VLIB_CLI_COMMAND (add_static_mapping_command, static) = { - .path = "snat add static mapping", - .function = add_static_mapping_command_fn, - .short_help = - "snat add static mapping local [] external [] [vrf ] [del]", -}; - -static clib_error_t * -set_workers_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - uword *bitmap = 0; - int rv = 0; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "%U", unformat_bitmap_list, &bitmap)) - ; - else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); - } - unformat_free (line_input); - - if (bitmap == 0) - return clib_error_return (0, "List of workers must be specified."); - - rv = snat_set_workers(bitmap); - - clib_bitmap_free (bitmap); - - switch (rv) - { - case VNET_API_ERROR_INVALID_WORKER: - return clib_error_return (0, "Invalid worker(s)."); - break; - case VNET_API_ERROR_FEATURE_DISABLED: - return clib_error_return (0, - "Supported only if 2 or more workes available."); - break; - default: - break; - } - - return 0; -} - -/*? - * @cliexpar - * @cliexstart{set snat workers} - * Set SNAT workers if 2 or more workers available, use: - * vpp# set snat workers 0-2,5 - * @cliexend -?*/ -VLIB_CLI_COMMAND (set_workers_command, static) = { - .path = "set snat workers", - .function = set_workers_command_fn, - .short_help = - "set snat workers ", -}; - -static clib_error_t * -snat_config (vlib_main_t * vm, unformat_input_t * input) -{ - snat_main_t * sm = &snat_main; - u32 translation_buckets = 1024; - u32 translation_memory_size = 128<<20; - u32 user_buckets = 128; - u32 user_memory_size = 64<<20; - u32 max_translations_per_user = 100; - u32 outside_vrf_id = 0; - u32 inside_vrf_id = 0; - u32 static_mapping_buckets = 1024; - u32 static_mapping_memory_size = 64<<20; - u8 static_mapping_only = 0; - u8 static_mapping_connection_tracking = 0; - vlib_thread_main_t *tm = vlib_get_thread_main (); - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "translation hash buckets %d", &translation_buckets)) - ; - else if (unformat (input, "translation hash memory %d", - &translation_memory_size)); - else if (unformat (input, "user hash buckets %d", &user_buckets)) - ; - else if (unformat (input, "user hash memory %d", - &user_memory_size)) - ; - else if (unformat (input, "max translations per user %d", - &max_translations_per_user)) - ; - else if (unformat (input, "outside VRF id %d", - &outside_vrf_id)) - ; - else if (unformat (input, "inside VRF id %d", - &inside_vrf_id)) - ; - else if (unformat (input, "static mapping only")) - { - static_mapping_only = 1; - if (unformat (input, "connection tracking")) - static_mapping_connection_tracking = 1; - } - else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); - } - - /* for show commands, etc. */ - sm->translation_buckets = translation_buckets; - sm->translation_memory_size = translation_memory_size; - sm->user_buckets = user_buckets; - sm->user_memory_size = user_memory_size; - sm->max_translations_per_user = max_translations_per_user; - sm->outside_vrf_id = outside_vrf_id; - sm->outside_fib_index = ~0; - sm->inside_vrf_id = inside_vrf_id; - sm->inside_fib_index = ~0; - sm->static_mapping_only = static_mapping_only; - sm->static_mapping_connection_tracking = static_mapping_connection_tracking; - - if (!static_mapping_only || - (static_mapping_only && static_mapping_connection_tracking)) - { - clib_bihash_init_8_8 (&sm->worker_by_in, "worker-by-in", user_buckets, - user_memory_size); - - clib_bihash_init_8_8 (&sm->worker_by_out, "worker-by-out", user_buckets, - user_memory_size); - - vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1); - - clib_bihash_init_8_8 (&sm->in2out, "in2out", translation_buckets, - translation_memory_size); - - clib_bihash_init_8_8 (&sm->out2in, "out2in", translation_buckets, - translation_memory_size); - - clib_bihash_init_8_8 (&sm->user_hash, "users", user_buckets, - user_memory_size); - } - clib_bihash_init_8_8 (&sm->static_mapping_by_local, - "static_mapping_by_local", static_mapping_buckets, - static_mapping_memory_size); - - clib_bihash_init_8_8 (&sm->static_mapping_by_external, - "static_mapping_by_external", static_mapping_buckets, - static_mapping_memory_size); - return 0; -} - -VLIB_CONFIG_FUNCTION (snat_config, "snat"); - -u8 * format_snat_key (u8 * s, va_list * args) -{ - snat_session_key_t * key = va_arg (*args, snat_session_key_t *); - char * protocol_string = "unknown"; - static char *protocol_strings[] = { - "UDP", - "TCP", - "ICMP", - }; - - if (key->protocol < ARRAY_LEN(protocol_strings)) - protocol_string = protocol_strings[key->protocol]; - - s = format (s, "%U proto %s port %d fib %d", - format_ip4_address, &key->addr, protocol_string, - clib_net_to_host_u16 (key->port), key->fib_index); - return s; -} - -u8 * format_snat_session (u8 * s, va_list * args) -{ - snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *); - snat_session_t * sess = va_arg (*args, snat_session_t *); - - s = format (s, " i2o %U\n", format_snat_key, &sess->in2out); - s = format (s, " o2i %U\n", format_snat_key, &sess->out2in); - s = format (s, " last heard %.2f\n", sess->last_heard); - s = format (s, " total pkts %d, total bytes %lld\n", - sess->total_pkts, sess->total_bytes); - if (snat_is_session_static (sess)) - s = format (s, " static translation\n"); - else - s = format (s, " dynamic translation\n"); - - return s; -} - -u8 * format_snat_user (u8 * s, va_list * args) -{ - snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *); - snat_user_t * u = va_arg (*args, snat_user_t *); - int verbose = va_arg (*args, int); - dlist_elt_t * head, * elt; - u32 elt_index, head_index; - u32 session_index; - snat_session_t * sess; - - s = format (s, "%U: %d dynamic translations, %d static translations\n", - format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions); - - if (verbose == 0) - return s; - - if (u->nsessions || u->nstaticsessions) - { - head_index = u->sessions_per_user_list_head_index; - head = pool_elt_at_index (sm->list_pool, head_index); - - elt_index = head->next; - elt = pool_elt_at_index (sm->list_pool, elt_index); - session_index = elt->value; - - while (session_index != ~0) - { - sess = pool_elt_at_index (sm->sessions, session_index); - - s = format (s, " %U\n", format_snat_session, sm, sess); - - elt_index = elt->next; - elt = pool_elt_at_index (sm->list_pool, elt_index); - session_index = elt->value; - } - } - - return s; -} - -u8 * format_snat_static_mapping (u8 * s, va_list * args) -{ - snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *); - - if (m->addr_only) - s = format (s, "local %U external %U vrf %d", - format_ip4_address, &m->local_addr, - format_ip4_address, &m->external_addr, - m->vrf_id); - else - s = format (s, "local %U:%d external %U:%d vrf %d", - format_ip4_address, &m->local_addr, m->local_port, - format_ip4_address, &m->external_addr, m->external_port, - m->vrf_id); - - return s; -} - -static clib_error_t * -show_snat_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - int verbose = 0; - snat_main_t * sm = &snat_main; - snat_user_t * u; - snat_static_mapping_t *m; - snat_interface_t *i; - snat_address_t * ap; - vnet_main_t *vnm = vnet_get_main(); - snat_main_per_thread_data_t *tsm; - u32 users_num = 0, sessions_num = 0, *worker; - uword j = 0; - - if (unformat (input, "detail")) - verbose = 1; - else if (unformat (input, "verbose")) - verbose = 2; - - if (sm->static_mapping_only) - { - if (sm->static_mapping_connection_tracking) - vlib_cli_output (vm, "SNAT mode: static mapping only connection " - "tracking"); - else - vlib_cli_output (vm, "SNAT mode: static mapping only"); - } - else - { - vlib_cli_output (vm, "SNAT mode: dynamic translations enabled"); - } - - if (verbose > 0) - { - pool_foreach (i, sm->interfaces, - ({ - vlib_cli_output (vm, "%U %s", format_vnet_sw_interface_name, vnm, - vnet_get_sw_interface (vnm, i->sw_if_index), - i->is_inside ? "in" : "out"); - })); - - vec_foreach (ap, sm->addresses) - { - u8 * s = format (0, ""); - vlib_cli_output (vm, "%U", format_ip4_address, &ap->addr); - clib_bitmap_foreach (j, ap->busy_port_bitmap, - ({ - s = format (s, " %d", j); - })); - vlib_cli_output (vm, " %d busy ports:%v", ap->busy_ports, s); - } - } - - if (sm->num_workers > 1) - { - vlib_cli_output (vm, "%d workers", vec_len (sm->workers)); - if (verbose > 0) - { - vec_foreach (worker, sm->workers) - { - vlib_worker_thread_t *w = - vlib_worker_threads + *worker + sm->first_worker_index; - vlib_cli_output (vm, " %v", w->name); - } - } - } - - if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)) - { - vlib_cli_output (vm, "%d static mappings", - pool_elts (sm->static_mappings)); - - if (verbose > 0) - { - pool_foreach (m, sm->static_mappings, - ({ - vlib_cli_output (vm, "%U", format_snat_static_mapping, m); - })); - } - } - else - { - vec_foreach (tsm, sm->per_thread_data) - { - users_num += pool_elts (tsm->users); - sessions_num += pool_elts (tsm->sessions); - } - - vlib_cli_output (vm, "%d users, %d outside addresses, %d active sessions," - " %d static mappings", - users_num, - vec_len (sm->addresses), - sessions_num, - pool_elts (sm->static_mappings)); - - if (verbose > 0) - { - vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->in2out, - verbose - 1); - vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->out2in, - verbose - 1); - vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_in, - verbose - 1); - vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_out, - verbose - 1); - vec_foreach_index (j, sm->per_thread_data) - { - tsm = vec_elt_at_index (sm->per_thread_data, j); - - if (pool_elts (tsm->users) == 0) - continue; - - vlib_worker_thread_t *w = vlib_worker_threads + j; - vlib_cli_output (vm, "Thread %d (%v at lcore %u):", j, w->name, - w->lcore_id); - vlib_cli_output (vm, " %d list pool elements", - pool_elts (tsm->list_pool)); - - pool_foreach (u, tsm->users, - ({ - vlib_cli_output (vm, " %U", format_snat_user, tsm, u, - verbose - 1); - })); - } - - if (pool_elts (sm->static_mappings)) - { - vlib_cli_output (vm, "static mappings:"); - pool_foreach (m, sm->static_mappings, - ({ - vlib_cli_output (vm, "%U", format_snat_static_mapping, m); - })); - } - } - } - - return 0; -} - -VLIB_CLI_COMMAND (show_snat_command, static) = { - .path = "show snat", - .short_help = "show snat", - .function = show_snat_command_fn, -}; diff --git a/plugins/snat-plugin/snat/snat.h b/plugins/snat-plugin/snat/snat.h deleted file mode 100644 index cb31dc51..00000000 --- a/plugins/snat-plugin/snat/snat.h +++ /dev/null @@ -1,259 +0,0 @@ - -/* - * snat.h - simple nat definitions - * - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_snat_h__ -#define __included_snat_h__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Key */ -typedef struct { - union - { - struct - { - ip4_address_t addr; - u16 port; - u16 protocol:3, - fib_index:13; - }; - u64 as_u64; - }; -} snat_session_key_t; - -typedef struct { - union - { - struct - { - ip4_address_t addr; - u32 fib_index; - }; - u64 as_u64; - }; -} snat_user_key_t; - -typedef struct { - union - { - struct - { - ip4_address_t addr; - u16 port; - u16 fib_index; - }; - u64 as_u64; - }; -} snat_static_mapping_key_t; - - -typedef enum { - SNAT_PROTOCOL_UDP = 0, - SNAT_PROTOCOL_TCP, - SNAT_PROTOCOL_ICMP, -} snat_protocol_t; - - -#define SNAT_SESSION_FLAG_STATIC_MAPPING 1 - -typedef CLIB_PACKED(struct { - snat_session_key_t out2in; /* 0-15 */ - - snat_session_key_t in2out; /* 16-31 */ - - u32 flags; /* 32-35 */ - - /* per-user translations */ - u32 per_user_index; /* 36-39 */ - - u32 per_user_list_head_index; /* 40-43 */ - - /* Last heard timer */ - f64 last_heard; /* 44-51 */ - - u64 total_bytes; /* 52-59 */ - - u32 total_pkts; /* 60-63 */ - - /* Outside address */ - u32 outside_address_index; /* 64-67 */ - -}) snat_session_t; - - -typedef struct { - ip4_address_t addr; - u32 sessions_per_user_list_head_index; - u32 nsessions; - u32 nstaticsessions; -} snat_user_t; - -typedef struct { - ip4_address_t addr; - u32 busy_ports; - uword * busy_port_bitmap; -} snat_address_t; - -typedef struct { - ip4_address_t local_addr; - ip4_address_t external_addr; - u16 local_port; - u16 external_port; - u8 addr_only; - u32 vrf_id; - u32 fib_index; -} snat_static_mapping_t; - -typedef struct { - u32 sw_if_index; - u8 is_inside; -} snat_interface_t; - -typedef struct { - /* User pool */ - snat_user_t * users; - - /* Session pool */ - snat_session_t * sessions; - - /* Pool of doubly-linked list elements */ - dlist_elt_t * list_pool; -} snat_main_per_thread_data_t; - -typedef struct { - /* Main lookup tables */ - clib_bihash_8_8_t out2in; - clib_bihash_8_8_t in2out; - - /* Find-a-user => src address lookup */ - clib_bihash_8_8_t user_hash; - - /* Non-translated packets worker lookup => src address + VRF */ - clib_bihash_8_8_t worker_by_in; - - /* Translated packets worker lookup => IP address + port number */ - clib_bihash_8_8_t worker_by_out; - - u32 num_workers; - u32 first_worker_index; - u32 next_worker; - u32 * workers; - - /* Per thread data */ - snat_main_per_thread_data_t * per_thread_data; - - /* Find a static mapping by local */ - clib_bihash_8_8_t static_mapping_by_local; - - /* Find a static mapping by external */ - clib_bihash_8_8_t static_mapping_by_external; - - /* Static mapping pool */ - snat_static_mapping_t * static_mappings; - - /* Interface pool */ - snat_interface_t * interfaces; - - /* Vector of outside addresses */ - snat_address_t * addresses; - - /* Randomize port allocation order */ - u32 random_seed; - - /* Worker handoff index */ - u32 fq_in2out_index; - u32 fq_out2in_index; - - /* Config parameters */ - u8 static_mapping_only; - u8 static_mapping_connection_tracking; - u32 translation_buckets; - u32 translation_memory_size; - u32 user_buckets; - u32 user_memory_size; - u32 max_translations_per_user; - u32 outside_vrf_id; - u32 outside_fib_index; - u32 inside_vrf_id; - u32 inside_fib_index; - - /* API message ID base */ - u16 msg_id_base; - - /* convenience */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; - ip4_main_t * ip4_main; - ip_lookup_main_t * ip4_lookup_main; - ethernet_main_t * ethernet_main; - api_main_t * api_main; -} snat_main_t; - -extern snat_main_t snat_main; -extern vlib_node_registration_t snat_in2out_node; -extern vlib_node_registration_t snat_out2in_node; -extern vlib_node_registration_t snat_in2out_fast_node; -extern vlib_node_registration_t snat_out2in_fast_node; -extern vlib_node_registration_t snat_in2out_worker_handoff_node; -extern vlib_node_registration_t snat_out2in_worker_handoff_node; - -void snat_free_outside_address_and_port (snat_main_t * sm, - snat_session_key_t * k, - u32 address_index); - -int snat_alloc_outside_address_and_port (snat_main_t * sm, - snat_session_key_t * k, - u32 * address_indexp); - -int snat_static_mapping_match (snat_main_t * sm, - snat_session_key_t match, - snat_session_key_t * mapping, - u8 by_external); - -format_function_t format_snat_user; - -typedef struct { - u32 cached_sw_if_index; - u32 cached_ip4_address; -} snat_runtime_t; - -/** \brief Check if SNAT session is created from static mapping. - @param s SNAT session - @return 1 if SNAT session is created from static mapping otherwise 0 -*/ -#define snat_is_session_static(s) s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING - -/* - * Why is this here? Because we don't need to touch this layer to - * simply reply to an icmp. We need to change id to a unique - * value to NAT an echo request/reply. - */ - -typedef struct { - u16 identifier; - u16 sequence; -} icmp_echo_header_t; - -#endif /* __included_snat_h__ */ diff --git a/plugins/snat-plugin/snat/snat_all_api_h.h b/plugins/snat-plugin/snat/snat_all_api_h.h deleted file mode 100644 index 49017700..00000000 --- a/plugins/snat-plugin/snat/snat_all_api_h.h +++ /dev/null @@ -1,19 +0,0 @@ - -/* - * snat_all_api_h.h - skeleton vpp engine plug-in api #include file - * - * Copyright (c) - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Include the generated file, see BUILT_SOURCES in Makefile.am */ -#include diff --git a/plugins/snat-plugin/snat/snat_msg_enum.h b/plugins/snat-plugin/snat/snat_msg_enum.h deleted file mode 100644 index 2c76fd51..00000000 --- a/plugins/snat-plugin/snat/snat_msg_enum.h +++ /dev/null @@ -1,31 +0,0 @@ - -/* - * snat_msg_enum.h - skeleton vpp engine plug-in message enumeration - * - * Copyright (c) - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef included_snat_msg_enum_h -#define included_snat_msg_enum_h - -#include - -#define vl_msg_id(n,h) n, -typedef enum { -#include - /* We'll want to know how many messages IDs we need... */ - VL_MSG_FIRST_AVAILABLE, -} vl_msg_id_t; -#undef vl_msg_id - -#endif /* included_snat_msg_enum_h */ diff --git a/plugins/snat-plugin/snat/snat_test.c b/plugins/snat-plugin/snat/snat_test.c deleted file mode 100644 index 2a003ba6..00000000 --- a/plugins/snat-plugin/snat/snat_test.c +++ /dev/null @@ -1,602 +0,0 @@ - -/* - * snat.c - skeleton vpp-api-test plug-in - * - * Copyright (c) - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -uword unformat_sw_if_index (unformat_input_t * input, va_list * args); - -/* Declare message IDs */ -#include - -/* define message structures */ -#define vl_typedefs -#include -#undef vl_typedefs - -/* declare message handlers for each api */ - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) -#define vl_printfun -#include -#undef vl_printfun - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -typedef struct { - /* API message ID base */ - u16 msg_id_base; - vat_main_t *vat_main; -} snat_test_main_t; - -snat_test_main_t snat_test_main; - -#define foreach_standard_reply_retval_handler \ -_(snat_add_address_range_reply) \ -_(snat_interface_add_del_feature_reply) \ -_(snat_add_static_mapping_reply) \ -_(snat_set_workers_reply) - -#define _(n) \ - static void vl_api_##n##_t_handler \ - (vl_api_##n##_t * mp) \ - { \ - vat_main_t * vam = snat_test_main.vat_main; \ - i32 retval = ntohl(mp->retval); \ - if (vam->async_mode) { \ - vam->async_errors += (retval < 0); \ - } else { \ - vam->retval = retval; \ - vam->result_ready = 1; \ - } \ - } -foreach_standard_reply_retval_handler; -#undef _ - -/* - * Table of message reply handlers, must include boilerplate handlers - * we just generated - */ -#define foreach_vpe_api_reply_msg \ -_(SNAT_ADD_ADDRESS_RANGE_REPLY, snat_add_address_range_reply) \ -_(SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY, \ - snat_interface_add_del_feature_reply) \ -_(SNAT_ADD_STATIC_MAPPING_REPLY, snat_add_static_mapping_reply) \ -_(SNAT_CONTROL_PING_REPLY, snat_control_ping_reply) \ -_(SNAT_STATIC_MAPPING_DETAILS, snat_static_mapping_details) \ -_(SNAT_SHOW_CONFIG_REPLY, snat_show_config_reply) \ -_(SNAT_ADDRESS_DETAILS, snat_address_details) \ -_(SNAT_INTERFACE_DETAILS, snat_interface_details) \ -_(SNAT_SET_WORKERS_REPLY, snat_set_workers_reply) \ -_(SNAT_WORKER_DETAILS, snat_worker_details) - -/* M: construct, but don't yet send a message */ -#define M(T,t) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -#define M2(T,t,n) \ -do { \ - vam->result_ready = 0; \ - mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ - memset (mp, 0, sizeof (*mp)); \ - mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ - mp->client_index = vam->my_client_index; \ -} while(0); - -/* S: send a message */ -#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) - -/* W: wait for results, with timeout */ -#define W \ -do { \ - timeout = vat_time_now (vam) + 1.0; \ - \ - while (vat_time_now (vam) < timeout) { \ - if (vam->result_ready == 1) { \ - return (vam->retval); \ - } \ - } \ - return -99; \ -} while(0); - -static int api_snat_add_address_range (vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - ip4_address_t start_addr, end_addr; - u32 start_host_order, end_host_order; - vl_api_snat_add_address_range_t * mp; - u8 is_add = 1; - int count; - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "%U - %U", - unformat_ip4_address, &start_addr, - unformat_ip4_address, &end_addr)) - ; - else if (unformat (i, "%U", unformat_ip4_address, &start_addr)) - end_addr = start_addr; - else if (unformat (i, "del")) - is_add = 0; - else - { - clib_warning("unknown input '%U'", format_unformat_error, i); - return -99; - } - } - - start_host_order = clib_host_to_net_u32 (start_addr.as_u32); - end_host_order = clib_host_to_net_u32 (end_addr.as_u32); - - if (end_host_order < start_host_order) - { - errmsg ("end address less than start address\n"); - return -99; - } - - count = (end_host_order - start_host_order) + 1; - - if (count > 1024) - { - errmsg ("%U - %U, %d addresses...\n", - format_ip4_address, &start_addr, - format_ip4_address, &end_addr, - count); - } - - M(SNAT_ADD_ADDRESS_RANGE, snat_add_address_range); - - memcpy (mp->first_ip_address, &start_addr, 4); - memcpy (mp->last_ip_address, &end_addr, 4); - mp->is_ip4 = 1; - mp->is_add = is_add; - - S; W; - - /* NOTREACHED */ - return 0; -} - -static int api_snat_interface_add_del_feature (vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_snat_interface_add_del_feature_t * mp; - u32 sw_if_index; - u8 sw_if_index_set = 0; - u8 is_inside = 1; - u8 is_add = 1; - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) - sw_if_index_set = 1; - else if (unformat (i, "sw_if_index %d", &sw_if_index)) - sw_if_index_set = 1; - else if (unformat (i, "out")) - is_inside = 0; - else if (unformat (i, "in")) - is_inside = 1; - else if (unformat (i, "del")) - is_add = 0; - else - { - clib_warning("unknown input '%U'", format_unformat_error, i); - return -99; - } - } - - if (sw_if_index_set == 0) - { - errmsg ("interface / sw_if_index required\n"); - return -99; - } - - M(SNAT_INTERFACE_ADD_DEL_FEATURE, snat_interface_add_del_feature); - mp->sw_if_index = ntohl(sw_if_index); - mp->is_add = is_add; - mp->is_inside = is_inside; - - S; W; - /* NOTREACHED */ - return 0; -} - -static int api_snat_add_static_mapping(vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_snat_add_static_mapping_t * mp; - u8 addr_set_n = 0; - u8 is_add = 1; - u8 addr_only = 1; - ip4_address_t local_addr, external_addr; - u32 local_port = 0, external_port = 0, vrf_id = ~0; - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "local_addr %U", unformat_ip4_address, &local_addr)) - addr_set_n++; - else if (unformat (i, "external_addr %U", unformat_ip4_address, - &external_addr)) - addr_set_n++; - else if (unformat (i, "local_port %u", &local_port)) - addr_only = 0; - else if (unformat (i, "external_port %u", &external_port)) - addr_only = 0; - else if (unformat (i, "vrf %u", &vrf_id)) - ; - else if (unformat (i, "del")) - is_add = 0; - else - { - clib_warning("unknown input '%U'", format_unformat_error, i); - return -99; - } - } - - if (addr_set_n != 2) - { - errmsg ("local_addr and remote_addr required\n"); - return -99; - } - - M(SNAT_ADD_STATIC_MAPPING, snat_add_static_mapping); - mp->is_add = is_add; - mp->is_ip4 = 1; - mp->addr_only = addr_only; - mp->local_port = ntohs ((u16) local_port); - mp->external_port = ntohs ((u16) external_port); - mp->vrf_id = ntohl (vrf_id); - memcpy (mp->local_ip_address, &local_addr, 4); - memcpy (mp->external_ip_address, &external_addr, 4); - - S; W; - /* NOTREACHED */ - return 0; -} - -static void vl_api_snat_control_ping_reply_t_handler - (vl_api_snat_control_ping_reply_t * mp) -{ - vat_main_t *vam = &vat_main; - i32 retval = ntohl (mp->retval); - if (vam->async_mode) - { - vam->async_errors += (retval < 0); - } - else - { - vam->retval = retval; - vam->result_ready = 1; - } -} - -static void vl_api_snat_static_mapping_details_t_handler - (vl_api_snat_static_mapping_details_t *mp) -{ - snat_test_main_t * sm = &snat_test_main; - vat_main_t *vam = sm->vat_main; - - if (mp->addr_only) - fformat (vam->ofp, "%15U%6s%15U%6s%11d\n", - format_ip4_address, &mp->local_ip_address, "", - format_ip4_address, &mp->external_ip_address, "", - ntohl (mp->vrf_id)); - else - fformat (vam->ofp, "%15U%6d%15U%6d%11d\n", - format_ip4_address, &mp->local_ip_address, - ntohs (mp->local_port), - format_ip4_address, &mp->external_ip_address, - ntohs (mp->external_port), - ntohl (mp->vrf_id)); - -} - -static int api_snat_static_mapping_dump(vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - f64 timeout; - vl_api_snat_static_mapping_dump_t * mp; - - if (vam->json_output) - { - clib_warning ("JSON output not supported for snat_static_mapping_dump"); - return -99; - } - - fformat (vam->ofp, "%21s%21s\n", "local", "external"); - fformat (vam->ofp, "%15s%6s%15s%6s%11s\n", "address", "port", "address", - "port", "vrf"); - - M(SNAT_STATIC_MAPPING_DUMP, snat_static_mapping_dump); - S; - /* Use a control ping for synchronization */ - { - vl_api_snat_control_ping_t *mp; - M (SNAT_CONTROL_PING, snat_control_ping); - S; - } - W; - /* NOTREACHED */ - return 0; -} - -static void vl_api_snat_show_config_reply_t_handler - (vl_api_snat_show_config_reply_t *mp) -{ - snat_test_main_t * sm = &snat_test_main; - vat_main_t *vam = sm->vat_main; - i32 retval = ntohl (mp->retval); - - if (retval >= 0) - { - fformat (vam->ofp, "translation hash buckets %d\n", - ntohl (mp->translation_buckets)); - fformat (vam->ofp, "translation hash memory %d\n", - ntohl (mp->translation_memory_size)); - fformat (vam->ofp, "user hash buckets %d\n", ntohl (mp->user_buckets)); - fformat (vam->ofp, "user hash memory %d\n", ntohl (mp->user_memory_size)); - fformat (vam->ofp, "max translations per user %d\n", - ntohl (mp->max_translations_per_user)); - fformat (vam->ofp, "outside VRF id %d\n", ntohl (mp->outside_vrf_id)); - fformat (vam->ofp, "inside VRF id %d\n", ntohl (mp->inside_vrf_id)); - if (mp->static_mapping_only) - { - fformat (vam->ofp, "static mapping only"); - if (mp->static_mapping_connection_tracking) - fformat (vam->ofp, " connection tracking"); - fformat (vam->ofp, "\n"); - } - } - vam->retval = retval; - vam->result_ready = 1; -} - -static int api_snat_show_config(vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - f64 timeout; - vl_api_snat_show_config_t * mp; - - if (vam->json_output) - { - clib_warning ("JSON output not supported for snat_show_config"); - return -99; - } - - M(SNAT_SHOW_CONFIG, snat_show_config); - S; W; - /* NOTREACHED */ - return 0; -} - -static void vl_api_snat_address_details_t_handler - (vl_api_snat_address_details_t *mp) -{ - snat_test_main_t * sm = &snat_test_main; - vat_main_t *vam = sm->vat_main; - - fformat (vam->ofp, "%U\n", format_ip4_address, &mp->ip_address); -} - -static int api_snat_address_dump(vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - f64 timeout; - vl_api_snat_address_dump_t * mp; - - if (vam->json_output) - { - clib_warning ("JSON output not supported for snat_address_dump"); - return -99; - } - - M(SNAT_ADDRESS_DUMP, snat_address_dump); - S; - /* Use a control ping for synchronization */ - { - vl_api_snat_control_ping_t *mp; - M (SNAT_CONTROL_PING, snat_control_ping); - S; - } - W; - /* NOTREACHED */ - return 0; -} - -static void vl_api_snat_interface_details_t_handler - (vl_api_snat_interface_details_t *mp) -{ - snat_test_main_t * sm = &snat_test_main; - vat_main_t *vam = sm->vat_main; - - fformat (vam->ofp, "sw_if_index %d %s\n", ntohl (mp->sw_if_index), - mp->is_inside ? "in" : "out"); -} - -static int api_snat_interface_dump(vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - f64 timeout; - vl_api_snat_interface_dump_t * mp; - - if (vam->json_output) - { - clib_warning ("JSON output not supported for snat_address_dump"); - return -99; - } - - M(SNAT_INTERFACE_DUMP, snat_interface_dump); - S; - /* Use a control ping for synchronization */ - { - vl_api_snat_control_ping_t *mp; - M (SNAT_CONTROL_PING, snat_control_ping); - S; - } - W; - /* NOTREACHED */ - return 0; -} - -static int api_snat_set_workers (vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - unformat_input_t * i = vam->input; - f64 timeout; - vl_api_snat_set_workers_t * mp; - uword *bitmap; - - while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) - { - if (unformat (i, "%U", unformat_bitmap_list, &bitmap)) - ; - else - { - clib_warning("unknown input '%U'", format_unformat_error, i); - return -99; - } - } - - M(SNAT_SET_WORKERS, snat_set_workers); - mp->worker_mask = clib_host_to_net_u64 (bitmap[0]); - - S; W; - - /* NOTREACHED */ - return 0; -} - -static void vl_api_snat_worker_details_t_handler - (vl_api_snat_worker_details_t *mp) -{ - snat_test_main_t * sm = &snat_test_main; - vat_main_t *vam = sm->vat_main; - - fformat (vam->ofp, "worker_index %d (%s at lcore %u)\n", - ntohl (mp->worker_index), mp->name, ntohl (mp->lcore_id)); -} - -static int api_snat_worker_dump(vat_main_t * vam) -{ - snat_test_main_t * sm = &snat_test_main; - f64 timeout; - vl_api_snat_worker_dump_t * mp; - - if (vam->json_output) - { - clib_warning ("JSON output not supported for snat_address_dump"); - return -99; - } - - M(SNAT_WORKER_DUMP, snat_worker_dump); - S; - /* Use a control ping for synchronization */ - { - vl_api_snat_control_ping_t *mp; - M (SNAT_CONTROL_PING, snat_control_ping); - S; - } - W; - /* NOTREACHED */ - return 0; -} - -/* - * List of messages that the api test plugin sends, - * and that the data plane plugin processes - */ -#define foreach_vpe_api_msg \ -_(snat_add_address_range, " [- | sw_if_index [in] [out] [del]") \ -_(snat_add_static_mapping, "local_addr external_addr " \ - "[local_port ] [external_port ] [vrf ] [del]") \ -_(snat_set_workers, "") \ -_(snat_static_mapping_dump, "") \ -_(snat_show_config, "") \ -_(snat_address_dump, "") \ -_(snat_interface_dump, "") \ -_(snat_worker_dump, "") - -void vat_api_hookup (vat_main_t *vam) -{ - snat_test_main_t * sm __attribute__((unused)) = &snat_test_main; - /* Hook up handlers for replies from the data plane plug-in */ -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_vpe_api_reply_msg; -#undef _ - - /* API messages we can send */ -#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); - foreach_vpe_api_msg; -#undef _ - - /* Help strings */ -#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); - foreach_vpe_api_msg; -#undef _ -} - -clib_error_t * vat_plugin_register (vat_main_t *vam) -{ - snat_test_main_t * sm = &snat_test_main; - u8 * name; - - sm->vat_main = vam; - - /* Ask the vpp engine for the first assigned message-id */ - name = format (0, "snat_%08x%c", api_version, 0); - sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); - - if (sm->msg_id_base != (u16) ~0) - vat_api_hookup (vam); - - vec_free(name); - - return 0; -} diff --git a/src/Makefile.am b/src/Makefile.am index e691a539..0fc437a4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -79,7 +79,15 @@ include vnet.am include vpp.am include vpp-api-test.am -SUBDIRS += vpp-api/python plugins +SUBDIRS += plugins + +if ENABLE_PAPI +SUBDIRS += vpp-api/python +endif + +if ENABLE_JAPI +SUBDIRS += vpp-api/java +endif ############################################################################### # API diff --git a/src/configure.ac b/src/configure.ac index f8e4d94f..4ed55705 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -3,7 +3,8 @@ LT_INIT AC_CONFIG_AUX_DIR([.]) AM_INIT_AUTOMAKE([subdir-objects]) AM_SILENT_RULES([yes]) -AC_CONFIG_FILES([Makefile plugins/Makefile vpp-api/python/Makefile]) +AC_CONFIG_FILES([Makefile plugins/Makefile vpp-api/python/Makefile vpp-api/java/Makefile]) +AC_CONFIG_MACRO_DIR([m4]) AC_PROG_CC AM_PROG_AS @@ -89,6 +90,8 @@ ENABLE_ARG(g2, [Enable g2]) # --disable-X DISABLE_ARG(vlib, [Disable vlib and dependant libs and binaries]) DISABLE_ARG(svm, [Disable svm and dependant libs and binaries]) +DISABLE_ARG(papi, [Disable Python API bindings]) +DISABLE_ARG(japi, [Disable Java API bindings]) # --with-X WITH_ARG(dpdk, [Use use DPDK]) @@ -131,6 +134,25 @@ AC_DEFINE_UNQUOTED(DPDK_CRYPTO, [${n_with_dpdk_crypto}]) AC_DEFINE_UNQUOTED(IPSEC, [${n_with_ipsec}]) AC_DEFINE_UNQUOTED(IPV6SR, [${n_with_ipv6sr}]) + +# Silence following noise: +# ar: `u' modifier ignored since `D' is the default (see `U') +AR_FLAGS=cr +AC_SUBST(AR_FLAGS) + +############################################################################### +# Plugins +############################################################################### + +# Please keep alphabetical order +PLUGIN_ENABLED(acl) +PLUGIN_ENABLED(flowperpkt) +PLUGIN_ENABLED(ila) +PLUGIN_ENABLED(ioam) +PLUGIN_ENABLED(lb) +PLUGIN_ENABLED(sixrd) +PLUGIN_ENABLED(snat) + ############################################################################### # Dependency checks ############################################################################### @@ -151,12 +173,17 @@ AM_COND_IF([ENABLE_G2], ]) ############################################################################### -# Plugins +# JAVA ############################################################################### -PLUGIN_ENABLED(sixrd) -PLUGIN_ENABLED(ila) -PLUGIN_ENABLED(flowperpkt) +AM_COND_IF([ENABLE_JAPI], +[ + AX_VPP_FIND_JDK8 + AC_SUBST(JAVA_HOME) + AC_SUBST(JAVAC) + AC_SUBST(JAVAH) + AC_SUBST(JAR) +]) ############################################################################### # Output @@ -172,6 +199,11 @@ PRINT_VAL([includedir], ${includedir}) PRINT_VAL([CFLAGS], ${CFLAGS}) PRINT_VAL([CPPFLAGS], ${CPPFLAGS}) PRINT_VAL([LDFLAGS], ${LDFLAGS}) +AM_COND_IF([ENABLE_JAPI], +[ + PRINT_VAL([JAVA_VERSION], ${JAVA_VERSION}) + PRINT_VAL([JAVA_HOME], ${JAVA_HOME}) +]) AC_MSG_RESULT([]) AC_MSG_RESULT([with:]) diff --git a/src/examples/sample-plugin/Makefile.am b/src/examples/sample-plugin/Makefile.am new file mode 100644 index 00000000..e221f8c1 --- /dev/null +++ b/src/examples/sample-plugin/Makefile.am @@ -0,0 +1,56 @@ +# Copyright (c) 2015 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AUTOMAKE_OPTIONS = foreign subdir-objects + +AM_CFLAGS = -Wall +AM_LDFLAGS = -module -shared -avoid-version + +vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins +vpppluginsdir = ${libdir}/vpp_plugins + +vppapitestplugins_LTLIBRARIES = sample_test_plugin.la +vppplugins_LTLIBRARIES = sample_plugin.la + +sample_plugin_la_SOURCES = sample/sample.c sample/node.c \ + sample/sample_plugin.api.h + +BUILT_SOURCES = sample/sample.api.h sample/sample.api.json + +SUFFIXES = .api.h .api + +%.api.h: %.api + mkdir -p `dirname $@` ; \ + $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ + | vppapigen --input - --output $@ --show-name $@ + +%.api.json: %.api + @echo " JSON APIGEN " $@ ; \ + mkdir -p `dirname $@` ; \ + $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ + | vppapigen --input - --json $@ + +apidir = $(prefix)/sample/ +api_DATA = sample.api.json + +noinst_HEADERS = \ + sample/sample_all_api_h.h \ + sample/sample_msg_enum.h \ + sample/sample.api.h + +sample_test_plugin_la_SOURCES = sample/sample_test.c sample/sample_plugin.api.h + +# Remove *.la files +install-data-hook: + @(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES)) + @(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES)) diff --git a/src/examples/sample-plugin/configure.ac b/src/examples/sample-plugin/configure.ac new file mode 100644 index 00000000..43642732 --- /dev/null +++ b/src/examples/sample-plugin/configure.ac @@ -0,0 +1,9 @@ +AC_INIT(sample_plugin, 1.0) +LT_INIT +AM_INIT_AUTOMAKE +AM_SILENT_RULES([yes]) +AC_PREFIX_DEFAULT([/usr]) + +AC_PROG_CC + +AC_OUTPUT([Makefile]) diff --git a/src/examples/sample-plugin/sample/node.c b/src/examples/sample-plugin/sample/node.c new file mode 100644 index 00000000..94c1706b --- /dev/null +++ b/src/examples/sample-plugin/sample/node.c @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +typedef struct { + u32 next_index; + u32 sw_if_index; + u8 new_src_mac[6]; + u8 new_dst_mac[6]; +} sample_trace_t; + +static u8 * +format_mac_address (u8 * s, va_list * args) +{ + u8 *a = va_arg (*args, u8 *); + return format (s, "%02x:%02x:%02x:%02x:%02x:%02x", + a[0], a[1], a[2], a[3], a[4], a[5]); +} + +/* packet trace format function */ +static u8 * format_sample_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + sample_trace_t * t = va_arg (*args, sample_trace_t *); + + s = format (s, "SAMPLE: sw_if_index %d, next index %d\n", + t->sw_if_index, t->next_index); + s = format (s, " new src %U -> new dst %U", + format_mac_address, t->new_src_mac, + format_mac_address, t->new_dst_mac); + + return s; +} + +vlib_node_registration_t sample_node; + +#define foreach_sample_error \ +_(SWAPPED, "Mac swap packets processed") + +typedef enum { +#define _(sym,str) SAMPLE_ERROR_##sym, + foreach_sample_error +#undef _ + SAMPLE_N_ERROR, +} sample_error_t; + +static char * sample_error_strings[] = { +#define _(sym,string) string, + foreach_sample_error +#undef _ +}; + +typedef enum { + SAMPLE_NEXT_INTERFACE_OUTPUT, + SAMPLE_N_NEXT, +} sample_next_t; + +#define foreach_mac_address_offset \ +_(0) \ +_(1) \ +_(2) \ +_(3) \ +_(4) \ +_(5) + +static uword +sample_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + u32 n_left_from, * from, * to_next; + sample_next_t next_index; + u32 pkts_swapped = 0; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 next0 = SAMPLE_NEXT_INTERFACE_OUTPUT; + u32 next1 = SAMPLE_NEXT_INTERFACE_OUTPUT; + u32 sw_if_index0, sw_if_index1; + u8 tmp0[6], tmp1[6]; + ethernet_header_t *en0, *en1; + u32 bi0, bi1; + vlib_buffer_t * b0, * b1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * p2, * p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + /* speculatively enqueue b0 and b1 to the current next frame */ + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + ASSERT (b0->current_data == 0); + ASSERT (b1->current_data == 0); + + en0 = vlib_buffer_get_current (b0); + en1 = vlib_buffer_get_current (b1); + + /* This is not the fastest way to swap src + dst mac addresses */ +#define _(a) tmp0[a] = en0->src_address[a]; + foreach_mac_address_offset; +#undef _ +#define _(a) en0->src_address[a] = en0->dst_address[a]; + foreach_mac_address_offset; +#undef _ +#define _(a) en0->dst_address[a] = tmp0[a]; + foreach_mac_address_offset; +#undef _ + +#define _(a) tmp1[a] = en1->src_address[a]; + foreach_mac_address_offset; +#undef _ +#define _(a) en1->src_address[a] = en1->dst_address[a]; + foreach_mac_address_offset; +#undef _ +#define _(a) en1->dst_address[a] = tmp1[a]; + foreach_mac_address_offset; +#undef _ + + + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX]; + + /* Send pkt back out the RX interface */ + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sw_if_index0; + vnet_buffer(b1)->sw_if_index[VLIB_TX] = sw_if_index1; + + pkts_swapped += 2; + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (b0->flags & VLIB_BUFFER_IS_TRACED) + { + sample_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + clib_memcpy (t->new_src_mac, en0->src_address, + sizeof (t->new_src_mac)); + clib_memcpy (t->new_dst_mac, en0->dst_address, + sizeof (t->new_dst_mac)); + + } + if (b1->flags & VLIB_BUFFER_IS_TRACED) + { + sample_trace_t *t = + vlib_add_trace (vm, node, b1, sizeof (*t)); + t->sw_if_index = sw_if_index1; + t->next_index = next1; + clib_memcpy (t->new_src_mac, en1->src_address, + sizeof (t->new_src_mac)); + clib_memcpy (t->new_dst_mac, en1->dst_address, + sizeof (t->new_dst_mac)); + } + } + + /* verify speculative enqueues, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, + to_next, n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t * b0; + u32 next0 = SAMPLE_NEXT_INTERFACE_OUTPUT; + u32 sw_if_index0; + u8 tmp0[6]; + ethernet_header_t *en0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + /* + * Direct from the driver, we should be at offset 0 + * aka at &b0->data[0] + */ + ASSERT (b0->current_data == 0); + + en0 = vlib_buffer_get_current (b0); + + /* This is not the fastest way to swap src + dst mac addresses */ +#define _(a) tmp0[a] = en0->src_address[a]; + foreach_mac_address_offset; +#undef _ +#define _(a) en0->src_address[a] = en0->dst_address[a]; + foreach_mac_address_offset; +#undef _ +#define _(a) en0->dst_address[a] = tmp0[a]; + foreach_mac_address_offset; +#undef _ + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + + /* Send pkt back out the RX interface */ + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sw_if_index0; + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) { + sample_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + clib_memcpy (t->new_src_mac, en0->src_address, + sizeof (t->new_src_mac)); + clib_memcpy (t->new_dst_mac, en0->dst_address, + sizeof (t->new_dst_mac)); + } + + pkts_swapped += 1; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, sample_node.index, + SAMPLE_ERROR_SWAPPED, pkts_swapped); + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (sample_node) = { + .function = sample_node_fn, + .name = "sample", + .vector_size = sizeof (u32), + .format_trace = format_sample_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(sample_error_strings), + .error_strings = sample_error_strings, + + .n_next_nodes = SAMPLE_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [SAMPLE_NEXT_INTERFACE_OUTPUT] = "interface-output", + }, +}; diff --git a/src/examples/sample-plugin/sample/sample.api b/src/examples/sample-plugin/sample/sample.api new file mode 100644 index 00000000..f99cdb38 --- /dev/null +++ b/src/examples/sample-plugin/sample/sample.api @@ -0,0 +1,39 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Define a simple binary API to control the feature */ + +define sample_macswap_enable_disable { + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + + /* Enable / disable the feature */ + u8 enable_disable; + + /* Interface handle */ + u32 sw_if_index; +}; + +define sample_macswap_enable_disable_reply { + /* From the request */ + u32 context; + + /* Return value, zero means all OK */ + i32 retval; +}; diff --git a/src/examples/sample-plugin/sample/sample.c b/src/examples/sample-plugin/sample/sample.c new file mode 100644 index 00000000..603cb2d0 --- /dev/null +++ b/src/examples/sample-plugin/sample/sample.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * sample.c - simple MAC-swap API / debug CLI handling + *------------------------------------------------------------------ + */ + +#include +#include +#include + +#include +#include +#include + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + + +/* List of message types that this plugin understands */ + +#define foreach_sample_plugin_api_msg \ +_(SAMPLE_MACSWAP_ENABLE_DISABLE, sample_macswap_enable_disable) + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + sample_main_t * sm = &sample_main; + clib_error_t * error = 0; + + sm->vlib_main = vm; + sm->vnet_main = h->vnet_main; + sm->ethernet_main = h->ethernet_main; + + return error; +} + +/* Action function shared between message handler and debug CLI */ + +int sample_macswap_enable_disable (sample_main_t * sm, u32 sw_if_index, + int enable_disable) +{ + vnet_sw_interface_t * sw; + int rv = 0; + + /* Utterly wrong? */ + if (pool_is_free_index (sm->vnet_main->interface_main.sw_interfaces, + sw_if_index)) + return VNET_API_ERROR_INVALID_SW_IF_INDEX; + + /* Not a physical port? */ + sw = vnet_get_sw_interface (sm->vnet_main, sw_if_index); + if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE) + return VNET_API_ERROR_INVALID_SW_IF_INDEX; + + vnet_feature_enable_disable ("device-input", "sample", + sw_if_index, enable_disable, 0, 0); + + return rv; +} + +static clib_error_t * +macswap_enable_disable_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + sample_main_t * sm = &sample_main; + u32 sw_if_index = ~0; + int enable_disable = 1; + + int rv; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { + if (unformat (input, "disable")) + enable_disable = 0; + else if (unformat (input, "%U", unformat_vnet_sw_interface, + sm->vnet_main, &sw_if_index)) + ; + else + break; + } + + if (sw_if_index == ~0) + return clib_error_return (0, "Please specify an interface..."); + + rv = sample_macswap_enable_disable (sm, sw_if_index, enable_disable); + + switch(rv) { + case 0: + break; + + case VNET_API_ERROR_INVALID_SW_IF_INDEX: + return clib_error_return + (0, "Invalid interface, only works on physical ports"); + break; + + case VNET_API_ERROR_UNIMPLEMENTED: + return clib_error_return (0, "Device driver doesn't support redirection"); + break; + + default: + return clib_error_return (0, "sample_macswap_enable_disable returned %d", + rv); + } + return 0; +} + +VLIB_CLI_COMMAND (sr_content_command, static) = { + .path = "sample macswap", + .short_help = + "sample macswap [disable]", + .function = macswap_enable_disable_command_fn, +}; + +/* API message handler */ +static void vl_api_sample_macswap_enable_disable_t_handler +(vl_api_sample_macswap_enable_disable_t * mp) +{ + vl_api_sample_macswap_enable_disable_reply_t * rmp; + sample_main_t * sm = &sample_main; + int rv; + + rv = sample_macswap_enable_disable (sm, ntohl(mp->sw_if_index), + (int) (mp->enable_disable)); + + REPLY_MACRO(VL_API_SAMPLE_MACSWAP_ENABLE_DISABLE_REPLY); +} + +/* Set up the API message handling tables */ +static clib_error_t * +sample_plugin_api_hookup (vlib_main_t *vm) +{ + sample_main_t * sm = &sample_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_sample_plugin_api_msg; +#undef _ + + return 0; +} + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (sample_main_t * sm, api_main_t *am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); + foreach_vl_msg_name_crc_sample; +#undef _ +} + +static clib_error_t * sample_init (vlib_main_t * vm) +{ + sample_main_t * sm = &sample_main; + clib_error_t * error = 0; + u8 * name; + + name = format (0, "sample_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + sm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + + error = sample_plugin_api_hookup (vm); + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (sm, &api_main); + + vec_free(name); + + return error; +} + +VLIB_INIT_FUNCTION (sample_init); + +VNET_FEATURE_INIT (sample, static) = +{ + .arc_name = "device-input", + .node_name = "sample", + .runs_before = VNET_FEATURES ("ethernet-input"), +}; diff --git a/src/examples/sample-plugin/sample/sample.h b/src/examples/sample-plugin/sample/sample.h new file mode 100644 index 00000000..d268d482 --- /dev/null +++ b/src/examples/sample-plugin/sample/sample.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_sample_h__ +#define __included_sample_h__ + +#include +#include +#include + +#include +#include +#include + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; + ethernet_main_t * ethernet_main; +} sample_main_t; + +sample_main_t sample_main; + +vlib_node_registration_t sample_node; + +#endif /* __included_sample_h__ */ diff --git a/src/examples/sample-plugin/sample/sample_all_api_h.h b/src/examples/sample-plugin/sample/sample_all_api_h.h new file mode 100644 index 00000000..774d782f --- /dev/null +++ b/src/examples/sample-plugin/sample/sample_all_api_h.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/examples/sample-plugin/sample/sample_msg_enum.h b/src/examples/sample-plugin/sample/sample_msg_enum.h new file mode 100644 index 00000000..af4172f7 --- /dev/null +++ b/src/examples/sample-plugin/sample/sample_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_sample_msg_enum_h +#define included_sample_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_sample_msg_enum_h */ diff --git a/src/examples/sample-plugin/sample/sample_test.c b/src/examples/sample-plugin/sample/sample_test.c new file mode 100644 index 00000000..dd1b0215 --- /dev/null +++ b/src/examples/sample-plugin/sample/sample_test.c @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * sample_test.c - test harness plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include + +uword unformat_sw_if_index (unformat_input_t * input, va_list * args); + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} sample_test_main_t; + +sample_test_main_t sample_test_main; + +#define foreach_standard_reply_retval_handler \ +_(sample_macswap_enable_disable_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = sample_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(SAMPLE_MACSWAP_ENABLE_DISABLE_REPLY, sample_macswap_enable_disable_reply) + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int api_sample_macswap_enable_disable (vat_main_t * vam) +{ + sample_test_main_t * sm = &sample_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + int enable_disable = 1; + u32 sw_if_index = ~0; + vl_api_sample_macswap_enable_disable_t * mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + ; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + ; + else if (unformat (i, "disable")) + enable_disable = 0; + else + break; + } + + if (sw_if_index == ~0) { + errmsg ("missing interface name / explicit sw_if_index number \n"); + return -99; + } + + /* Construct the API message */ + M(SAMPLE_MACSWAP_ENABLE_DISABLE, sample_macswap_enable_disable); + mp->sw_if_index = ntohl (sw_if_index); + mp->enable_disable = enable_disable; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(sample_macswap_enable_disable, " [disable]") + +void vat_api_hookup (vat_main_t *vam) +{ + sample_test_main_t * sm = &sample_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * vat_plugin_register (vat_main_t *vam) +{ + sample_test_main_t * sm = &sample_test_main; + u8 * name; + + sm->vat_main = vam; + + name = format (0, "sample_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~0) + vat_api_hookup (vam); + + vec_free(name); + + return 0; +} diff --git a/src/m4/ax_vpp_find_jdk8.m4 b/src/m4/ax_vpp_find_jdk8.m4 new file mode 100644 index 00000000..29459822 --- /dev/null +++ b/src/m4/ax_vpp_find_jdk8.m4 @@ -0,0 +1,29 @@ + + +AC_DEFUN([AX_VPP_FIND_JDK8], +[ +while true +do + test "${JAVA_HOME+set}" = set && break + + for dir in $(find /usr/lib/jvm/* -maxdepth 0 -type d); do + AC_MSG_CHECKING([${dir} for Java 8 compiler]) + JAVA_VERSION=$(${dir}/bin/javac -source 8 -version 2>&1) + if test 0 -eq "$?"; then + JAVA_VERSION=$(echo "${JAVA_VERSION}" | cut -d\ -f2) + JAVA_HOME=${dir} + JAVAC=${dir}/bin/javac + JAVAH=${dir}/bin/javah + JAR=${dir}/bin/jar + AC_MSG_RESULT([found version $JAVA_VERSION]) + break + else + JAVA_VERSION="" + AC_MSG_RESULT([no]) + fi + done + + test "${JAVA_HOME}set" = set && AC_MSG_ERROR([Could not find Java 8 compiler]) + break +done +]) diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index ffc4b3ab..987310b7 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -22,10 +22,15 @@ BUILT_SOURCES = vppplugins_LTLIBRARIES = vppapitestplugins_LTLIBRARIES = noinst_HEADERS = +nobase_apiinclude_HEADERS = vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins vpppluginsdir = ${libdir}/vpp_plugins +if ENABLE_ACL_PLUGIN +include acl.am +endif + if ENABLE_FLOWPERPKT_PLUGIN include flowperpkt.am endif @@ -34,10 +39,22 @@ if ENABLE_ILA_PLUGIN include ila.am endif +if ENABLE_IOAM_PLUGIN +include ioam.am +endif + +if ENABLE_LB_PLUGIN +include lb.am +endif + if ENABLE_SIXRD_PLUGIN include sixrd.am endif +if ENABLE_SNAT_PLUGIN +include snat.am +endif + include ../suffix-rules.mk # Remove *.la files @@ -50,6 +67,7 @@ install-data-hook: ############################################################################### apidir = $(prefix)/share/vpp/api/plugins +apiincludedir = ${includedir}/vpp_plugins api_DATA = \ $(patsubst %.api,%.api.json,$(API_FILES)) diff --git a/src/plugins/acl.am b/src/plugins/acl.am new file mode 100644 index 00000000..efed31c2 --- /dev/null +++ b/src/plugins/acl.am @@ -0,0 +1,35 @@ +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +vppapitestplugins_LTLIBRARIES += acl_test_plugin.la +vppplugins_LTLIBRARIES += acl_plugin.la + +acl_plugin_la_SOURCES = \ + acl/acl.c \ + acl/node_in.c \ + acl/node_out.c \ + acl/l2sess.c \ + acl/l2sess_node.c \ + acl/l2sess.h \ + acl/acl_plugin.api.h + +API_FILES += acl/acl.api + +nobase_apiinclude_HEADERS += \ + acl/acl_all_api_h.h \ + acl/acl_msg_enum.h \ + acl/acl.api.h + +acl_test_plugin_la_SOURCES = acl/acl_test.c acl/acl_plugin.api.h + +# vi:syntax=automake diff --git a/src/plugins/acl/acl.api b/src/plugins/acl/acl.api new file mode 100644 index 00000000..58a5a171 --- /dev/null +++ b/src/plugins/acl/acl.api @@ -0,0 +1,444 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** \file + This file defines the vpp control-plane API messages + used to control the ACL plugin +*/ + + +/** \brief Get the plugin version + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ + +define acl_plugin_get_version +{ + u32 client_index; + u32 context; +}; + +/** \brief Reply to get the plugin version + @param context - returned sender context, to match reply w/ request + @param major - Incremented every time a known breaking behavior change is introduced + @param minor - Incremented with small changes, may be used to avoid buggy versions +*/ + +define acl_plugin_get_version_reply +{ + u32 context; + u32 major; + u32 minor; +}; + +/** \brief Access List Rule entry + @param is_permit - deny (0), permit (1), or permit+reflect(2) action on this rule. + @param is_ipv6 - IP addresses in this rule are IPv6 (1) or IPv4 (0) + @param src_ip_addr - Source prefix value + @param src_ip_prefix_len - Source prefix length + @param dst_ip_addr - Destination prefix value + @param dst_ip_prefix_len - Destination prefix length + @param proto - L4 protocol (http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml) + @param srcport_or_icmptype_first - beginning of source port or ICMP4/6 type range + @param srcport_or_icmptype_last - end of source port or ICMP4/6 type range + @param dstport_or_icmpcode_first - beginning of destination port or ICMP4/6 code range + @param dstport_or_icmpcode_last - end of destination port or ICMP4/6 code range + @param tcp_flags_mask - if proto==6, match masked TCP flags with this value + @param tcp_flags_value - if proto==6, mask to AND the TCP flags in the packet with +*/ + +typeonly manual_print manual_endian define acl_rule +{ + u8 is_permit; + u8 is_ipv6; + u8 src_ip_addr[16]; + u8 src_ip_prefix_len; + u8 dst_ip_addr[16]; + u8 dst_ip_prefix_len; +/* + * L4 protocol. IANA number. 1 = ICMP, 58 = ICMPv6, 6 = TCP, 17 = UDP. + * 0 => ignore L4 and ignore the ports/tcpflags when matching. + */ + u8 proto; +/* + * If the L4 protocol is TCP or UDP, the below + * hold ranges of ports, else if the L4 is ICMP/ICMPv6 + * they hold ranges of ICMP(v6) types/codes. + * + * Ranges are inclusive, i.e. to match "any" TCP/UDP port, + * use first=0,last=65535. For ICMP(v6), + * use first=0,last=255. + */ + u16 srcport_or_icmptype_first; + u16 srcport_or_icmptype_last; + u16 dstport_or_icmpcode_first; + u16 dstport_or_icmpcode_last; +/* + * for proto = 6, this matches if the + * TCP flags in the packet, ANDed with tcp_flags_mask, + * is equal to tcp_flags_value. + */ + u8 tcp_flags_mask; + u8 tcp_flags_value; +}; + +/** \brief MACIP Access List Rule entry + @param is_permit - deny (0), permit (1) action on this rule. + @param is_ipv6 - IP addresses in this rule are IPv6 (1) or IPv4 (0) + @param src_mac - match masked source MAC address against this value + @param src_mac_mask - AND source MAC address with this value before matching + @param src_ip_addr - Source prefix value + @param src_ip_prefix_len - Source prefix length +*/ + +typeonly manual_print manual_endian define macip_acl_rule +{ + u8 is_permit; + u8 is_ipv6; +/* + * The source mac of the packet ANDed with src_mac_mask. + * The source ip[46] address in the packet is matched + * against src_ip_addr, with src_ip_prefix_len set to 0. + * + * For better performance, minimize the number of + * (src_mac_mask, src_ip_prefix_len) combinations + * in a MACIP ACL. + */ + u8 src_mac[6]; + u8 src_mac_mask[6]; + u8 src_ip_addr[16]; + u8 src_ip_prefix_len; +}; + +/** \brief Replace an existing ACL in-place or create a new ACL + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param acl_index - an existing ACL entry (0..0xfffffffe) to replace, or 0xffffffff to make new ACL + @param tag - a string value stored along with the ACL, for descriptive purposes + @param count - number of ACL rules + @r - Rules for this access-list +*/ + +manual_print manual_endian define acl_add_replace +{ + u32 client_index; + u32 context; + u32 acl_index; /* ~0 to add, existing ACL# to replace */ + u8 tag[64]; /* What gets in here gets out in the corresponding tag field when dumping the ACLs. */ + u32 count; + vl_api_acl_rule_t r[count]; +}; + +/** \brief Reply to add/replace ACL + @param context - returned sender context, to match reply w/ request + @param acl_index - index of the updated or newly created ACL + @param retval 0 - no error +*/ + +define acl_add_replace_reply +{ + u32 context; + u32 acl_index; + i32 retval; +}; + +/** \brief Delete an ACL + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param acl_index - ACL index to delete +*/ + +define acl_del +{ + u32 client_index; + u32 context; + u32 acl_index; +}; + +/** \brief Reply to delete the ACL + @param context - returned sender context, to match reply w/ request + @param retval 0 - no error +*/ + +define acl_del_reply +{ + u32 context; + i32 retval; +}; + +/* acl_interface_add_del(_reply) to be deprecated in lieu of acl_interface_set_acl_list */ +/** \brief Use acl_interface_set_acl_list instead + Append/remove an ACL index to/from the list of ACLs checked for an interface + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param is_add - add or delete the ACL index from the list + @param is_input - check the ACL on input (1) or output (0) + @param sw_if_index - the interface to alter the list of ACLs on + @param acl_index - index of ACL for the operation +*/ + +define acl_interface_add_del +{ + u32 client_index; + u32 context; + u8 is_add; +/* + * is_input = 0 => ACL applied on interface egress + * is_input = 1 => ACL applied on interface ingress + */ + u8 is_input; + u32 sw_if_index; + u32 acl_index; +}; + +/** \brief Reply to alter the ACL list + @param context - returned sender context, to match reply w/ request + @param retval 0 - no error +*/ + +define acl_interface_add_del_reply +{ + u32 context; + i32 retval; +}; + +/** \brief Set the vector of input/output ACLs checked for an interface + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param sw_if_index - the interface to alter the list of ACLs on + @param count - total number of ACL indices in the vector + @param n_input - this many first elements correspond to input ACLs, the rest - output + @param acls - vector of ACL indices +*/ + +manual_endian define acl_interface_set_acl_list +{ + u32 client_index; + u32 context; + u32 sw_if_index; + u8 count; + u8 n_input; /* First n_input ACLs are set as a list of input ACLs, the rest are applied as output */ + u32 acls[count]; +}; + +/** \brief Reply to set the ACL list on an interface + @param context - returned sender context, to match reply w/ request + @param retval 0 - no error +*/ + +define acl_interface_set_acl_list_reply +{ + u32 context; + i32 retval; +}; + +/** \brief Dump the specific ACL contents or all of the ACLs' contents + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param acl_index - ACL index to dump, ~0 to dump all ACLs +*/ + +define acl_dump +{ + u32 client_index; + u32 context; + u32 acl_index; /* ~0 for all ACLs */ +}; + +/** \brief Details about a single ACL contents + @param context - returned sender context, to match reply w/ request + @param acl_index - ACL index whose contents are being sent in this message + @param tag - Descriptive tag value which was supplied at ACL creation + @param count - Number of rules in this ACL + @param r - Array of rules within this ACL +*/ + +manual_print manual_endian define acl_details +{ + u32 context; + u32 acl_index; + u8 tag[64]; /* Same blob that was supplied to us when creating the ACL, one hopes. */ + u32 count; + vl_api_acl_rule_t r[count]; +}; + +/** \brief Dump the list(s) of ACL applied to specific or all interfaces + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param sw_if_index - interface to dump the ACL list for +*/ + +define acl_interface_list_dump +{ + u32 client_index; + u32 context; + u32 sw_if_index; /* ~0 for all interfaces */ +}; + +/** \brief Details about a single ACL contents + @param context - returned sender context, to match reply w/ request + @param sw_if_index - interface for which the list of ACLs is applied + @param count - total length of acl indices vector + @param n_input - this many of indices in the beginning are input ACLs, the rest - output + @param acls - the vector of ACL indices +*/ + +manual_endian define acl_interface_list_details +{ + u32 context; + u32 sw_if_index; + u8 count; + u8 n_input; + u32 acls[count]; +}; + +/** \brief Add a MACIP ACL + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param tag - descriptive value for this MACIP ACL + @param count - number of rules in this ACL + @param r - vector of MACIP ACL rules +*/ + +manual_print manual_endian define macip_acl_add +{ + u32 client_index; + u32 context; + u8 tag[64]; + u32 count; + vl_api_macip_acl_rule_t r[count]; +}; + +/** \brief Reply to add MACIP ACL + @param context - returned sender context, to match reply w/ request + @param acl_index - index of the newly created ACL + @param retval 0 - no error +*/ + +define macip_acl_add_reply +{ + u32 context; + u32 acl_index; + i32 retval; +}; + +/** \brief Delete a MACIP ACL + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param acl_index - MACIP ACL index to delete +*/ + +define macip_acl_del +{ + u32 client_index; + u32 context; + u32 acl_index; +}; + +/** \brief Reply to delete the MACIP ACL + @param context - returned sender context, to match reply w/ request + @param retval 0 - no error +*/ + +define macip_acl_del_reply +{ + u32 context; + i32 retval; +}; + +/** \brief Add or delete a MACIP ACL to/from interface + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param is_add - add (1) or delete (0) ACL from being used on an interface + @param sw_if_index - interface to apply the action to + @param acl_index - MACIP ACL index +*/ + +define macip_acl_interface_add_del +{ + u32 client_index; + u32 context; + u8 is_add; + /* macip ACLs are always input */ + u32 sw_if_index; + u32 acl_index; +}; + +/** \brief Reply to apply/unapply the MACIP ACL + @param context - returned sender context, to match reply w/ request + @param retval 0 - no error +*/ + +define macip_acl_interface_add_del_reply +{ + u32 context; + i32 retval; +}; + +/** \brief Dump one or all defined MACIP ACLs + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param acl_index - MACIP ACL index or ~0 to dump all ACLs +*/ + +define macip_acl_dump +{ + u32 client_index; + u32 context; + u32 acl_index; /* ~0 for all ACLs */ +}; + +/** \brief Details about one MACIP ACL + @param context - returned sender context, to match reply w/ request + @param acl_index - index of this MACIP ACL + @param tag - descriptive tag which was supplied during the creation + @param count - length of the vector of MACIP ACL rules + @param r - rules comprising this ACL +*/ + +manual_print manual_endian define macip_acl_details +{ + u32 context; + u32 acl_index; + u8 tag[64]; + u32 count; + vl_api_macip_acl_rule_t r[count]; +}; + +/** \brief Get the vector of MACIP ACL IDs applied to the interfaces + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ + +define macip_acl_interface_get +{ + u32 client_index; + u32 context; +}; + +/** \brief Reply with the vector of MACIP ACLs by sw_if_index + @param context - returned sender context, to match reply w/ request + @param count - total number of elements in the vector + @param acls - the vector of active MACACL indices per sw_if_index +*/ + +define macip_acl_interface_get_reply +{ + u32 context; + u32 count; + u32 acls[count]; +}; + diff --git a/src/plugins/acl/acl.c b/src/plugins/acl/acl.c new file mode 100644 index 00000000..8ff5a6b7 --- /dev/null +++ b/src/plugins/acl/acl.c @@ -0,0 +1,1901 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +#include "node_in.h" +#include "node_out.h" + +acl_main_t acl_main; + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +#define REPLY_MACRO2(t, body) \ +do { \ + unix_shared_memory_queue_t * q; \ + rv = vl_msg_api_pd_handler (mp, rv); \ + q = vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+am->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + do {body;} while (0); \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +#define REPLY_MACRO3(t, n, body) \ +do { \ + unix_shared_memory_queue_t * q; \ + rv = vl_msg_api_pd_handler (mp, rv); \ + q = vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp) + n); \ + rmp->_vl_msg_id = ntohs((t)+am->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + do {body;} while (0); \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + + +/* List of message types that this plugin understands */ + +#define foreach_acl_plugin_api_msg \ +_(ACL_PLUGIN_GET_VERSION, acl_plugin_get_version) \ +_(ACL_ADD_REPLACE, acl_add_replace) \ +_(ACL_DEL, acl_del) \ +_(ACL_INTERFACE_ADD_DEL, acl_interface_add_del) \ +_(ACL_INTERFACE_SET_ACL_LIST, acl_interface_set_acl_list) \ +_(ACL_DUMP, acl_dump) \ +_(ACL_INTERFACE_LIST_DUMP, acl_interface_list_dump) \ +_(MACIP_ACL_ADD, macip_acl_add) \ +_(MACIP_ACL_DEL, macip_acl_del) \ +_(MACIP_ACL_INTERFACE_ADD_DEL, macip_acl_interface_add_del) \ +_(MACIP_ACL_DUMP, macip_acl_dump) \ +_(MACIP_ACL_INTERFACE_GET, macip_acl_interface_get) + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + acl_main_t *am = &acl_main; + clib_error_t *error = 0; + + am->vlib_main = vm; + am->vnet_main = h->vnet_main; + am->ethernet_main = h->ethernet_main; + + l2sess_vlib_plugin_register(vm, h, from_early_init); + + return error; +} + + +static void +vl_api_acl_plugin_get_version_t_handler (vl_api_acl_plugin_get_version_t * mp) +{ + acl_main_t *am = &acl_main; + vl_api_acl_plugin_get_version_reply_t *rmp; + int msg_size = sizeof (*rmp); + unix_shared_memory_queue_t *q; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + { + return; + } + + rmp = vl_msg_api_alloc (msg_size); + memset (rmp, 0, msg_size); + rmp->_vl_msg_id = + ntohs (VL_API_ACL_PLUGIN_GET_VERSION_REPLY + am->msg_id_base); + rmp->context = mp->context; + rmp->major = htonl (ACL_PLUGIN_VERSION_MAJOR); + rmp->minor = htonl (ACL_PLUGIN_VERSION_MINOR); + + vl_msg_api_send_shmem (q, (u8 *) & rmp); +} + + +static int +acl_add_list (u32 count, vl_api_acl_rule_t rules[], + u32 * acl_list_index, u8 * tag) +{ + acl_main_t *am = &acl_main; + acl_list_t *a; + acl_rule_t *r; + acl_rule_t *acl_new_rules; + int i; + + if (*acl_list_index != ~0) + { + /* They supplied some number, let's see if this ACL exists */ + if (pool_is_free_index (am->acls, *acl_list_index)) + { + /* tried to replace a non-existent ACL, no point doing anything */ + return -1; + } + } + + /* Create and populate the rules */ + acl_new_rules = clib_mem_alloc_aligned (sizeof (acl_rule_t) * count, + CLIB_CACHE_LINE_BYTES); + if (!acl_new_rules) + { + /* Could not allocate rules. New or existing ACL - bail out regardless */ + return -1; + } + + for (i = 0; i < count; i++) + { + r = &acl_new_rules[i]; + r->is_permit = rules[i].is_permit; + r->is_ipv6 = rules[i].is_ipv6; + if (r->is_ipv6) + { + memcpy (&r->src, rules[i].src_ip_addr, sizeof (r->src)); + memcpy (&r->dst, rules[i].dst_ip_addr, sizeof (r->dst)); + } + else + { + memcpy (&r->src.ip4, rules[i].src_ip_addr, sizeof (r->src.ip4)); + memcpy (&r->dst.ip4, rules[i].dst_ip_addr, sizeof (r->dst.ip4)); + } + r->src_prefixlen = rules[i].src_ip_prefix_len; + r->dst_prefixlen = rules[i].dst_ip_prefix_len; + r->proto = rules[i].proto; + r->src_port_or_type_first = rules[i].srcport_or_icmptype_first; + r->src_port_or_type_last = rules[i].srcport_or_icmptype_last; + r->dst_port_or_code_first = rules[i].dstport_or_icmpcode_first; + r->dst_port_or_code_last = rules[i].dstport_or_icmpcode_last; + r->tcp_flags_value = rules[i].tcp_flags_value; + r->tcp_flags_mask = rules[i].tcp_flags_mask; + } + + if (~0 == *acl_list_index) + { + /* Get ACL index */ + pool_get_aligned (am->acls, a, CLIB_CACHE_LINE_BYTES); + memset (a, 0, sizeof (*a)); + /* Will return the newly allocated ACL index */ + *acl_list_index = a - am->acls; + } + else + { + a = am->acls + *acl_list_index; + /* Get rid of the old rules */ + clib_mem_free (a->rules); + } + a->rules = acl_new_rules; + a->count = count; + memcpy (a->tag, tag, sizeof (a->tag)); + + return 0; +} + +static int +acl_del_list (u32 acl_list_index) +{ + acl_main_t *am = &acl_main; + acl_list_t *a; + int i, ii; + if (pool_is_free_index (am->acls, acl_list_index)) + { + return -1; + } + + /* delete any references to the ACL */ + for (i = 0; i < vec_len (am->output_acl_vec_by_sw_if_index); i++) + { + for (ii = 0; ii < vec_len (am->output_acl_vec_by_sw_if_index[i]); + /* see body */ ) + { + if (acl_list_index == am->output_acl_vec_by_sw_if_index[i][ii]) + { + vec_del1 (am->output_acl_vec_by_sw_if_index[i], ii); + } + else + { + ii++; + } + } + } + for (i = 0; i < vec_len (am->input_acl_vec_by_sw_if_index); i++) + { + for (ii = 0; ii < vec_len (am->input_acl_vec_by_sw_if_index[i]); + /* see body */ ) + { + if (acl_list_index == am->input_acl_vec_by_sw_if_index[i][ii]) + { + vec_del1 (am->input_acl_vec_by_sw_if_index[i], ii); + } + else + { + ii++; + } + } + } + + /* now we can delete the ACL itself */ + a = &am->acls[acl_list_index]; + if (a->rules) + { + clib_mem_free (a->rules); + } + pool_put (am->acls, a); + return 0; +} + +/* Some aids in ASCII graphing the content */ +#define XX "\377" +#define __ "\000" +#define _(x) +#define v + +u8 ip4_5tuple_mask[] = +_(" dmac smac etype ") +_(ether) __ __ __ __ __ __ v __ __ __ __ __ __ v __ __ v + _(" v ihl totlen ") + _(0x0000) + __ __ __ __ + _(" ident fl+fo ") + _(0x0004) + __ __ __ __ + _(" ttl pr checksum ") + _(0x0008) + __ XX __ __ + _(" src address ") + _(0x000C) + XX XX XX XX + _(" dst address ") + _(0x0010) + XX XX XX XX + _("L4 T/U sport dport ") + _(tcpudp) + XX XX XX XX + _(padpad) + __ __ __ __ + _(padpad) + __ __ __ __ + _(padeth) + __ __; + + u8 ip6_5tuple_mask[] = + _(" dmac smac etype ") + _(ether) __ __ __ __ __ __ v __ __ __ __ __ __ v __ __ v + _(" v tc + flow ") + _(0x0000) __ __ __ __ + _(" plen nh hl ") + _(0x0004) __ __ XX __ + _(" src address ") + _(0x0008) XX XX XX XX + _(0x000C) XX XX XX XX + _(0x0010) XX XX XX XX + _(0x0014) XX XX XX XX + _(" dst address ") + _(0x0018) XX XX XX XX + _(0x001C) XX XX XX XX + _(0x0020) XX XX XX XX + _(0x0024) XX XX XX XX + _("L4T/U sport dport ") + _(tcpudp) XX XX XX XX _(padpad) __ __ __ __ _(padeth) __ __; + +#undef XX +#undef __ +#undef _ +#undef v + + static int count_skip (u8 * p, u32 size) +{ + u64 *p64 = (u64 *) p; + /* Be tolerant to null pointer */ + if (0 == p) + return 0; + + while ((0ULL == *p64) && ((u8 *) p64 - p) < size) + { + p64++; + } + return (p64 - (u64 *) p) / 2; +} + +static int +acl_classify_add_del_table_big (vnet_classify_main_t * cm, u8 * mask, + u32 mask_len, u32 next_table_index, + u32 miss_next_index, u32 * table_index, + int is_add) +{ + u32 nbuckets = 65536; + u32 memory_size = 2 << 30; + u32 skip = count_skip (mask, mask_len); + u32 match = (mask_len / 16) - skip; + u8 *skip_mask_ptr = mask + 16 * skip; + u32 current_data_flag = 0; + int current_data_offset = 0; + + if (0 == match) + match = 1; + + return vnet_classify_add_del_table (cm, skip_mask_ptr, nbuckets, + memory_size, skip, match, + next_table_index, miss_next_index, + table_index, current_data_flag, + current_data_offset, is_add, + 1 /* delete_chain */); +} + +static int +acl_classify_add_del_table_small (vnet_classify_main_t * cm, u8 * mask, + u32 mask_len, u32 next_table_index, + u32 miss_next_index, u32 * table_index, + int is_add) +{ + u32 nbuckets = 32; + u32 memory_size = 2 << 20; + u32 skip = count_skip (mask, mask_len); + u32 match = (mask_len / 16) - skip; + u8 *skip_mask_ptr = mask + 16 * skip; + u32 current_data_flag = 0; + int current_data_offset = 0; + + if (0 == match) + match = 1; + + return vnet_classify_add_del_table (cm, skip_mask_ptr, nbuckets, + memory_size, skip, match, + next_table_index, miss_next_index, + table_index, current_data_flag, + current_data_offset, is_add, + 1 /* delete_chain */); +} + + +static int +acl_unhook_l2_input_classify (acl_main_t * am, u32 sw_if_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + u32 ip4_table_index = ~0; + u32 ip6_table_index = ~0; + + vec_validate_init_empty (am->acl_ip4_input_classify_table_by_sw_if_index, + sw_if_index, ~0); + vec_validate_init_empty (am->acl_ip6_input_classify_table_by_sw_if_index, + sw_if_index, ~0); + + vnet_l2_input_classify_enable_disable (sw_if_index, 0); + + if (am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index] != ~0) + { + ip4_table_index = + am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index]; + am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index] = ~0; + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip4_table_index, 0); + } + if (am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index] != ~0) + { + ip6_table_index = + am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index]; + am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index] = ~0; + acl_classify_add_del_table_big (cm, ip6_5tuple_mask, + sizeof (ip6_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip6_table_index, 0); + } + + return 0; +} + +static int +acl_unhook_l2_output_classify (acl_main_t * am, u32 sw_if_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + u32 ip4_table_index = ~0; + u32 ip6_table_index = ~0; + + vec_validate_init_empty (am->acl_ip4_output_classify_table_by_sw_if_index, + sw_if_index, ~0); + vec_validate_init_empty (am->acl_ip6_output_classify_table_by_sw_if_index, + sw_if_index, ~0); + + vnet_l2_output_classify_enable_disable (sw_if_index, 0); + + if (am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index] != ~0) + { + ip4_table_index = + am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index]; + am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index] = ~0; + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip4_table_index, 0); + } + if (am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index] != ~0) + { + ip6_table_index = + am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index]; + am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index] = ~0; + acl_classify_add_del_table_big (cm, ip6_5tuple_mask, + sizeof (ip6_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip6_table_index, 0); + } + + return 0; +} + +static int +acl_hook_l2_input_classify (acl_main_t * am, u32 sw_if_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + u32 ip4_table_index = ~0; + u32 ip6_table_index = ~0; + int rv; + + /* in case there were previous tables attached */ + acl_unhook_l2_input_classify (am, sw_if_index); + rv = + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip4_table_index, 1); + if (rv) + return rv; + rv = + acl_classify_add_del_table_big (cm, ip6_5tuple_mask, + sizeof (ip6_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip6_table_index, 1); + if (rv) + { + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip4_table_index, 0); + return rv; + } + rv = + vnet_l2_input_classify_set_tables (sw_if_index, ip4_table_index, + ip6_table_index, ~0); + clib_warning + ("ACL enabling on interface sw_if_index %d, setting tables to the following: ip4: %d ip6: %d\n", + sw_if_index, ip4_table_index, ip6_table_index); + if (rv) + { + acl_classify_add_del_table_big (cm, ip6_5tuple_mask, + sizeof (ip6_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip6_table_index, 0); + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_input_classify_next_acl, + &ip4_table_index, 0); + return rv; + } + + am->acl_ip4_input_classify_table_by_sw_if_index[sw_if_index] = + ip4_table_index; + am->acl_ip6_input_classify_table_by_sw_if_index[sw_if_index] = + ip6_table_index; + + vnet_l2_input_classify_enable_disable (sw_if_index, 1); + return rv; +} + +static int +acl_hook_l2_output_classify (acl_main_t * am, u32 sw_if_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + u32 ip4_table_index = ~0; + u32 ip6_table_index = ~0; + int rv; + + /* in case there were previous tables attached */ + acl_unhook_l2_output_classify (am, sw_if_index); + rv = + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip4_table_index, 1); + if (rv) + return rv; + rv = + acl_classify_add_del_table_big (cm, ip6_5tuple_mask, + sizeof (ip6_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip6_table_index, 1); + if (rv) + { + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip4_table_index, 0); + return rv; + } + rv = + vnet_l2_output_classify_set_tables (sw_if_index, ip4_table_index, + ip6_table_index, ~0); + clib_warning + ("ACL enabling on interface sw_if_index %d, setting tables to the following: ip4: %d ip6: %d\n", + sw_if_index, ip4_table_index, ip6_table_index); + if (rv) + { + acl_classify_add_del_table_big (cm, ip6_5tuple_mask, + sizeof (ip6_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip6_table_index, 0); + acl_classify_add_del_table_big (cm, ip4_5tuple_mask, + sizeof (ip4_5tuple_mask) - 1, ~0, + am->l2_output_classify_next_acl, + &ip4_table_index, 0); + return rv; + } + + am->acl_ip4_output_classify_table_by_sw_if_index[sw_if_index] = + ip4_table_index; + am->acl_ip6_output_classify_table_by_sw_if_index[sw_if_index] = + ip6_table_index; + + vnet_l2_output_classify_enable_disable (sw_if_index, 1); + return rv; +} + + +int +acl_interface_in_enable_disable (acl_main_t * am, u32 sw_if_index, + int enable_disable) +{ + int rv; + + /* Utterly wrong? */ + if (pool_is_free_index (am->vnet_main->interface_main.sw_interfaces, + sw_if_index)) + return VNET_API_ERROR_INVALID_SW_IF_INDEX; + + if (enable_disable) + { + rv = acl_hook_l2_input_classify (am, sw_if_index); + } + else + { + rv = acl_unhook_l2_input_classify (am, sw_if_index); + } + + return rv; +} + +int +acl_interface_out_enable_disable (acl_main_t * am, u32 sw_if_index, + int enable_disable) +{ + int rv; + + /* Utterly wrong? */ + if (pool_is_free_index (am->vnet_main->interface_main.sw_interfaces, + sw_if_index)) + return VNET_API_ERROR_INVALID_SW_IF_INDEX; + + if (enable_disable) + { + rv = acl_hook_l2_output_classify (am, sw_if_index); + } + else + { + rv = acl_unhook_l2_output_classify (am, sw_if_index); + } + + return rv; +} + + +static int +acl_interface_add_inout_acl (u32 sw_if_index, u8 is_input, u32 acl_list_index) +{ + acl_main_t *am = &acl_main; + if (is_input) + { + vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); + vec_add (am->input_acl_vec_by_sw_if_index[sw_if_index], &acl_list_index, + 1); + acl_interface_in_enable_disable (am, sw_if_index, 1); + } + else + { + vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); + vec_add (am->output_acl_vec_by_sw_if_index[sw_if_index], + &acl_list_index, 1); + acl_interface_out_enable_disable (am, sw_if_index, 1); + } + return 0; +} + +static int +acl_interface_del_inout_acl (u32 sw_if_index, u8 is_input, u32 acl_list_index) +{ + acl_main_t *am = &acl_main; + int i; + int rv = -1; + if (is_input) + { + vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); + for (i = 0; i < vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]); + i++) + { + if (acl_list_index == + am->input_acl_vec_by_sw_if_index[sw_if_index][i]) + { + vec_del1 (am->input_acl_vec_by_sw_if_index[sw_if_index], i); + rv = 0; + break; + } + } + if (0 == vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index])) + { + acl_interface_in_enable_disable (am, sw_if_index, 0); + } + } + else + { + vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); + for (i = 0; + i < vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]); i++) + { + if (acl_list_index == + am->output_acl_vec_by_sw_if_index[sw_if_index][i]) + { + vec_del1 (am->output_acl_vec_by_sw_if_index[sw_if_index], i); + rv = 0; + break; + } + } + if (0 == vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index])) + { + acl_interface_out_enable_disable (am, sw_if_index, 0); + } + } + return rv; +} + +static void +acl_interface_reset_inout_acls (u32 sw_if_index, u8 is_input) +{ + acl_main_t *am = &acl_main; + if (is_input) + { + acl_interface_in_enable_disable (am, sw_if_index, 0); + vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); + vec_reset_length (am->input_acl_vec_by_sw_if_index[sw_if_index]); + } + else + { + acl_interface_out_enable_disable (am, sw_if_index, 0); + vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); + vec_reset_length (am->output_acl_vec_by_sw_if_index[sw_if_index]); + } +} + +static int +acl_interface_add_del_inout_acl (u32 sw_if_index, u8 is_add, u8 is_input, + u32 acl_list_index) +{ + int rv = -1; + if (is_add) + { + rv = + acl_interface_add_inout_acl (sw_if_index, is_input, acl_list_index); + } + else + { + rv = + acl_interface_del_inout_acl (sw_if_index, is_input, acl_list_index); + } + return rv; +} + + +static void * +get_ptr_to_offset (vlib_buffer_t * b0, int offset) +{ + u8 *p = vlib_buffer_get_current (b0) + offset; + return p; +} + +static u8 +acl_get_l4_proto (vlib_buffer_t * b0, int node_is_ip6) +{ + u8 proto; + int proto_offset; + if (node_is_ip6) + { + proto_offset = 20; + } + else + { + proto_offset = 23; + } + proto = *((u8 *) vlib_buffer_get_current (b0) + proto_offset); + return proto; +} + +static int +acl_match_addr (ip46_address_t * addr1, ip46_address_t * addr2, int prefixlen, + int is_ip6) +{ + if (prefixlen == 0) + { + /* match any always succeeds */ + return 1; + } + if (is_ip6) + { + if (memcmp (addr1, addr2, prefixlen / 8)) + { + /* If the starting full bytes do not match, no point in bittwidling the thumbs further */ + return 0; + } + if (prefixlen % 8) + { + u8 b1 = *((u8 *) addr1 + 1 + prefixlen / 8); + u8 b2 = *((u8 *) addr2 + 1 + prefixlen / 8); + u8 mask0 = (0xff - ((1 << (8 - (prefixlen % 8))) - 1)); + return (b1 & mask0) == b2; + } + else + { + /* The prefix fits into integer number of bytes, so nothing left to do */ + return 1; + } + } + else + { + uint32_t a1 = ntohl (addr1->ip4.as_u32); + uint32_t a2 = ntohl (addr2->ip4.as_u32); + uint32_t mask0 = 0xffffffff - ((1 << (32 - prefixlen)) - 1); + return (a1 & mask0) == a2; + } +} + +static int +acl_match_port (u16 port, u16 port_first, u16 port_last, int is_ip6) +{ + return ((port >= port_first) && (port <= port_last)); +} + +static int +acl_packet_match (acl_main_t * am, u32 acl_index, vlib_buffer_t * b0, + u8 * r_action, int *r_is_ip6, u32 * r_acl_match_p, + u32 * r_rule_match_p, u32 * trace_bitmap) +{ + ethernet_header_t *h0; + u16 type0; + + ip46_address_t src, dst; + int is_ip6; + int is_ip4; + u8 proto; + u16 src_port; + u16 dst_port; + u8 tcp_flags = 0; + int i; + acl_list_t *a; + acl_rule_t *r; + + h0 = vlib_buffer_get_current (b0); + type0 = clib_net_to_host_u16 (h0->type); + is_ip4 = (type0 == ETHERNET_TYPE_IP4); + is_ip6 = (type0 == ETHERNET_TYPE_IP6); + + if (!(is_ip4 || is_ip6)) + { + return 0; + } + /* The bunch of hardcoded offsets here is intentional to get rid of them + ASAP, when getting to a faster matching code */ + if (is_ip4) + { + clib_memcpy (&src.ip4, get_ptr_to_offset (b0, 26), 4); + clib_memcpy (&dst.ip4, get_ptr_to_offset (b0, 30), 4); + proto = acl_get_l4_proto (b0, 0); + if (1 == proto) + { + *trace_bitmap |= 0x00000001; + /* type */ + src_port = *(u8 *) get_ptr_to_offset (b0, 34); + /* code */ + dst_port = *(u8 *) get_ptr_to_offset (b0, 35); + } + else + { + /* assume TCP/UDP */ + src_port = (*(u16 *) get_ptr_to_offset (b0, 34)); + dst_port = (*(u16 *) get_ptr_to_offset (b0, 36)); + /* UDP gets ability to check on an oddball data byte as a bonus */ + tcp_flags = *(u8 *) get_ptr_to_offset (b0, 14 + 20 + 13); + } + } + else /* is_ipv6 implicitly */ + { + clib_memcpy (&src, get_ptr_to_offset (b0, 22), 16); + clib_memcpy (&dst, get_ptr_to_offset (b0, 38), 16); + proto = acl_get_l4_proto (b0, 1); + if (58 == proto) + { + *trace_bitmap |= 0x00000002; + /* type */ + src_port = *(u8 *) get_ptr_to_offset (b0, 54); + /* code */ + dst_port = *(u8 *) get_ptr_to_offset (b0, 55); + } + else + { + /* assume TCP/UDP */ + src_port = (*(u16 *) get_ptr_to_offset (b0, 54)); + dst_port = (*(u16 *) get_ptr_to_offset (b0, 56)); + tcp_flags = *(u8 *) get_ptr_to_offset (b0, 14 + 40 + 13); + } + } + if (pool_is_free_index (am->acls, acl_index)) + { + if (r_acl_match_p) + *r_acl_match_p = acl_index; + if (r_rule_match_p) + *r_rule_match_p = -1; + /* the ACL does not exist but is used for policy. Block traffic. */ + return 0; + } + a = am->acls + acl_index; + for (i = 0; i < a->count; i++) + { + r = a->rules + i; + if (is_ip6 != r->is_ipv6) + { + continue; + } + if (!acl_match_addr (&dst, &r->dst, r->dst_prefixlen, is_ip6)) + continue; + if (!acl_match_addr (&src, &r->src, r->src_prefixlen, is_ip6)) + continue; + if (r->proto) + { + if (proto != r->proto) + continue; + if (!acl_match_port + (src_port, r->src_port_or_type_first, r->src_port_or_type_last, + is_ip6)) + continue; + if (!acl_match_port + (dst_port, r->dst_port_or_code_first, r->dst_port_or_code_last, + is_ip6)) + continue; + /* No need for check of proto == TCP, since in other rules both fields should be zero, so this match will succeed */ + if ((tcp_flags & r->tcp_flags_mask) != r->tcp_flags_value) + continue; + } + /* everything matches! */ + *r_action = r->is_permit; + *r_is_ip6 = is_ip6; + if (r_acl_match_p) + *r_acl_match_p = acl_index; + if (r_rule_match_p) + *r_rule_match_p = i; + return 1; + } + return 0; +} + +void +input_acl_packet_match (u32 sw_if_index, vlib_buffer_t * b0, u32 * nextp, + u32 * acl_match_p, u32 * rule_match_p, + u32 * trace_bitmap) +{ + acl_main_t *am = &acl_main; + uint8_t action = 0; + int is_ip6 = 0; + int i; + vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); + for (i = 0; i < vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]); + i++) + { + if (acl_packet_match + (am, am->input_acl_vec_by_sw_if_index[sw_if_index][i], b0, &action, + &is_ip6, acl_match_p, rule_match_p, trace_bitmap)) + { + if (is_ip6) + { + *nextp = am->acl_in_ip6_match_next[action]; + } + else + { + *nextp = am->acl_in_ip4_match_next[action]; + } + return; + } + } + if (vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]) > 0) + { + /* If there are ACLs and none matched, deny by default */ + *nextp = 0; + } + +} + +void +output_acl_packet_match (u32 sw_if_index, vlib_buffer_t * b0, u32 * nextp, + u32 * acl_match_p, u32 * rule_match_p, + u32 * trace_bitmap) +{ + acl_main_t *am = &acl_main; + uint8_t action = 0; + int is_ip6 = 0; + int i; + vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); + for (i = 0; i < vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]); + i++) + { + if (acl_packet_match + (am, am->output_acl_vec_by_sw_if_index[sw_if_index][i], b0, &action, + &is_ip6, acl_match_p, rule_match_p, trace_bitmap)) + { + if (is_ip6) + { + *nextp = am->acl_out_ip6_match_next[action]; + } + else + { + *nextp = am->acl_out_ip4_match_next[action]; + } + return; + } + } + if (vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]) > 0) + { + /* If there are ACLs and none matched, deny by default */ + *nextp = 0; + } +} + +typedef struct +{ + u8 is_ipv6; + u8 mac_mask[6]; + u8 prefix_len; + u32 count; + u32 table_index; +} macip_match_type_t; + +static u32 +macip_find_match_type (macip_match_type_t * mv, u8 * mac_mask, u8 prefix_len, + u8 is_ipv6) +{ + u32 i; + if (mv) + { + for (i = 0; i < vec_len (mv); i++) + { + if ((mv[i].prefix_len == prefix_len) && (mv[i].is_ipv6 == is_ipv6) + && (0 == memcmp (mv[i].mac_mask, mac_mask, 6))) + { + return i; + } + } + } + return ~0; +} + + +/* Get metric used to sort match types. + The more specific and the more often seen - the bigger the metric */ +static int +match_type_metric (macip_match_type_t * m) +{ + /* FIXME: count the ones in the MAC mask as well, check how well this heuristic works in real life */ + return m->prefix_len + m->is_ipv6 + 10 * m->count; +} + +static int +match_type_compare (macip_match_type_t * m1, macip_match_type_t * m2) +{ + /* Ascending sort based on the metric values */ + return match_type_metric (m1) - match_type_metric (m2); +} + +/* Get the offset of L3 source within ethernet packet */ +static int +get_l3_src_offset(int is6) +{ + if(is6) + return (sizeof(ethernet_header_t) + offsetof(ip6_header_t, src_address)); + else + return (sizeof(ethernet_header_t) + offsetof(ip4_header_t, src_address)); +} + +static int +macip_create_classify_tables (acl_main_t * am, u32 macip_acl_index) +{ + macip_match_type_t *mvec = NULL; + macip_match_type_t *mt; + macip_acl_list_t *a = &am->macip_acls[macip_acl_index]; + int i; + u32 match_type_index; + u32 last_table; + u8 mask[5 * 16]; + vnet_classify_main_t *cm = &vnet_classify_main; + + /* Count the number of different types of rules */ + for (i = 0; i < a->count; i++) + { + if (~0 == + (match_type_index = + macip_find_match_type (mvec, a->rules[i].src_mac_mask, + a->rules[i].src_prefixlen, + a->rules[i].is_ipv6))) + { + match_type_index = vec_len (mvec); + vec_validate (mvec, match_type_index); + memcpy (mvec[match_type_index].mac_mask, + a->rules[match_type_index].src_mac_mask, 6); + mvec[match_type_index].prefix_len = a->rules[i].src_prefixlen; + mvec[match_type_index].is_ipv6 = a->rules[i].is_ipv6; + mvec[match_type_index].table_index = ~0; + } + mvec[match_type_index].count++; + } + /* Put the most frequently used tables last in the list so we can create classifier tables in reverse order */ + vec_sort_with_function (mvec, match_type_compare); + /* Create the classifier tables */ + last_table = ~0; + vec_foreach (mt, mvec) + { + int mask_len; + int is6 = mt->is_ipv6; + int l3_src_offs = get_l3_src_offset(is6); + memset (mask, 0, sizeof (mask)); + memcpy (&mask[6], mt->mac_mask, 6); + for (i = 0; i < (mt->prefix_len / 8); i++) + { + mask[l3_src_offs + i] = 0xff; + } + if (mt->prefix_len % 8) + { + mask[l3_src_offs + (mt->prefix_len / 8)] = + 0xff - ((1 << (8 - mt->prefix_len % 8)) - 1); + } + /* + * Round-up the number of bytes needed to store the prefix, + * and round up the number of vectors too + */ + mask_len = ((l3_src_offs + ((mt->prefix_len+7) / 8) + + (sizeof (u32x4)-1))/sizeof(u32x4)) * sizeof (u32x4); + acl_classify_add_del_table_small (cm, mask, mask_len, last_table, + (~0 == last_table) ? 0 : ~0, &mt->table_index, + 1); + last_table = mt->table_index; + } + a->ip4_table_index = ~0; + a->ip6_table_index = ~0; + a->l2_table_index = last_table; + + /* Populate the classifier tables with rules from the MACIP ACL */ + for (i = 0; i < a->count; i++) + { + u32 action = 0; + u32 metadata = 0; + int is6 = a->rules[i].is_ipv6; + int l3_src_offs = get_l3_src_offset(is6); + memset (mask, 0, sizeof (mask)); + memcpy (&mask[6], a->rules[i].src_mac, 6); + if (is6) + { + memcpy (&mask[l3_src_offs], &a->rules[i].src_ip_addr.ip6, 16); + } + else + { + memcpy (&mask[l3_src_offs], &a->rules[i].src_ip_addr.ip4, 4); + } + match_type_index = + macip_find_match_type (mvec, a->rules[i].src_mac_mask, + a->rules[i].src_prefixlen, + a->rules[i].is_ipv6); + /* add session to table mvec[match_type_index].table_index; */ + vnet_classify_add_del_session (cm, mvec[match_type_index].table_index, + mask, a->rules[i].is_permit ? ~0 : 0, i, + 0, action, metadata, 1); + } + return 0; +} + +static void +macip_destroy_classify_tables (acl_main_t * am, u32 macip_acl_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + macip_acl_list_t *a = &am->macip_acls[macip_acl_index]; + + if (a->ip4_table_index != ~0) + { + acl_classify_add_del_table_small (cm, 0, ~0, ~0, ~0, &a->ip4_table_index, 0); + a->ip4_table_index = ~0; + } + if (a->ip6_table_index != ~0) + { + acl_classify_add_del_table_small (cm, 0, ~0, ~0, ~0, &a->ip6_table_index, 0); + a->ip6_table_index = ~0; + } + if (a->l2_table_index != ~0) + { + acl_classify_add_del_table_small (cm, 0, ~0, ~0, ~0, &a->l2_table_index, 0); + a->l2_table_index = ~0; + } +} + +static int +macip_acl_add_list (u32 count, vl_api_macip_acl_rule_t rules[], + u32 * acl_list_index, u8 * tag) +{ + acl_main_t *am = &acl_main; + macip_acl_list_t *a; + macip_acl_rule_t *r; + macip_acl_rule_t *acl_new_rules; + int i; + + /* Create and populate the rules */ + acl_new_rules = clib_mem_alloc_aligned (sizeof (macip_acl_rule_t) * count, + CLIB_CACHE_LINE_BYTES); + if (!acl_new_rules) + { + /* Could not allocate rules. New or existing ACL - bail out regardless */ + return -1; + } + + for (i = 0; i < count; i++) + { + r = &acl_new_rules[i]; + r->is_permit = rules[i].is_permit; + r->is_ipv6 = rules[i].is_ipv6; + memcpy (&r->src_mac, rules[i].src_mac, 6); + memcpy (&r->src_mac_mask, rules[i].src_mac_mask, 6); + if(rules[i].is_ipv6) + memcpy (&r->src_ip_addr.ip6, rules[i].src_ip_addr, 16); + else + memcpy (&r->src_ip_addr.ip4, rules[i].src_ip_addr, 4); + r->src_prefixlen = rules[i].src_ip_prefix_len; + } + + /* Get ACL index */ + pool_get_aligned (am->macip_acls, a, CLIB_CACHE_LINE_BYTES); + memset (a, 0, sizeof (*a)); + /* Will return the newly allocated ACL index */ + *acl_list_index = a - am->macip_acls; + + a->rules = acl_new_rules; + a->count = count; + memcpy (a->tag, tag, sizeof (a->tag)); + + /* Create and populate the classifer tables */ + macip_create_classify_tables (am, *acl_list_index); + + return 0; +} + + +/* No check for validity of sw_if_index - the callers were supposed to validate */ + +static int +macip_acl_interface_del_acl (acl_main_t * am, u32 sw_if_index) +{ + int rv; + u32 macip_acl_index; + macip_acl_list_t *a; + vec_validate_init_empty (am->macip_acl_by_sw_if_index, sw_if_index, ~0); + macip_acl_index = am->macip_acl_by_sw_if_index[sw_if_index]; + /* No point in deleting MACIP ACL which is not applied */ + if (~0 == macip_acl_index) + return -1; + a = &am->macip_acls[macip_acl_index]; + /* remove the classifier tables off the interface L2 ACL */ + rv = + vnet_set_input_acl_intfc (am->vlib_main, sw_if_index, a->ip4_table_index, + a->ip6_table_index, a->l2_table_index, 0); + /* Unset the MACIP ACL index */ + am->macip_acl_by_sw_if_index[sw_if_index] = ~0; + return rv; +} + +/* No check for validity of sw_if_index - the callers were supposed to validate */ + +static int +macip_acl_interface_add_acl (acl_main_t * am, u32 sw_if_index, + u32 macip_acl_index) +{ + macip_acl_list_t *a; + int rv; + if (pool_is_free_index (am->macip_acls, macip_acl_index)) + { + return -1; + } + a = &am->macip_acls[macip_acl_index]; + vec_validate_init_empty (am->macip_acl_by_sw_if_index, sw_if_index, ~0); + /* If there already a MACIP ACL applied, unapply it */ + if (~0 != am->macip_acl_by_sw_if_index[sw_if_index]) + macip_acl_interface_del_acl(am, sw_if_index); + am->macip_acl_by_sw_if_index[sw_if_index] = macip_acl_index; + /* Apply the classifier tables for L2 ACLs */ + rv = + vnet_set_input_acl_intfc (am->vlib_main, sw_if_index, a->ip4_table_index, + a->ip6_table_index, a->l2_table_index, 1); + return rv; +} + +static int +macip_acl_del_list (u32 acl_list_index) +{ + acl_main_t *am = &acl_main; + macip_acl_list_t *a; + int i; + if (pool_is_free_index (am->macip_acls, acl_list_index)) + { + return -1; + } + + /* delete any references to the ACL */ + for (i = 0; i < vec_len (am->macip_acl_by_sw_if_index); i++) + { + if (am->macip_acl_by_sw_if_index[i] == acl_list_index) + { + macip_acl_interface_del_acl (am, i); + } + } + + /* Now that classifier tables are detached, clean them up */ + macip_destroy_classify_tables (am, acl_list_index); + + /* now we can delete the ACL itself */ + a = &am->macip_acls[acl_list_index]; + if (a->rules) + { + clib_mem_free (a->rules); + } + pool_put (am->macip_acls, a); + return 0; +} + + +static int +macip_acl_interface_add_del_acl (u32 sw_if_index, u8 is_add, + u32 acl_list_index) +{ + acl_main_t *am = &acl_main; + int rv = -1; + if (is_add) + { + rv = macip_acl_interface_add_acl (am, sw_if_index, acl_list_index); + } + else + { + rv = macip_acl_interface_del_acl (am, sw_if_index); + } + return rv; +} + +/* API message handler */ +static void +vl_api_acl_add_replace_t_handler (vl_api_acl_add_replace_t * mp) +{ + vl_api_acl_add_replace_reply_t *rmp; + acl_main_t *am = &acl_main; + int rv; + u32 acl_list_index = ntohl (mp->acl_index); + + rv = acl_add_list (ntohl (mp->count), mp->r, &acl_list_index, mp->tag); + + /* *INDENT-OFF* */ + REPLY_MACRO2(VL_API_ACL_ADD_REPLACE_REPLY, + ({ + rmp->acl_index = htonl(acl_list_index); + })); + /* *INDENT-ON* */ +} + +static void +vl_api_acl_del_t_handler (vl_api_acl_del_t * mp) +{ + acl_main_t *sm = &acl_main; + vl_api_acl_del_reply_t *rmp; + int rv; + + rv = acl_del_list (ntohl (mp->acl_index)); + + REPLY_MACRO (VL_API_ACL_DEL_REPLY); +} + +static void +vl_api_acl_interface_add_del_t_handler (vl_api_acl_interface_add_del_t * mp) +{ + acl_main_t *sm = &acl_main; + vnet_interface_main_t *im = &sm->vnet_main->interface_main; + u32 sw_if_index = ntohl (mp->sw_if_index); + vl_api_acl_interface_add_del_reply_t *rmp; + int rv = -1; + + if (pool_is_free_index(im->sw_interfaces, sw_if_index)) + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; + else + rv = + acl_interface_add_del_inout_acl (sw_if_index, mp->is_add, + mp->is_input, ntohl (mp->acl_index)); + + REPLY_MACRO (VL_API_ACL_INTERFACE_ADD_DEL_REPLY); +} + +static void +vl_api_acl_interface_set_acl_list_t_handler + (vl_api_acl_interface_set_acl_list_t * mp) +{ + acl_main_t *sm = &acl_main; + vl_api_acl_interface_set_acl_list_reply_t *rmp; + int rv = 0; + int i; + vnet_interface_main_t *im = &sm->vnet_main->interface_main; + u32 sw_if_index = ntohl (mp->sw_if_index); + + if (pool_is_free_index(im->sw_interfaces, sw_if_index)) + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; + else + { + acl_interface_reset_inout_acls (sw_if_index, 0); + acl_interface_reset_inout_acls (sw_if_index, 1); + + for (i = 0; i < mp->count; i++) + { + acl_interface_add_del_inout_acl (sw_if_index, 1, (i < mp->n_input), + ntohl (mp->acls[i])); + } + } + + REPLY_MACRO (VL_API_ACL_INTERFACE_SET_ACL_LIST_REPLY); +} + +static void +copy_acl_rule_to_api_rule (vl_api_acl_rule_t * api_rule, acl_rule_t * r) +{ + api_rule->is_permit = r->is_permit; + api_rule->is_ipv6 = r->is_ipv6; + if(r->is_ipv6) + { + memcpy (api_rule->src_ip_addr, &r->src, sizeof (r->src)); + memcpy (api_rule->dst_ip_addr, &r->dst, sizeof (r->dst)); + } + else + { + memcpy (api_rule->src_ip_addr, &r->src.ip4, sizeof (r->src.ip4)); + memcpy (api_rule->dst_ip_addr, &r->dst.ip4, sizeof (r->dst.ip4)); + } + api_rule->src_ip_prefix_len = r->src_prefixlen; + api_rule->dst_ip_prefix_len = r->dst_prefixlen; + api_rule->proto = r->proto; + api_rule->srcport_or_icmptype_first = r->src_port_or_type_first; + api_rule->srcport_or_icmptype_last = r->src_port_or_type_last; + api_rule->dstport_or_icmpcode_first = r->dst_port_or_code_first; + api_rule->dstport_or_icmpcode_last = r->dst_port_or_code_last; + api_rule->tcp_flags_mask = r->tcp_flags_mask; + api_rule->tcp_flags_value = r->tcp_flags_value; +} + +static void +send_acl_details (acl_main_t * am, unix_shared_memory_queue_t * q, + acl_list_t * acl, u32 context) +{ + vl_api_acl_details_t *mp; + vl_api_acl_rule_t *rules; + int i; + int msg_size = sizeof (*mp) + sizeof (mp->r[0]) * acl->count; + + mp = vl_msg_api_alloc (msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = ntohs (VL_API_ACL_DETAILS + am->msg_id_base); + + /* fill in the message */ + mp->context = context; + mp->count = htonl (acl->count); + mp->acl_index = htonl (acl - am->acls); + memcpy (mp->tag, acl->tag, sizeof (mp->tag)); + // clib_memcpy (mp->r, acl->rules, acl->count * sizeof(acl->rules[0])); + rules = mp->r; + for (i = 0; i < acl->count; i++) + { + copy_acl_rule_to_api_rule (&rules[i], &acl->rules[i]); + } + + clib_warning("Sending acl details for ACL index %d", ntohl(mp->acl_index)); + vl_msg_api_send_shmem (q, (u8 *) & mp); +} + + +static void +vl_api_acl_dump_t_handler (vl_api_acl_dump_t * mp) +{ + acl_main_t *am = &acl_main; + u32 acl_index; + acl_list_t *acl; + + int rv = -1; + unix_shared_memory_queue_t *q; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + { + return; + } + + if (mp->acl_index == ~0) + { + /* *INDENT-OFF* */ + /* Just dump all ACLs */ + pool_foreach (acl, am->acls, + ({ + send_acl_details(am, q, acl, mp->context); + })); + /* *INDENT-ON* */ + } + else + { + acl_index = ntohl (mp->acl_index); + if (!pool_is_free_index (am->acls, acl_index)) + { + acl = &am->acls[acl_index]; + send_acl_details (am, q, acl, mp->context); + } + } + + if (rv == -1) + { + /* FIXME API: should we signal an error here at all ? */ + return; + } +} + +static void +send_acl_interface_list_details (acl_main_t * am, + unix_shared_memory_queue_t * q, + u32 sw_if_index, u32 context) +{ + vl_api_acl_interface_list_details_t *mp; + int msg_size; + int n_input; + int n_output; + int count; + int i = 0; + + vec_validate (am->input_acl_vec_by_sw_if_index, sw_if_index); + vec_validate (am->output_acl_vec_by_sw_if_index, sw_if_index); + + n_input = vec_len (am->input_acl_vec_by_sw_if_index[sw_if_index]); + n_output = vec_len (am->output_acl_vec_by_sw_if_index[sw_if_index]); + count = n_input + n_output; + + msg_size = sizeof (*mp); + msg_size += sizeof (mp->acls[0]) * count; + + mp = vl_msg_api_alloc (msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = + ntohs (VL_API_ACL_INTERFACE_LIST_DETAILS + am->msg_id_base); + + /* fill in the message */ + mp->context = context; + mp->sw_if_index = htonl (sw_if_index); + mp->count = count; + mp->n_input = n_input; + for (i = 0; i < n_input; i++) + { + mp->acls[i] = htonl (am->input_acl_vec_by_sw_if_index[sw_if_index][i]); + } + for (i = 0; i < n_output; i++) + { + mp->acls[n_input + i] = + htonl (am->output_acl_vec_by_sw_if_index[sw_if_index][i]); + } + + vl_msg_api_send_shmem (q, (u8 *) & mp); +} + +static void +vl_api_acl_interface_list_dump_t_handler (vl_api_acl_interface_list_dump_t * + mp) +{ + acl_main_t *am = &acl_main; + vnet_sw_interface_t *swif; + vnet_interface_main_t *im = &am->vnet_main->interface_main; + + u32 sw_if_index; + unix_shared_memory_queue_t *q; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + { + return; + } + + if (mp->sw_if_index == ~0) + { + /* *INDENT-OFF* */ + pool_foreach (swif, im->sw_interfaces, + ({ + send_acl_interface_list_details(am, q, swif->sw_if_index, mp->context); + })); + /* *INDENT-ON* */ + } + else + { + sw_if_index = ntohl (mp->sw_if_index); + if (!pool_is_free_index(im->sw_interfaces, sw_if_index)) + send_acl_interface_list_details (am, q, sw_if_index, mp->context); + } +} + +/* MACIP ACL API handlers */ + +static void +vl_api_macip_acl_add_t_handler (vl_api_macip_acl_add_t * mp) +{ + vl_api_macip_acl_add_reply_t *rmp; + acl_main_t *am = &acl_main; + int rv; + u32 acl_list_index = ~0; + + rv = + macip_acl_add_list (ntohl (mp->count), mp->r, &acl_list_index, mp->tag); + + /* *INDENT-OFF* */ + REPLY_MACRO2(VL_API_MACIP_ACL_ADD_REPLY, + ({ + rmp->acl_index = htonl(acl_list_index); + })); + /* *INDENT-ON* */ +} + +static void +vl_api_macip_acl_del_t_handler (vl_api_macip_acl_del_t * mp) +{ + acl_main_t *sm = &acl_main; + vl_api_macip_acl_del_reply_t *rmp; + int rv; + + rv = macip_acl_del_list (ntohl (mp->acl_index)); + + REPLY_MACRO (VL_API_MACIP_ACL_DEL_REPLY); +} + +static void +vl_api_macip_acl_interface_add_del_t_handler + (vl_api_macip_acl_interface_add_del_t * mp) +{ + acl_main_t *sm = &acl_main; + vl_api_macip_acl_interface_add_del_reply_t *rmp; + int rv = -1; + vnet_interface_main_t *im = &sm->vnet_main->interface_main; + u32 sw_if_index = ntohl (mp->sw_if_index); + + if (pool_is_free_index(im->sw_interfaces, sw_if_index)) + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; + else + rv = + macip_acl_interface_add_del_acl (ntohl (mp->sw_if_index), mp->is_add, + ntohl (mp->acl_index)); + + REPLY_MACRO (VL_API_MACIP_ACL_INTERFACE_ADD_DEL_REPLY); +} + +static void +send_macip_acl_details (acl_main_t * am, unix_shared_memory_queue_t * q, + macip_acl_list_t * acl, u32 context) +{ + vl_api_macip_acl_details_t *mp; + vl_api_macip_acl_rule_t *rules; + macip_acl_rule_t *r; + int i; + int msg_size = sizeof (*mp) + (acl ? sizeof (mp->r[0]) * acl->count : 0); + + mp = vl_msg_api_alloc (msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = ntohs (VL_API_MACIP_ACL_DETAILS + am->msg_id_base); + + /* fill in the message */ + mp->context = context; + if (acl) + { + memcpy (mp->tag, acl->tag, sizeof (mp->tag)); + mp->count = htonl (acl->count); + mp->acl_index = htonl (acl - am->macip_acls); + rules = mp->r; + for (i = 0; i < acl->count; i++) + { + r = &acl->rules[i]; + rules[i].is_permit = r->is_permit; + rules[i].is_ipv6 = r->is_ipv6; + memcpy (rules[i].src_mac, &r->src_mac, sizeof (r->src_mac)); + memcpy (rules[i].src_mac_mask, &r->src_mac_mask, + sizeof (r->src_mac_mask)); + if (r->is_ipv6) + memcpy (rules[i].src_ip_addr, &r->src_ip_addr.ip6, + sizeof (r->src_ip_addr.ip6)); + else + memcpy (rules[i].src_ip_addr, &r->src_ip_addr.ip4, + sizeof (r->src_ip_addr.ip4)); + rules[i].src_ip_prefix_len = r->src_prefixlen; + } + } + else + { + /* No martini, no party - no ACL applied to this interface. */ + mp->acl_index = ~0; + mp->count = 0; + } + + vl_msg_api_send_shmem (q, (u8 *) & mp); +} + + +static void +vl_api_macip_acl_dump_t_handler (vl_api_macip_acl_dump_t * mp) +{ + acl_main_t *am = &acl_main; + macip_acl_list_t *acl; + + unix_shared_memory_queue_t *q; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + { + return; + } + + if (mp->acl_index == ~0) + { + /* Just dump all ACLs for now, with sw_if_index = ~0 */ + pool_foreach (acl, am->macip_acls, ( + { + send_macip_acl_details (am, q, acl, + mp-> + context);} + )); + /* *INDENT-ON* */ + } + else + { + u32 acl_index = ntohl (mp->acl_index); + if (!pool_is_free_index (am->macip_acls, acl_index)) + { + acl = &am->macip_acls[acl_index]; + send_macip_acl_details (am, q, acl, mp->context); + } + } +} + +static void +vl_api_macip_acl_interface_get_t_handler (vl_api_macip_acl_interface_get_t * + mp) +{ + acl_main_t *am = &acl_main; + vl_api_macip_acl_interface_get_reply_t *rmp; + u32 count = vec_len (am->macip_acl_by_sw_if_index); + int msg_size = sizeof (*rmp) + sizeof (rmp->acls[0]) * count; + unix_shared_memory_queue_t *q; + int i; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + { + return; + } + + rmp = vl_msg_api_alloc (msg_size); + memset (rmp, 0, msg_size); + rmp->_vl_msg_id = + ntohs (VL_API_MACIP_ACL_INTERFACE_GET_REPLY + am->msg_id_base); + rmp->context = mp->context; + rmp->count = htonl (count); + for (i = 0; i < count; i++) + { + rmp->acls[i] = htonl (am->macip_acl_by_sw_if_index[i]); + } + + vl_msg_api_send_shmem (q, (u8 *) & rmp); +} + + + +/* Set up the API message handling tables */ +static clib_error_t * +acl_plugin_api_hookup (vlib_main_t * vm) +{ + acl_main_t *sm = &acl_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_acl_plugin_api_msg; +#undef _ + + return 0; +} + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (acl_main_t * sm, api_main_t * am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); + foreach_vl_msg_name_crc_acl; +#undef _ +} + +u32 +register_match_action_nexts (u32 next_in_ip4, u32 next_in_ip6, + u32 next_out_ip4, u32 next_out_ip6) +{ + acl_main_t *am = &acl_main; + u32 act = am->n_match_actions; + if (am->n_match_actions == 255) + { + return ~0; + } + am->n_match_actions++; + am->acl_in_ip4_match_next[act] = next_in_ip4; + am->acl_in_ip6_match_next[act] = next_in_ip6; + am->acl_out_ip4_match_next[act] = next_out_ip4; + am->acl_out_ip6_match_next[act] = next_out_ip6; + return act; +} + +void +acl_setup_nodes (void) +{ + vlib_main_t *vm = vlib_get_main (); + acl_main_t *am = &acl_main; + vlib_node_t *n; + + n = vlib_get_node_by_name (vm, (u8 *) "l2-input-classify"); + am->l2_input_classify_next_acl = + vlib_node_add_next_with_slot (vm, n->index, acl_in_node.index, ~0); + n = vlib_get_node_by_name (vm, (u8 *) "l2-output-classify"); + am->l2_output_classify_next_acl = + vlib_node_add_next_with_slot (vm, n->index, acl_out_node.index, ~0); + + feat_bitmap_init_next_nodes (vm, acl_in_node.index, L2INPUT_N_FEAT, + l2input_get_feat_names (), + am->acl_in_node_input_next_node_index); + + memset (&am->acl_in_ip4_match_next[0], 0, + sizeof (am->acl_in_ip4_match_next)); + memset (&am->acl_in_ip6_match_next[0], 0, + sizeof (am->acl_in_ip6_match_next)); + memset (&am->acl_out_ip4_match_next[0], 0, + sizeof (am->acl_out_ip4_match_next)); + memset (&am->acl_out_ip6_match_next[0], 0, + sizeof (am->acl_out_ip6_match_next)); + am->n_match_actions = 0; + + register_match_action_nexts (0, 0, 0, 0); /* drop */ + register_match_action_nexts (~0, ~0, ~0, ~0); /* permit */ + register_match_action_nexts (ACL_IN_L2S_INPUT_IP4_ADD, ACL_IN_L2S_INPUT_IP6_ADD, ACL_OUT_L2S_OUTPUT_IP4_ADD, ACL_OUT_L2S_OUTPUT_IP6_ADD); /* permit + create session */ +} + + + +static clib_error_t * +acl_init (vlib_main_t * vm) +{ + acl_main_t *am = &acl_main; + clib_error_t *error = 0; + memset (am, 0, sizeof (*am)); + am->vlib_main = vm; + am->vnet_main = vnet_get_main (); + + u8 *name = format (0, "acl_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + am->msg_id_base = vl_msg_api_get_msg_ids ((char *) name, + VL_MSG_FIRST_AVAILABLE); + + error = acl_plugin_api_hookup (vm); + acl_setup_nodes (); + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (am, &api_main); + + vec_free (name); + + return error; +} + +VLIB_INIT_FUNCTION (acl_init); diff --git a/src/plugins/acl/acl.h b/src/plugins/acl/acl.h new file mode 100644 index 00000000..afc9b289 --- /dev/null +++ b/src/plugins/acl/acl.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_acl_h +#define included_acl_h + +#include +#include +#include +#include + + +#include +#include +#include + +#define ACL_PLUGIN_VERSION_MAJOR 1 +#define ACL_PLUGIN_VERSION_MINOR 1 + +extern vlib_node_registration_t acl_in_node; +extern vlib_node_registration_t acl_out_node; + +void input_acl_packet_match(u32 sw_if_index, vlib_buffer_t * b0, u32 *nextp, u32 *acl_match_p, u32 *rule_match_p, u32 *trace_bitmap); +void output_acl_packet_match(u32 sw_if_index, vlib_buffer_t * b0, u32 *nextp, u32 *acl_match_p, u32 *rule_match_p, u32 *trace_bitmap); + +enum address_e { IP4, IP6 }; +typedef struct +{ + enum address_e type; + union { + ip6_address_t ip6; + ip4_address_t ip4; + } addr; +} address_t; + +/* + * ACL rules + */ +typedef struct +{ + u8 is_permit; + u8 is_ipv6; + ip46_address_t src; + u8 src_prefixlen; + ip46_address_t dst; + u8 dst_prefixlen; + u8 proto; + u16 src_port_or_type_first; + u16 src_port_or_type_last; + u16 dst_port_or_code_first; + u16 dst_port_or_code_last; + u8 tcp_flags_value; + u8 tcp_flags_mask; +} acl_rule_t; + +typedef struct +{ + u8 is_permit; + u8 is_ipv6; + u8 src_mac[6]; + u8 src_mac_mask[6]; + ip46_address_t src_ip_addr; + u8 src_prefixlen; +} macip_acl_rule_t; + +/* + * ACL + */ +typedef struct +{ + u8 tag[64]; + u32 count; + acl_rule_t *rules; +} acl_list_t; + +typedef struct +{ + u8 tag[64]; + u32 count; + macip_acl_rule_t *rules; + /* References to the classifier tables that will enforce the rules */ + u32 ip4_table_index; + u32 ip6_table_index; + u32 l2_table_index; +} macip_acl_list_t; + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + + acl_list_t *acls; /* Pool of ACLs */ + macip_acl_list_t *macip_acls; /* Pool of MAC-IP ACLs */ + + /* ACLs associated with interfaces */ + u32 **input_acl_vec_by_sw_if_index; + u32 **output_acl_vec_by_sw_if_index; + + /* + * Classify tables used to grab the packets for the ACL check, + * and serving as the 5-tuple session tables at the same time + */ + u32 *acl_ip4_input_classify_table_by_sw_if_index; + u32 *acl_ip6_input_classify_table_by_sw_if_index; + u32 *acl_ip4_output_classify_table_by_sw_if_index; + u32 *acl_ip6_output_classify_table_by_sw_if_index; + + /* MACIP (input) ACLs associated with the interfaces */ + u32 *macip_acl_by_sw_if_index; + + /* next indices for our nodes in the l2-classify tables */ + u32 l2_input_classify_next_acl; + u32 l2_output_classify_next_acl; + + /* next node indices for feature bitmap */ + u32 acl_in_node_input_next_node_index[32]; + /* the respective thing for the output feature */ + l2_output_next_nodes_st acl_out_output_next_nodes; + + /* ACL match actions (must be coherent across in/out ACLs to next indices (can differ) */ + + u32 acl_in_ip4_match_next[256]; + u32 acl_in_ip6_match_next[256]; + u32 acl_out_ip4_match_next[256]; + u32 acl_out_ip6_match_next[256]; + u32 n_match_actions; + + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; + ethernet_main_t * ethernet_main; +} acl_main_t; + +extern acl_main_t acl_main; + + +#endif diff --git a/src/plugins/acl/acl_all_api_h.h b/src/plugins/acl/acl_all_api_h.h new file mode 100644 index 00000000..96eca56d --- /dev/null +++ b/src/plugins/acl/acl_all_api_h.h @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include + +#ifdef vl_printfun + +#ifdef LP64 +#define _uword_fmt "%lld" +#define _uword_cast (long long) +#else +#define _uword_fmt "%ld" +#define _uword_cast long +#endif + +static inline void * +vl_api_acl_rule_t_print (vl_api_acl_rule_t * a, void *handle) +{ + vl_print (handle, "vl_api_acl_rule_t:\n"); + vl_print (handle, "is_permit: %u\n", (unsigned) a->is_permit); + vl_print (handle, "is_ipv6: %u\n", (unsigned) a->is_ipv6); + { + int _i; + for (_i = 0; _i < 16; _i++) + { + vl_print (handle, "src_ip_addr[%d]: %u\n", _i, a->src_ip_addr[_i]); + } + } + vl_print (handle, "src_ip_prefix_len: %u\n", + (unsigned) a->src_ip_prefix_len); + { + int _i; + for (_i = 0; _i < 16; _i++) + { + vl_print (handle, "dst_ip_addr[%d]: %u\n", _i, a->dst_ip_addr[_i]); + } + } + vl_print (handle, "dst_ip_prefix_len: %u\n", + (unsigned) a->dst_ip_prefix_len); + vl_print (handle, "proto: %u\n", (unsigned) a->proto); + vl_print (handle, "srcport_or_icmptype_first: %u\n", + (unsigned) a->srcport_or_icmptype_first); + vl_print (handle, "srcport_or_icmptype_last: %u\n", + (unsigned) a->srcport_or_icmptype_last); + vl_print (handle, "dstport_or_icmpcode_first: %u\n", + (unsigned) a->dstport_or_icmpcode_first); + vl_print (handle, "dstport_or_icmpcode_last: %u\n", + (unsigned) a->dstport_or_icmpcode_last); + vl_print (handle, "tcp_flags_mask: %u\n", (unsigned) a->tcp_flags_mask); + vl_print (handle, "tcp_flags_value: %u\n", (unsigned) a->tcp_flags_value); + return handle; +} + +static inline void * +vl_api_acl_add_replace_t_print (vl_api_acl_add_replace_t * a, void *handle) +{ + int i; + vl_print (handle, "vl_api_acl_add_replace_t:\n"); + vl_print (handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); + vl_print (handle, "client_index: %u\n", (unsigned) a->client_index); + vl_print (handle, "context: %u\n", (unsigned) a->context); + vl_print (handle, "acl_index: %u\n", (unsigned) a->acl_index); + vl_print (handle, "count: %u\n", (unsigned) a->count); + vl_print (handle, "r ----- \n"); + for (i = 0; i < a->count; i++) + { + vl_print (handle, " r[%d]:\n", i); + vl_api_acl_rule_t_print (&a->r[i], handle); + } + vl_print (handle, "r ----- END \n"); + return handle; +} + + +static inline void *vl_api_acl_details_t_print (vl_api_acl_details_t *a,void *handle) +{ + vl_print(handle, "vl_api_acl_details_t:\n"); + vl_print(handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); + vl_print(handle, "context: %u\n", (unsigned) a->context); + vl_print(handle, "acl_index: %u\n", (unsigned) a->acl_index); + { + int _i; + for (_i = 0; _i < 64; _i++) { + vl_print(handle, "tag[%d]: %u\n", _i, a->tag[_i]); + } + } + vl_print(handle, "count: %u\n", (unsigned) a->count); + vl_print(handle, "r ----- \n"); + // FIXME vl_api_acl_rule_t_print(&a->r, handle); + vl_print(handle, "r ----- END \n"); + return handle; +} + +static inline void * +vl_api_macip_acl_rule_t_print (vl_api_macip_acl_rule_t * a, void *handle) +{ + vl_print (handle, "vl_api_macip_acl_rule_t:\n"); + vl_print (handle, "is_permit: %u\n", (unsigned) a->is_permit); + vl_print (handle, "is_ipv6: %u\n", (unsigned) a->is_ipv6); + { + int _i; + for (_i = 0; _i < 6; _i++) + { + vl_print (handle, "src_mac[%d]: %u\n", _i, a->src_mac[_i]); + } + } + { + int _i; + for (_i = 0; _i < 6; _i++) + { + vl_print (handle, "src_mac_mask[%d]: %u\n", _i, a->src_mac_mask[_i]); + } + } + { + int _i; + for (_i = 0; _i < 16; _i++) + { + vl_print (handle, "src_ip_addr[%d]: %u\n", _i, a->src_ip_addr[_i]); + } + } + vl_print (handle, "src_ip_prefix_len: %u\n", + (unsigned) a->src_ip_prefix_len); + return handle; +} + +static inline void * +vl_api_macip_acl_add_t_print (vl_api_macip_acl_add_t * a, void *handle) +{ + int i; + vl_print (handle, "vl_api_macip_acl_add_t:\n"); + vl_print (handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); + vl_print (handle, "client_index: %u\n", (unsigned) a->client_index); + vl_print (handle, "context: %u\n", (unsigned) a->context); + vl_print (handle, "count: %u\n", (unsigned) a->count); + vl_print (handle, "r ----- \n"); + for (i = 0; i < a->count; i++) + { + vl_print (handle, " r[%d]:\n", i); + vl_api_macip_acl_rule_t_print (&a->r[i], handle); + } + vl_print (handle, "r ----- END \n"); + return handle; +} + +static inline void *vl_api_macip_acl_details_t_print (vl_api_macip_acl_details_t *a,void *handle) +{ + int i; + vl_print(handle, "vl_api_macip_acl_details_t:\n"); + vl_print(handle, "_vl_msg_id: %u\n", (unsigned) a->_vl_msg_id); + vl_print(handle, "context: %u\n", (unsigned) a->context); + vl_print(handle, "acl_index: %u\n", (unsigned) a->acl_index); + { + int _i; + for (_i = 0; _i < 64; _i++) { + vl_print(handle, "tag[%d]: %u\n", _i, a->tag[_i]); + } + } + vl_print(handle, "count: %u\n", (unsigned) a->count); + vl_print(handle, "r ----- \n"); + for (i = 0; i < a->count; i++) + { + vl_print (handle, " r[%d]:\n", i); + vl_api_macip_acl_rule_t_print (&a->r[i], handle); + } + vl_print(handle, "r ----- END \n"); + return handle; +} + +#endif /* vl_printfun */ + + +#ifdef vl_endianfun + +#undef clib_net_to_host_uword +#ifdef LP64 +#define clib_net_to_host_uword clib_net_to_host_u64 +#else +#define clib_net_to_host_uword clib_net_to_host_u32 +#endif + +/* + * Manual endian/print functions created by copypasting the automatically + * generated ones with small required adjustments. Appears the codegen + * can't make code to print the contents of custom-type array. + */ + +static inline void +vl_api_acl_rule_t_endian (vl_api_acl_rule_t * a) +{ + /* a->is_permit = a->is_permit (no-op) */ + /* a->is_ipv6 = a->is_ipv6 (no-op) */ + /* a->src_ip_addr[0..15] = a->src_ip_addr[0..15] (no-op) */ + /* a->src_ip_prefix_len = a->src_ip_prefix_len (no-op) */ + /* a->dst_ip_addr[0..15] = a->dst_ip_addr[0..15] (no-op) */ + /* a->dst_ip_prefix_len = a->dst_ip_prefix_len (no-op) */ + /* a->proto = a->proto (no-op) */ + a->srcport_or_icmptype_first = + clib_net_to_host_u16 (a->srcport_or_icmptype_first); + a->srcport_or_icmptype_last = + clib_net_to_host_u16 (a->srcport_or_icmptype_last); + a->dstport_or_icmpcode_first = + clib_net_to_host_u16 (a->dstport_or_icmpcode_first); + a->dstport_or_icmpcode_last = + clib_net_to_host_u16 (a->dstport_or_icmpcode_last); + /* a->tcp_flags_mask = a->tcp_flags_mask (no-op) */ + /* a->tcp_flags_value = a->tcp_flags_value (no-op) */ +} + +static inline void +vl_api_acl_add_replace_t_endian (vl_api_acl_add_replace_t * a) +{ + int i; + a->_vl_msg_id = clib_net_to_host_u16 (a->_vl_msg_id); + a->client_index = clib_net_to_host_u32 (a->client_index); + a->context = clib_net_to_host_u32 (a->context); + a->acl_index = clib_net_to_host_u32 (a->acl_index); + a->count = clib_net_to_host_u32 (a->count); + for (i = 0; i < a->count; i++) + { + vl_api_acl_rule_t_endian (&a->r[i]); + } +} + +static inline void vl_api_acl_details_t_endian (vl_api_acl_details_t *a) +{ + int i; + a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); + a->context = clib_net_to_host_u32(a->context); + a->acl_index = clib_net_to_host_u32(a->acl_index); + /* a->tag[0..63] = a->tag[0..63] (no-op) */ + a->count = clib_net_to_host_u32(a->count); + for (i = 0; i < a->count; i++) + { + vl_api_acl_rule_t_endian (&a->r[i]); + } +} + +static inline void vl_api_acl_interface_list_details_t_endian (vl_api_acl_interface_list_details_t *a) +{ + int i; + a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); + a->context = clib_net_to_host_u32(a->context); + a->sw_if_index = clib_net_to_host_u32(a->sw_if_index); + /* a->count = a->count (no-op) */ + /* a->n_input = a->n_input (no-op) */ + for(i=0; icount; i++) { + a->acls[i] = clib_net_to_host_u32(a->acls[i]); + } +} + +static inline void vl_api_acl_interface_set_acl_list_t_endian (vl_api_acl_interface_set_acl_list_t *a) +{ + int i; + a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); + a->client_index = clib_net_to_host_u32(a->client_index); + a->context = clib_net_to_host_u32(a->context); + a->sw_if_index = clib_net_to_host_u32(a->sw_if_index); + /* a->count = a->count (no-op) */ + /* a->n_input = a->n_input (no-op) */ + for(i=0; icount; i++) { + a->acls[i] = clib_net_to_host_u32(a->acls[i]); + } +} + +static inline void +vl_api_macip_acl_rule_t_endian (vl_api_macip_acl_rule_t * a) +{ + /* a->is_permit = a->is_permit (no-op) */ + /* a->is_ipv6 = a->is_ipv6 (no-op) */ + /* a->src_mac[0..5] = a->src_mac[0..5] (no-op) */ + /* a->src_mac_mask[0..5] = a->src_mac_mask[0..5] (no-op) */ + /* a->src_ip_addr[0..15] = a->src_ip_addr[0..15] (no-op) */ + /* a->src_ip_prefix_len = a->src_ip_prefix_len (no-op) */ +} + +static inline void +vl_api_macip_acl_add_t_endian (vl_api_macip_acl_add_t * a) +{ + int i; + a->_vl_msg_id = clib_net_to_host_u16 (a->_vl_msg_id); + a->client_index = clib_net_to_host_u32 (a->client_index); + a->context = clib_net_to_host_u32 (a->context); + a->count = clib_net_to_host_u32 (a->count); + for (i = 0; i < a->count; i++) + { + vl_api_macip_acl_rule_t_endian (&a->r[i]); + } +} + +static inline void vl_api_macip_acl_details_t_endian (vl_api_macip_acl_details_t *a) +{ + int i; + a->_vl_msg_id = clib_net_to_host_u16(a->_vl_msg_id); + a->context = clib_net_to_host_u32(a->context); + a->acl_index = clib_net_to_host_u32(a->acl_index); + /* a->tag[0..63] = a->tag[0..63] (no-op) */ + a->count = clib_net_to_host_u32(a->count); + for (i = 0; i < a->count; i++) + { + vl_api_macip_acl_rule_t_endian (&a->r[i]); + } +} + + + + +#endif /* vl_printfun */ + + diff --git a/src/plugins/acl/acl_msg_enum.h b/src/plugins/acl/acl_msg_enum.h new file mode 100644 index 00000000..14d8b48c --- /dev/null +++ b/src/plugins/acl/acl_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_acl_msg_enum_h +#define included_acl_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif diff --git a/src/plugins/acl/acl_test.c b/src/plugins/acl/acl_test.c new file mode 100644 index 00000000..a0e413e1 --- /dev/null +++ b/src/plugins/acl/acl_test.c @@ -0,0 +1,1024 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * acl_test.c - test harness plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include +#include +#include + +uword unformat_sw_if_index (unformat_input_t * input, va_list * args); + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define message structures */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} acl_test_main_t; + +acl_test_main_t acl_test_main; + +#define foreach_standard_reply_retval_handler \ +_(acl_del_reply) \ +_(acl_interface_add_del_reply) \ +_(macip_acl_interface_add_del_reply) \ +_(acl_interface_set_acl_list_reply) \ +_(macip_acl_del_reply) + +#define foreach_reply_retval_aclindex_handler \ +_(acl_add_replace_reply) \ +_(macip_acl_add_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = acl_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = acl_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + clib_warning("ACL index: %d", ntohl(mp->acl_index)); \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_reply_retval_aclindex_handler; +#undef _ + +/* These two ought to be in a library somewhere but they aren't */ +static uword +my_unformat_mac_address (unformat_input_t * input, va_list * args) +{ + u8 *a = va_arg (*args, u8 *); + return unformat (input, "%x:%x:%x:%x:%x:%x", &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5]); +} + +static u8 * +my_format_mac_address (u8 * s, va_list * args) +{ + u8 *a = va_arg (*args, u8 *); + return format (s, "%02x:%02x:%02x:%02x:%02x:%02x", + a[0], a[1], a[2], a[3], a[4], a[5]); +} + + + +static void vl_api_acl_plugin_get_version_reply_t_handler + (vl_api_acl_plugin_get_version_reply_t * mp) + { + vat_main_t * vam = acl_test_main.vat_main; + clib_warning("ACL plugin version: %d.%d", ntohl(mp->major), ntohl(mp->minor)); + vam->result_ready = 1; + } + +static void vl_api_acl_interface_list_details_t_handler + (vl_api_acl_interface_list_details_t * mp) + { + int i; + vat_main_t * vam = acl_test_main.vat_main; + u8 *out = 0; + vl_api_acl_interface_list_details_t_endian(mp); + out = format(out, "sw_if_index: %d, count: %d, n_input: %d\n", mp->sw_if_index, mp->count, mp->n_input); + out = format(out, " input "); + for(i=0; icount; i++) { + out = format(out, "%d ", mp->acls[i]); + if (i == mp->n_input-1) + out = format(out, "\n output "); + } + out = format(out, "\n"); + clib_warning("%s", out); + vec_free(out); + vam->result_ready = 1; + } + + +static inline u8 * +vl_api_acl_rule_t_pretty_format (u8 *out, vl_api_acl_rule_t * a) +{ + int af = a->is_ipv6 ? AF_INET6 : AF_INET; + u8 src[INET6_ADDRSTRLEN]; + u8 dst[INET6_ADDRSTRLEN]; + inet_ntop(af, a->src_ip_addr, (void *)src, sizeof(src)); + inet_ntop(af, a->dst_ip_addr, (void *)dst, sizeof(dst)); + + out = format(out, "%s action %d src %s/%d dst %s/%d proto %d sport %d-%d dport %d-%d tcpflags %d %d", + a->is_ipv6 ? "ipv6" : "ipv4", a->is_permit, + src, a->src_ip_prefix_len, + dst, a->dst_ip_prefix_len, + a->proto, + a->srcport_or_icmptype_first, a->srcport_or_icmptype_last, + a->dstport_or_icmpcode_first, a->dstport_or_icmpcode_last, + a->tcp_flags_mask, a->tcp_flags_value); + return(out); +} + + + +static void vl_api_acl_details_t_handler + (vl_api_acl_details_t * mp) + { + int i; + vat_main_t * vam = acl_test_main.vat_main; + vl_api_acl_details_t_endian(mp); + u8 *out = 0; + out = format(0, "acl_index: %d, count: %d\n tag {%s}\n", mp->acl_index, mp->count, mp->tag); + for(i=0; icount; i++) { + out = format(out, " "); + out = vl_api_acl_rule_t_pretty_format(out, &mp->r[i]); + out = format(out, "%s\n", icount-1 ? "," : ""); + } + clib_warning("%s", out); + vec_free(out); + vam->result_ready = 1; + } + +static inline u8 * +vl_api_macip_acl_rule_t_pretty_format (u8 *out, vl_api_macip_acl_rule_t * a) +{ + int af = a->is_ipv6 ? AF_INET6 : AF_INET; + u8 src[INET6_ADDRSTRLEN]; + inet_ntop(af, a->src_ip_addr, (void *)src, sizeof(src)); + + out = format(out, "%s action %d ip %s/%d mac %U mask %U", + a->is_ipv6 ? "ipv6" : "ipv4", a->is_permit, + src, a->src_ip_prefix_len, + my_format_mac_address, a->src_mac, + my_format_mac_address, a->src_mac_mask); + return(out); +} + + +static void vl_api_macip_acl_details_t_handler + (vl_api_macip_acl_details_t * mp) + { + int i; + vat_main_t * vam = acl_test_main.vat_main; + vl_api_macip_acl_details_t_endian(mp); + u8 *out = format(0,"MACIP acl_index: %d, count: %d\n tag {%s}\n", mp->acl_index, mp->count, mp->tag); + for(i=0; icount; i++) { + out = format(out, " "); + out = vl_api_macip_acl_rule_t_pretty_format(out, &mp->r[i]); + out = format(out, "%s\n", icount-1 ? "," : ""); + } + clib_warning("%s", out); + vec_free(out); + vam->result_ready = 1; + } + +static void vl_api_macip_acl_interface_get_reply_t_handler + (vl_api_macip_acl_interface_get_reply_t * mp) + { + int i; + vat_main_t * vam = acl_test_main.vat_main; + u8 *out = format(0, "sw_if_index with MACIP ACL count: %d\n", ntohl(mp->count)); + for(i=0; icount); i++) { + out = format(out, " macip_acl_interface_add_del sw_if_index %d add acl %d\n", i, ntohl(mp->acls[i])); + } + out = format(out, "\n"); + clib_warning("%s", out); + vec_free(out); + vam->result_ready = 1; + } + + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(ACL_ADD_REPLACE_REPLY, acl_add_replace_reply) \ +_(ACL_DEL_REPLY, acl_del_reply) \ +_(ACL_INTERFACE_ADD_DEL_REPLY, acl_interface_add_del_reply) \ +_(ACL_INTERFACE_SET_ACL_LIST_REPLY, acl_interface_set_acl_list_reply) \ +_(ACL_INTERFACE_LIST_DETAILS, acl_interface_list_details) \ +_(ACL_DETAILS, acl_details) \ +_(MACIP_ACL_ADD_REPLY, macip_acl_add_reply) \ +_(MACIP_ACL_DEL_REPLY, macip_acl_del_reply) \ +_(MACIP_ACL_DETAILS, macip_acl_details) \ +_(MACIP_ACL_INTERFACE_ADD_DEL_REPLY, macip_acl_interface_add_del_reply) \ +_(MACIP_ACL_INTERFACE_GET_REPLY, macip_acl_interface_get_reply) \ +_(ACL_PLUGIN_GET_VERSION_REPLY, acl_plugin_get_version_reply) + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int api_acl_plugin_get_version (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + vl_api_acl_plugin_get_version_t * mp; + u32 msg_size = sizeof(*mp); + f64 timeout; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client(msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = ntohs (VL_API_ACL_PLUGIN_GET_VERSION + sm->msg_id_base); + mp->client_index = vam->my_client_index; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; + + return 0; +} + +static int api_macip_acl_interface_get (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + vl_api_acl_plugin_get_version_t * mp; + u32 msg_size = sizeof(*mp); + f64 timeout; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client(msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = ntohs (VL_API_MACIP_ACL_INTERFACE_GET + sm->msg_id_base); + mp->client_index = vam->my_client_index; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; + + return 0; +} + +#define vec_validate_acl_rules(v, idx) \ + do { \ + if (vec_len(v) < idx+1) { \ + vec_validate(v, idx); \ + v[idx].is_permit = 0x1; \ + v[idx].srcport_or_icmptype_last = 0xffff; \ + v[idx].dstport_or_icmpcode_last = 0xffff; \ + } \ + } while (0) + + +static int api_acl_add_replace (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_acl_add_replace_t * mp; + u32 acl_index = ~0; + u32 msg_size = sizeof (*mp); /* without the rules */ + + vl_api_acl_rule_t *rules = 0; + int rule_idx = 0; + int n_rules = 0; + u32 proto = 0; + u32 port1 = 0; + u32 port2 = 0; + u32 action = 0; + u32 tcpflags, tcpmask; + u32 src_prefix_length = 0, dst_prefix_length = 0; + ip4_address_t src_v4address, dst_v4address; + ip6_address_t src_v6address, dst_v6address; + u8 *tag = 0; + + if (!unformat (i, "%d", &acl_index)) { + /* Just assume -1 */ + } + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "ipv6")) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].is_ipv6 = 1; + } + else if (unformat (i, "ipv4")) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].is_ipv6 = 0; + } + else if (unformat (i, "permit+reflect")) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].is_permit = 2; + } + else if (unformat (i, "permit")) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].is_permit = 1; + } + else if (unformat (i, "action %d", &action)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].is_permit = action; + } + else if (unformat (i, "src %U/%d", + unformat_ip4_address, &src_v4address, &src_prefix_length)) + { + vec_validate_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].src_ip_addr, &src_v4address, 4); + rules[rule_idx].src_ip_prefix_len = src_prefix_length; + rules[rule_idx].is_ipv6 = 0; + } + else if (unformat (i, "src %U/%d", + unformat_ip6_address, &src_v6address, &src_prefix_length)) + { + vec_validate_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].src_ip_addr, &src_v6address, 16); + rules[rule_idx].src_ip_prefix_len = src_prefix_length; + rules[rule_idx].is_ipv6 = 1; + } + else if (unformat (i, "dst %U/%d", + unformat_ip4_address, &dst_v4address, &dst_prefix_length)) + { + vec_validate_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].dst_ip_addr, &dst_v4address, 4); + rules[rule_idx].dst_ip_prefix_len = dst_prefix_length; + rules[rule_idx].is_ipv6 = 0; + } + else if (unformat (i, "dst %U/%d", + unformat_ip6_address, &dst_v6address, &dst_prefix_length)) + { + vec_validate_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].dst_ip_addr, &dst_v6address, 16); + rules[rule_idx].dst_ip_prefix_len = dst_prefix_length; + rules[rule_idx].is_ipv6 = 1; + } + else if (unformat (i, "sport %d-%d", &port1, &port2)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].srcport_or_icmptype_first = htons(port1); + rules[rule_idx].srcport_or_icmptype_last = htons(port2); + } + else if (unformat (i, "sport %d", &port1)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].srcport_or_icmptype_first = htons(port1); + rules[rule_idx].srcport_or_icmptype_last = htons(port1); + } + else if (unformat (i, "dport %d-%d", &port1, &port2)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].dstport_or_icmpcode_first = htons(port1); + rules[rule_idx].dstport_or_icmpcode_last = htons(port2); + } + else if (unformat (i, "dport %d", &port1)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].dstport_or_icmpcode_first = htons(port1); + rules[rule_idx].dstport_or_icmpcode_last = htons(port1); + } + else if (unformat (i, "tcpflags %d %d", &tcpflags, &tcpmask)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].tcp_flags_value = tcpflags; + rules[rule_idx].tcp_flags_mask = tcpmask; + } + else if (unformat (i, "proto %d", &proto)) + { + vec_validate_acl_rules(rules, rule_idx); + rules[rule_idx].proto = proto; + } + else if (unformat (i, "tag %s", &tag)) + { + } + else if (unformat (i, ",")) + { + rule_idx++; + vec_validate_acl_rules(rules, rule_idx); + } + else + break; + } + + /* Construct the API message */ + vam->result_ready = 0; + + if(rules) + n_rules = vec_len(rules); + else + n_rules = 0; + + msg_size += n_rules*sizeof(rules[0]); + + mp = vl_msg_api_alloc_as_if_client(msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = ntohs (VL_API_ACL_ADD_REPLACE + sm->msg_id_base); + mp->client_index = vam->my_client_index; + if (n_rules > 0) + clib_memcpy(mp->r, rules, n_rules*sizeof (vl_api_acl_rule_t)); + if (tag) + { + if (vec_len(tag) >= sizeof(mp->tag)) + { + tag[sizeof(mp->tag)-1] = 0; + _vec_len(tag) = sizeof(mp->tag); + } + clib_memcpy(mp->tag, tag, vec_len(tag)); + vec_free(tag); + } + mp->acl_index = ntohl(acl_index); + mp->count = htonl(n_rules); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_acl_del (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_acl_del_t * mp; + u32 acl_index = ~0; + + if (!unformat (i, "%d", &acl_index)) { + errmsg ("missing acl index\n"); + return -99; + } + + /* Construct the API message */ + M(ACL_DEL, acl_del); + mp->acl_index = ntohl(acl_index); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_macip_acl_del (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_acl_del_t * mp; + u32 acl_index = ~0; + + if (!unformat (i, "%d", &acl_index)) { + errmsg ("missing acl index\n"); + return -99; + } + + /* Construct the API message */ + M(MACIP_ACL_DEL, acl_del); + mp->acl_index = ntohl(acl_index); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_acl_interface_add_del (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_acl_interface_add_del_t * mp; + u32 sw_if_index = ~0; + u32 acl_index = ~0; + u8 is_input = 0; + u8 is_add = 0; + +// acl_interface_add_del | sw_if_index acl_index [out] [del] + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "%d", &acl_index)) + ; + else + break; + } + + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + ; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + ; + else if (unformat (i, "add")) + is_add = 1; + else if (unformat (i, "del")) + is_add = 0; + else if (unformat (i, "acl %d", &acl_index)) + ; + else if (unformat (i, "input")) + is_input = 1; + else if (unformat (i, "output")) + is_input = 0; + else + break; + } + + if (sw_if_index == ~0) { + errmsg ("missing interface name / explicit sw_if_index number \n"); + return -99; + } + + if (acl_index == ~0) { + errmsg ("missing ACL index\n"); + return -99; + } + + + + /* Construct the API message */ + M(ACL_INTERFACE_ADD_DEL, acl_interface_add_del); + mp->acl_index = ntohl(acl_index); + mp->sw_if_index = ntohl(sw_if_index); + mp->is_add = is_add; + mp->is_input = is_input; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_macip_acl_interface_add_del (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_macip_acl_interface_add_del_t * mp; + u32 sw_if_index = ~0; + u32 acl_index = ~0; + u8 is_add = 0; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + ; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + ; + else if (unformat (i, "add")) + is_add = 1; + else if (unformat (i, "del")) + is_add = 0; + else if (unformat (i, "acl %d", &acl_index)) + ; + else + break; + } + + if (sw_if_index == ~0) { + errmsg ("missing interface name / explicit sw_if_index number \n"); + return -99; + } + + if (acl_index == ~0) { + errmsg ("missing ACL index\n"); + return -99; + } + + + + /* Construct the API message */ + M(MACIP_ACL_INTERFACE_ADD_DEL, macip_acl_interface_add_del); + mp->acl_index = ntohl(acl_index); + mp->sw_if_index = ntohl(sw_if_index); + mp->is_add = is_add; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_acl_interface_set_acl_list (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_acl_interface_set_acl_list_t * mp; + u32 sw_if_index = ~0; + u32 acl_index = ~0; + u32 *inacls = 0; + u32 *outacls = 0; + u8 is_input = 0; + +// acl_interface_set_acl_list | sw_if_index input [acl-idx list] output [acl-idx list] + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + ; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + ; + else if (unformat (i, "%d", &acl_index)) + { + if(is_input) + vec_add1(inacls, htonl(acl_index)); + else + vec_add1(outacls, htonl(acl_index)); + } + else if (unformat (i, "acl %d", &acl_index)) + ; + else if (unformat (i, "input")) + is_input = 1; + else if (unformat (i, "output")) + is_input = 0; + else + break; + } + + if (sw_if_index == ~0) { + errmsg ("missing interface name / explicit sw_if_index number \n"); + return -99; + } + + /* Construct the API message */ + M2(ACL_INTERFACE_SET_ACL_LIST, acl_interface_set_acl_list, sizeof(u32) * (vec_len(inacls) + vec_len(outacls))); + mp->sw_if_index = ntohl(sw_if_index); + mp->n_input = vec_len(inacls); + mp->count = vec_len(inacls) + vec_len(outacls); + vec_append(inacls, outacls); + if (vec_len(inacls) > 0) + clib_memcpy(mp->acls, inacls, vec_len(inacls)*sizeof(u32)); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + + +static int api_acl_interface_list_dump (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + u32 sw_if_index = ~0; + vl_api_acl_interface_list_dump_t * mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + ; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + ; + else + break; + } + + /* Construct the API message */ + M(ACL_INTERFACE_LIST_DUMP, acl_interface_list_dump); + mp->sw_if_index = ntohl (sw_if_index); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_acl_dump (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + u32 acl_index = ~0; + vl_api_acl_dump_t * mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%d", &acl_index)) + ; + else + break; + } + + /* Construct the API message */ + M(ACL_DUMP, acl_dump); + mp->acl_index = ntohl (acl_index); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +static int api_macip_acl_dump (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + u32 acl_index = ~0; + vl_api_acl_dump_t * mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) { + if (unformat (i, "%d", &acl_index)) + ; + else + break; + } + + /* Construct the API message */ + M(MACIP_ACL_DUMP, macip_acl_dump); + mp->acl_index = ntohl (acl_index); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +#define vec_validate_macip_acl_rules(v, idx) \ + do { \ + if (vec_len(v) < idx+1) { \ + vec_validate(v, idx); \ + v[idx].is_permit = 0x1; \ + } \ + } while (0) + + +static int api_macip_acl_add (vat_main_t * vam) +{ + acl_test_main_t * sm = &acl_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_macip_acl_add_t * mp; + u32 msg_size = sizeof (*mp); /* without the rules */ + + vl_api_macip_acl_rule_t *rules = 0; + int rule_idx = 0; + int n_rules = 0; + u32 src_prefix_length = 0; + u32 action = 0; + ip4_address_t src_v4address; + ip6_address_t src_v6address; + u8 src_mac[6]; + u8 *tag = 0; + u8 mac_mask_all_1[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "ipv6")) + { + vec_validate_macip_acl_rules(rules, rule_idx); + rules[rule_idx].is_ipv6 = 1; + } + else if (unformat (i, "ipv4")) + { + vec_validate_macip_acl_rules(rules, rule_idx); + rules[rule_idx].is_ipv6 = 1; + } + else if (unformat (i, "permit")) + { + vec_validate_macip_acl_rules(rules, rule_idx); + rules[rule_idx].is_permit = 1; + } + else if (unformat (i, "deny")) + { + vec_validate_macip_acl_rules(rules, rule_idx); + rules[rule_idx].is_permit = 0; + } + else if (unformat (i, "action %d", &action)) + { + vec_validate_macip_acl_rules(rules, rule_idx); + rules[rule_idx].is_permit = action; + } + else if (unformat (i, "ip %U/%d", + unformat_ip4_address, &src_v4address, &src_prefix_length)) + { + vec_validate_macip_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].src_ip_addr, &src_v4address, 4); + rules[rule_idx].src_ip_prefix_len = src_prefix_length; + rules[rule_idx].is_ipv6 = 0; + } + else if (unformat (i, "ip %U/%d", + unformat_ip6_address, &src_v6address, &src_prefix_length)) + { + vec_validate_macip_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].src_ip_addr, &src_v6address, 16); + rules[rule_idx].src_ip_prefix_len = src_prefix_length; + rules[rule_idx].is_ipv6 = 1; + } + else if (unformat (i, "mac %U", + my_unformat_mac_address, &src_mac)) + { + vec_validate_macip_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].src_mac, &src_mac, 6); + memcpy (rules[rule_idx].src_mac_mask, &mac_mask_all_1, 6); + } + else if (unformat (i, "mask %U", + my_unformat_mac_address, &src_mac)) + { + vec_validate_macip_acl_rules(rules, rule_idx); + memcpy (rules[rule_idx].src_mac_mask, &src_mac, 6); + } + else if (unformat (i, "tag %s", &tag)) + { + } + else if (unformat (i, ",")) + { + rule_idx++; + vec_validate_macip_acl_rules(rules, rule_idx); + } + else + break; + } + + /* Construct the API message */ + vam->result_ready = 0; + + if(rules) + n_rules = vec_len(rules); + else + n_rules = 0; + + msg_size += n_rules*sizeof(rules[0]); + + mp = vl_msg_api_alloc_as_if_client(msg_size); + memset (mp, 0, msg_size); + mp->_vl_msg_id = ntohs (VL_API_MACIP_ACL_ADD + sm->msg_id_base); + mp->client_index = vam->my_client_index; + if (n_rules > 0) + clib_memcpy(mp->r, rules, n_rules*sizeof (mp->r[0])); + if (tag) + { + if (vec_len(tag) >= sizeof(mp->tag)) + { + tag[sizeof(mp->tag)-1] = 0; + _vec_len(tag) = sizeof(mp->tag); + } + clib_memcpy(mp->tag, tag, vec_len(tag)); + vec_free(tag); + } + + mp->count = htonl(n_rules); + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(acl_plugin_get_version, "") \ +_(acl_add_replace, " [ [src IP/plen] [dst IP/plen] [sport X-Y] [dport X-Y] [proto P] [tcpflags FL MASK], ... , ...") \ +_(acl_del, "") \ +_(acl_dump, "[]") \ +_(acl_interface_add_del, " | sw_if_index [add|del] [input|output] acl ") \ +_(acl_interface_set_acl_list, " | sw_if_index input [acl-idx list] output [acl-idx list]") \ +_(acl_interface_list_dump, "[ | sw_if_index ]") \ +_(macip_acl_add, "...") \ +_(macip_acl_del, "")\ +_(macip_acl_dump, "[]") \ +_(macip_acl_interface_add_del, " | sw_if_index [add|del] acl ") \ +_(macip_acl_interface_get, "") + + + +void vat_api_hookup (vat_main_t *vam) +{ + acl_test_main_t * sm = &acl_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * vat_plugin_register (vat_main_t *vam) +{ + acl_test_main_t * sm = &acl_test_main; + u8 * name; + + sm->vat_main = vam; + + name = format (0, "acl_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~0) + vat_api_hookup (vam); + + vec_free(name); + + return 0; +} diff --git a/src/plugins/acl/l2sess.c b/src/plugins/acl/l2sess.c new file mode 100644 index 00000000..cc9bde44 --- /dev/null +++ b/src/plugins/acl/l2sess.c @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * l2sess.c - simple MAC-swap API / debug CLI handling + *------------------------------------------------------------------ + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +void +l2sess_vlib_plugin_register (vlib_main_t * vm, void* hh, + int from_early_init) +{ + l2sess_main_t *sm = &l2sess_main; + vnet_plugin_handoff_t * h = hh; + memset (sm, 0, sizeof (*sm)); + + sm->vlib_main = vm; + sm->vnet_main = h->vnet_main; + sm->ethernet_main = h->ethernet_main; +} + +void +l2sess_init_next_features_input (vlib_main_t * vm, l2sess_main_t * sm) +{ +#define _(node_name, node_var, is_out, is_ip6, is_track) \ + if (!is_out) feat_bitmap_init_next_nodes(vm, node_var.index, L2INPUT_N_FEAT, l2input_get_feat_names (), sm->node_var ## _input_next_node_index); + foreach_l2sess_node +#undef _ +} + +void +l2sess_add_our_next_nodes (vlib_main_t * vm, l2sess_main_t * sm, + u8 * prev_node_name, int add_output_nodes) +{ + vlib_node_t *n; + n = vlib_get_node_by_name (vm, prev_node_name); +#define _(node_name, node_var, is_out, is_ip6, is_track) \ + if (is_out == add_output_nodes) { \ + u32 idx = vlib_node_add_next_with_slot(vm, n->index, node_var.index, ~0); \ + if (is_track) { \ + sm->next_slot_track_node_by_is_ip6_is_out[is_ip6][is_out] = idx; \ + } \ + } + foreach_l2sess_node +#undef _ +} + +void +l2sess_setup_nodes (void) +{ + vlib_main_t *vm = vlib_get_main (); + l2sess_main_t *sm = &l2sess_main; + + l2sess_init_next_features_input (vm, sm); + + l2sess_add_our_next_nodes (vm, sm, (u8 *) "l2-input-classify", 0); + l2sess_add_our_next_nodes (vm, sm, (u8 *) "l2-output-classify", 1); + +} + +static char * +get_l4_proto_str (int is_ip6, uint8_t l4_proto) +{ + switch (l4_proto) + { + case 6: + return "tcp"; + case 17: + return "udp"; + case 1: + return "icmp"; + case 58: + return "icmp6"; + default: + return ""; + } +} + +static clib_error_t * +l2sess_show_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + l2sess_main_t *sm = &l2sess_main; + clib_time_t *ct = &vm->clib_time; + l2s_session_t *s; + u64 now = clib_cpu_time_now (); + + vlib_cli_output (vm, "Timing wheel info: \n%U", format_timing_wheel, + &sm->timing_wheel, 255); + + pool_foreach (s, sm->sessions, ( + { + f64 ctime = + (now - + s->create_time) * ct->seconds_per_clock; + f64 atime0 = + (now - + s->side[0].active_time) * + ct->seconds_per_clock; + f64 atime1 = + (now - + s->side[1].active_time) * + ct->seconds_per_clock; +/* + f64 ctime = (s->create_time - vm->cpu_time_main_loop_start) * ct->seconds_per_clock; + f64 atime0 = (s->side[0].active_time - vm->cpu_time_main_loop_start) * ct->seconds_per_clock; + f64 atime1 = (s->side[1].active_time - vm->cpu_time_main_loop_start) * ct->seconds_per_clock; +*/ + u8 * out0 = + format (0, + "%5d: create time: %U pkts/bytes/active time: [ %ld %ld %U : %ld %ld %U ]\n", + (s - sm->sessions), + format_time_interval, "h:m:s:u", + ctime, s->side[0].n_packets, + s->side[0].n_bytes, + format_time_interval, "h:m:s:u", + atime0, s->side[1].n_packets, + s->side[1].n_bytes, + format_time_interval, "h:m:s:u", + atime1); u8 * out1 = 0; + if (s->is_ip6) + { + out1 = + format (0, "%s %U :%u <-> %U :%u", + get_l4_proto_str (s->is_ip6, + s->l4_proto), + format_ip6_address, + &s->side[0].addr.ip6, + s->side[0].port, + format_ip6_address, + &s->side[1].addr.ip6, + s->side[1].port);} + else + { + out1 = + format (0, "%s %U :%u <-> %U :%u", + get_l4_proto_str (s->is_ip6, + s->l4_proto), + format_ip4_address, + &s->side[0].addr.ip4, + s->side[0].port, + format_ip4_address, + &s->side[1].addr.ip4, + s->side[1].port);} + vlib_cli_output (vm, "%s %s", out0, + out1); vec_free (out0); + vec_free (out1);} + )); + return 0; +} + +static clib_error_t * +l2sess_show_count_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + l2sess_main_t *sm = &l2sess_main; + + vlib_cli_output (vm, "Timing wheel info: \n%U", format_timing_wheel, + &sm->timing_wheel, 255); + vlib_cli_output (vm, "session pool len: %d, pool elts: %d", + pool_len (sm->sessions), pool_elts (sm->sessions)); + vlib_cli_output (vm, + "attempted to delete sessions which were already free: %d", + sm->counter_attempted_delete_free_session); + return 0; +} + + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (l2sess_show_command, static) = { + .path = "show l2sess", + .short_help = "show l2sess", + .function = l2sess_show_command_fn, +}; + +VLIB_CLI_COMMAND (l2sess_show_count_command, static) = { + .path = "show l2sess count", + .short_help = "show l2sess count", + .function = l2sess_show_count_command_fn, +}; +/* *INDENT-OFF* */ + +static inline u64 +time_sec_to_clock( clib_time_t *ct, f64 sec) +{ + return (u64)(((f64)sec)/ct->seconds_per_clock); +} + +static clib_error_t * l2sess_init (vlib_main_t * vm) +{ + l2sess_main_t * sm = &l2sess_main; + clib_error_t * error = 0; + u64 cpu_time_now = clib_cpu_time_now(); + + + clib_time_t *ct = &vm->clib_time; + sm->udp_session_idle_timeout = time_sec_to_clock(ct, UDP_SESSION_IDLE_TIMEOUT_SEC); + sm->tcp_session_idle_timeout = time_sec_to_clock(ct, TCP_SESSION_IDLE_TIMEOUT_SEC); + sm->tcp_session_transient_timeout = time_sec_to_clock(ct, TCP_SESSION_TRANSIENT_TIMEOUT_SEC); + + /* The min sched time of 10e-1 causes erroneous behavior... */ + sm->timing_wheel.min_sched_time = 10e-2; + sm->timing_wheel.max_sched_time = 3600.0*48.0; + timing_wheel_init (&sm->timing_wheel, cpu_time_now, vm->clib_time.clocks_per_second); + sm->timer_wheel_next_expiring_time = 0; + sm->timer_wheel_tick = time_sec_to_clock(ct, sm->timing_wheel.min_sched_time); + /* Pre-allocate expired nodes. */ + vec_alloc (sm->data_from_advancing_timing_wheel, 32); + + l2sess_setup_nodes(); + l2output_init_output_node_vec (&sm->output_next_nodes.output_node_index_vec); + + return error; +} + +VLIB_INIT_FUNCTION (l2sess_init); + + diff --git a/src/plugins/acl/l2sess.h b/src/plugins/acl/l2sess.h new file mode 100644 index 00000000..db899917 --- /dev/null +++ b/src/plugins/acl/l2sess.h @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_l2sess_h__ +#define __included_l2sess_h__ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#define _(node_name, node_var, is_out, is_ip6, is_track) +#undef _ +#define foreach_l2sess_node \ + _("aclp-l2s-input-ip4-add", l2sess_in_ip4_add, 0, 0, 0) \ + _("aclp-l2s-input-ip6-add", l2sess_in_ip6_add, 0, 1, 0) \ + _("aclp-l2s-output-ip4-add", l2sess_out_ip4_add, 1, 0, 0) \ + _("aclp-l2s-output-ip6-add", l2sess_out_ip6_add, 1, 1, 0) \ + _("aclp-l2s-input-ip4-track", l2sess_in_ip4_track, 0, 0, 1) \ + _("aclp-l2s-input-ip6-track", l2sess_in_ip6_track, 0, 1, 1) \ + _("aclp-l2s-output-ip4-track",l2sess_out_ip4_track, 1, 0, 1) \ + _("aclp-l2s-output-ip6-track", l2sess_out_ip6_track, 1, 1, 1) + +#define _(node_name, node_var, is_out, is_ip6, is_track) \ + extern vlib_node_registration_t node_var; +foreach_l2sess_node +#undef _ + +#define TCP_FLAG_FIN 0x01 +#define TCP_FLAG_SYN 0x02 +#define TCP_FLAG_RST 0x04 +#define TCP_FLAG_PUSH 0x08 +#define TCP_FLAG_ACK 0x10 +#define TCP_FLAG_URG 0x20 +#define TCP_FLAG_ECE 0x40 +#define TCP_FLAG_CWR 0x80 +#define TCP_FLAGS_RSTFINACKSYN (TCP_FLAG_RST + TCP_FLAG_FIN + TCP_FLAG_SYN + TCP_FLAG_ACK) +#define TCP_FLAGS_ACKSYN (TCP_FLAG_SYN + TCP_FLAG_ACK) + +typedef struct { + ip46_address_t addr; + u64 active_time; + u64 n_packets; + u64 n_bytes; + u16 port; +} l2s_session_side_t; + +enum { + L2S_SESSION_SIDE_IN = 0, + L2S_SESSION_SIDE_OUT, + L2S_N_SESSION_SIDES +}; + +typedef struct { + u64 create_time; + l2s_session_side_t side[L2S_N_SESSION_SIDES]; + u8 l4_proto; + u8 is_ip6; + u16 tcp_flags_seen; /* u16 because of two sides */ +} l2s_session_t; + +#define PROD +#ifdef PROD +#define UDP_SESSION_IDLE_TIMEOUT_SEC 600 +#define TCP_SESSION_IDLE_TIMEOUT_SEC (3600*24) +#define TCP_SESSION_TRANSIENT_TIMEOUT_SEC 120 +#else +#define UDP_SESSION_IDLE_TIMEOUT_SEC 15 +#define TCP_SESSION_IDLE_TIMEOUT_SEC 15 +#define TCP_SESSION_TRANSIENT_TIMEOUT_SEC 5 +#endif + +typedef struct { + /* + * the next two fields are present for all nodes, but + * only one of them is used per node - depending + * on whether the node is an input or output one. + */ +#define _(node_name, node_var, is_out, is_ip6, is_track) \ + u32 node_var ## _input_next_node_index[32]; \ + l2_output_next_nodes_st node_var ## _next_nodes; +foreach_l2sess_node +#undef _ + l2_output_next_nodes_st output_next_nodes; + + /* Next indices of the tracker nodes */ + u32 next_slot_track_node_by_is_ip6_is_out[2][2]; + + /* + * Pairing of "forward" and "reverse" tables by table index. + * Each relationship has two entries - for one and the other table, + * so it is bidirectional. + */ + + u32 *fwd_to_rev_by_table_index; + + /* + * The vector of per-interface session pools + */ + + l2s_session_t *sessions; + + /* The session timeouts */ + u64 tcp_session_transient_timeout; + u64 tcp_session_idle_timeout; + u64 udp_session_idle_timeout; + + /* Timing wheel to time out the idle sessions */ + timing_wheel_t timing_wheel; + u32 *data_from_advancing_timing_wheel; + u64 timer_wheel_next_expiring_time; + u64 timer_wheel_tick; + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; + ethernet_main_t * ethernet_main; + + /* Counter(s) */ + u64 counter_attempted_delete_free_session; +} l2sess_main_t; + +l2sess_main_t l2sess_main; + +/* Just exposed for acl.c */ + +void +l2sess_vlib_plugin_register (vlib_main_t * vm, void * hh, + int from_early_init); + + +#endif /* __included_l2sess_h__ */ diff --git a/src/plugins/acl/l2sess_node.c b/src/plugins/acl/l2sess_node.c new file mode 100644 index 00000000..520e5929 --- /dev/null +++ b/src/plugins/acl/l2sess_node.c @@ -0,0 +1,816 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include + + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + u32 trace_flags; + u32 session_tables[2]; + u32 session_nexts[2]; + u8 l4_proto; +} l2sess_trace_t; + +/* packet trace format function */ + +#define _(node_name, node_var, is_out, is_ip6, is_track) \ +static u8 * format_## node_var ##_trace (u8 * s, va_list * args) \ +{ \ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); \ + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); \ + l2sess_trace_t * t = va_arg (*args, l2sess_trace_t *); \ + \ + s = format (s, node_name ": sw_if_index %d, next index %d trace_flags %08x L4 proto %d\n" \ + " tables [ %d, %d ] nexts [ %d, %d ]", \ + t->sw_if_index, t->next_index, t->trace_flags, t->l4_proto, \ + t->session_tables[0], t->session_tables[1], \ + t->session_nexts[0], t->session_nexts[1]); \ + return s; \ +} +foreach_l2sess_node +#undef _ +#define foreach_l2sess_error \ +_(SWAPPED, "Mac swap packets processed") + typedef enum +{ +#define _(sym,str) L2SESS_ERROR_##sym, + foreach_l2sess_error +#undef _ + L2SESS_N_ERROR, +} l2sess_error_t; + +static char *l2sess_error_strings[] = { +#define _(sym,string) string, + foreach_l2sess_error +#undef _ +}; + +typedef enum +{ + L2SESS_NEXT_DROP, + L2SESS_N_NEXT, +} l2sess_next_t; + +u8 +l2sess_get_l4_proto (vlib_buffer_t * b0, int node_is_ip6) +{ + u8 proto; + int proto_offset; + if (node_is_ip6) + { + proto_offset = 20; + } + else + { + proto_offset = 23; + } + proto = *((u8 *) vlib_buffer_get_current (b0) + proto_offset); + return proto; +} + + +u8 +l2sess_get_tcp_flags (vlib_buffer_t * b0, int node_is_ip6) +{ + u8 flags; + int flags_offset; + if (node_is_ip6) + { + flags_offset = 14 + 40 + 13; /* FIXME: no extension headers assumed */ + } + else + { + flags_offset = 14 + 20 + 13; + } + flags = *((u8 *) vlib_buffer_get_current (b0) + flags_offset); + return flags; +} + +static inline int +l4_tcp_or_udp (u8 proto) +{ + return ((proto == 6) || (proto == 17)); +} + +void +l2sess_get_session_tables (l2sess_main_t * sm, u32 sw_if_index, + int node_is_out, int node_is_ip6, u8 l4_proto, + u32 * session_tables) +{ +/* + * Based on the direction, l3 and l4 protocol, fill a u32[2] array: + * [0] is index for the "direct match" path, [1] is for "mirrored match". + * Store the indices of the tables to add the session to in session_tables[] + */ + l2_output_classify_main_t *l2om = &l2_output_classify_main; + l2_input_classify_main_t *l2im = &l2_input_classify_main; + + u32 output_table_index; + u32 input_table_index; + + if (!l4_tcp_or_udp (l4_proto)) + { + return; + } + + if (node_is_ip6) + { + vec_validate_init_empty (l2im-> + classify_table_index_by_sw_if_index + [L2_INPUT_CLASSIFY_TABLE_IP6], sw_if_index, + ~0); + input_table_index = + l2im-> + classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP6] + [sw_if_index]; + vec_validate_init_empty (l2om-> + classify_table_index_by_sw_if_index + [L2_OUTPUT_CLASSIFY_TABLE_IP6], sw_if_index, + ~0); + output_table_index = + l2om-> + classify_table_index_by_sw_if_index[L2_OUTPUT_CLASSIFY_TABLE_IP6] + [sw_if_index]; + } + else + { + vec_validate_init_empty (l2im-> + classify_table_index_by_sw_if_index + [L2_INPUT_CLASSIFY_TABLE_IP4], sw_if_index, + ~0); + input_table_index = + l2im-> + classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP4] + [sw_if_index]; + vec_validate_init_empty (l2om-> + classify_table_index_by_sw_if_index + [L2_OUTPUT_CLASSIFY_TABLE_IP4], sw_if_index, + ~0); + output_table_index = + l2om-> + classify_table_index_by_sw_if_index[L2_OUTPUT_CLASSIFY_TABLE_IP4] + [sw_if_index]; + } + + if (node_is_out) + { + session_tables[0] = output_table_index; + session_tables[1] = input_table_index; + } + else + { + session_tables[0] = input_table_index; + session_tables[1] = output_table_index; + } +} + +void +l2sess_get_session_nexts (l2sess_main_t * sm, u32 sw_if_index, + int node_is_out, int node_is_ip6, u8 l4_proto, + u32 * session_nexts) +{ +/* + * Based on the direction, l3 and l4 protocol, fill a u32[2] array: + * [0] is the index for the "direct match" path, [1] is for "mirrored match". + * Store the match_next_index in session_nexts[] for a new session entry which is being added to session tables. + */ + u32 input_node_index; + u32 output_node_index; + + if (!l4_tcp_or_udp (l4_proto)) + { + return; + } + + input_node_index = + sm->next_slot_track_node_by_is_ip6_is_out[node_is_ip6][0]; + output_node_index = + sm->next_slot_track_node_by_is_ip6_is_out[node_is_ip6][1]; + + if (node_is_out) + { + session_nexts[0] = output_node_index; + session_nexts[1] = input_node_index; + } + else + { + session_nexts[0] = input_node_index; + session_nexts[1] = output_node_index; + } +} + + +static inline void +swap_bytes (vlib_buffer_t * b0, int off_a, int off_b, int nbytes) +{ + u8 tmp; + u8 *pa = vlib_buffer_get_current (b0) + off_a; + u8 *pb = vlib_buffer_get_current (b0) + off_b; + while (nbytes--) + { + tmp = *pa; + *pa++ = *pb; + *pb++ = tmp; + } +} + +/* + * This quite pro[bv]ably is a terrible idea performance wise. Moreso doing it twice. + * Would having a long (ish) chunk of memory work better for this ? + * We will see when we get to the performance of this. + */ +void +l2sess_flip_l3l4_fields (vlib_buffer_t * b0, int node_is_ip6, u8 l4_proto) +{ + if (!l4_tcp_or_udp (l4_proto)) + { + return; + } + if (node_is_ip6) + { + swap_bytes (b0, 22, 38, 16); /* L3 */ + swap_bytes (b0, 54, 56, 2); /* L4 (when no EH!) */ + } + else + { + swap_bytes (b0, 26, 30, 4); /* L3 */ + swap_bytes (b0, 34, 36, 2); /* L4 */ + } +} + +void +l2sess_add_session (vlib_buffer_t * b0, int node_is_out, int node_is_ip6, + u32 session_table, u32 session_match_next, + u32 opaque_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + u32 action = 0; + u32 metadata = 0; + +#ifdef DEBUG_SESSIONS + printf ("Adding session to table %d with next %d\n", session_table, + session_match_next); +#endif + vnet_classify_add_del_session (cm, session_table, + vlib_buffer_get_current (b0), + session_match_next, opaque_index, 0, action, + metadata, 1); +} + + + +static void * +get_ptr_to_offset (vlib_buffer_t * b0, int offset) +{ + u8 *p = vlib_buffer_get_current (b0) + offset; + return p; +} + + +/* + * FIXME: Hardcoded offsets are ugly, although if casting to structs one + * would need to take care about alignment.. So let's for now be naive and simple. + */ + +void +session_store_ip4_l3l4_info (vlib_buffer_t * b0, l2s_session_t * sess, + int node_is_out) +{ + clib_memcpy (&sess->side[1 - node_is_out].addr.ip4, + get_ptr_to_offset (b0, 26), 4); + clib_memcpy (&sess->side[node_is_out].addr.ip4, get_ptr_to_offset (b0, 30), + 4); + sess->side[1 - node_is_out].port = + ntohs (*(u16 *) get_ptr_to_offset (b0, 34)); + sess->side[node_is_out].port = ntohs (*(u16 *) get_ptr_to_offset (b0, 36)); +} + +void +session_store_ip6_l3l4_info (vlib_buffer_t * b0, l2s_session_t * sess, + int node_is_out) +{ + clib_memcpy (&sess->side[1 - node_is_out].addr.ip6, + get_ptr_to_offset (b0, 22), 16); + clib_memcpy (&sess->side[node_is_out].addr.ip4, get_ptr_to_offset (b0, 38), + 16); + sess->side[1 - node_is_out].port = + ntohs (*(u16 *) get_ptr_to_offset (b0, 54)); + sess->side[node_is_out].port = ntohs (*(u16 *) get_ptr_to_offset (b0, 56)); +} + +static void +build_match_from_session (l2sess_main_t * sm, u8 * match, + l2s_session_t * sess, int is_out) +{ + if (sess->is_ip6) + { + match[20] = sess->l4_proto; + clib_memcpy (&match[22], &sess->side[1 - is_out].addr.ip6, 16); + clib_memcpy (&match[38], &sess->side[is_out].addr.ip4, 16); + *(u16 *) & match[54] = htons (sess->side[1 - is_out].port); + *(u16 *) & match[56] = htons (sess->side[is_out].port); + } + else + { + match[23] = sess->l4_proto; + clib_memcpy (&match[26], &sess->side[1 - is_out].addr.ip6, 4); + clib_memcpy (&match[30], &sess->side[is_out].addr.ip4, 4); + *(u16 *) & match[34] = htons (sess->side[1 - is_out].port); + *(u16 *) & match[36] = htons (sess->side[is_out].port); + } +} + +static void +delete_session (l2sess_main_t * sm, u32 sw_if_index, u32 session_index) +{ + vnet_classify_main_t *cm = &vnet_classify_main; + u8 match[5 * 16]; /* For building the mock of the packet to delete the classifier session */ + u32 session_tables[2] = { ~0, ~0 }; + l2s_session_t *sess = sm->sessions + session_index; + if (pool_is_free (sm->sessions, sess)) + { + sm->counter_attempted_delete_free_session++; + return; + } + l2sess_get_session_tables (sm, sw_if_index, 0, sess->is_ip6, sess->l4_proto, + session_tables); + if (session_tables[1] != ~0) + { + build_match_from_session (sm, match, sess, 1); + vnet_classify_add_del_session (cm, session_tables[1], match, 0, 0, 0, 0, + 0, 0); + } + if (session_tables[1] != ~0) + { + build_match_from_session (sm, match, sess, 1); + vnet_classify_add_del_session (cm, session_tables[1], match, 0, 0, 0, 0, + 0, 0); + } + pool_put (sm->sessions, sess); +} + +static void +udp_session_account_buffer (vlib_buffer_t * b0, l2s_session_t * s, + int which_side, u64 now) +{ + l2s_session_side_t *ss = &s->side[which_side]; + ss->active_time = now; + ss->n_packets++; + ss->n_bytes += b0->current_data + b0->current_length; +} + +static inline u64 +udp_session_get_timeout (l2sess_main_t * sm, l2s_session_t * sess, u64 now) +{ + return (sm->udp_session_idle_timeout); +} + +static void +tcp_session_account_buffer (vlib_buffer_t * b0, l2s_session_t * s, + int which_side, u64 now) +{ + l2s_session_side_t *ss = &s->side[which_side]; + ss->active_time = now; + ss->n_packets++; + ss->n_bytes += b0->current_data + b0->current_length; + /* Very very lightweight TCP state tracking: just record which flags were seen */ + s->tcp_flags_seen |= + l2sess_get_tcp_flags (b0, s->is_ip6) << (8 * which_side); +} + +/* + * Since we are tracking for the purposes of timing the sessions out, + * we mostly care about two states: established (maximize the idle timeouts) + * and transient (halfopen/halfclosed/reset) - we need to have a reasonably short timeout to + * quickly get rid of sessions but not short enough to violate the TCP specs. + */ + +static inline u64 +tcp_session_get_timeout (l2sess_main_t * sm, l2s_session_t * sess, u64 now) +{ + /* seen both SYNs and ACKs but not FINs means we are in establshed state */ + u16 masked_flags = + sess->tcp_flags_seen & ((TCP_FLAGS_RSTFINACKSYN << 8) + + TCP_FLAGS_RSTFINACKSYN); + if (((TCP_FLAGS_ACKSYN << 8) + TCP_FLAGS_ACKSYN) == masked_flags) + { + return (sm->tcp_session_idle_timeout); + } + else + { + return (sm->tcp_session_transient_timeout); + } +} + +static inline u64 +session_get_timeout (l2sess_main_t * sm, l2s_session_t * sess, u64 now) +{ + u64 timeout; + + switch (sess->l4_proto) + { + case 6: + timeout = tcp_session_get_timeout (sm, sess, now); + break; + case 17: + timeout = udp_session_get_timeout (sm, sess, now); + break; + default: + timeout = 0; + } + + return timeout; +} + +static inline u64 +get_session_last_active_time(l2s_session_t * sess) +{ + u64 last_active = + sess->side[0].active_time > + sess->side[1].active_time ? sess->side[0].active_time : sess->side[1]. + active_time; + return last_active; +} + +static int +session_is_alive (l2sess_main_t * sm, l2s_session_t * sess, u64 now, u64 *last_active_cache) +{ + u64 last_active = get_session_last_active_time(sess); + u64 timeout = session_get_timeout (sm, sess, now); + int is_alive = ((now - last_active) < timeout); + if (last_active_cache) + *last_active_cache = last_active; + return is_alive; +} + +static void +check_idle_sessions (l2sess_main_t * sm, u32 sw_if_index, u64 now) +{ + sm->timer_wheel_next_expiring_time = 0; + sm->data_from_advancing_timing_wheel + = + timing_wheel_advance (&sm->timing_wheel, now, + sm->data_from_advancing_timing_wheel, + &sm->timer_wheel_next_expiring_time); +#ifdef DEBUG_SESSIONS_VERBOSE + { + clib_time_t *ct = &sm->vlib_main->clib_time; + f64 ctime; + ctime = now * ct->seconds_per_clock; + clib_warning ("Now : %U", format_time_interval, "h:m:s:u", ctime); + ctime = sm->timer_wheel_next_expiring_time * ct->seconds_per_clock; + clib_warning ("Next expire: %U", format_time_interval, "h:m:s:u", ctime); + clib_warning ("Expired items: %d", + (int) vec_len (sm->data_from_advancing_timing_wheel)); + } +#endif + + sm->timer_wheel_next_expiring_time = now + sm->timer_wheel_tick; + if (PREDICT_FALSE ( 0 == sm->data_from_advancing_timing_wheel )) { + return; + } + + if (PREDICT_FALSE (_vec_len (sm->data_from_advancing_timing_wheel) > 0)) + { + uword i; + for (i = 0; i < _vec_len (sm->data_from_advancing_timing_wheel); i++) + { + u32 session_index = sm->data_from_advancing_timing_wheel[i]; + if (!pool_is_free_index (sm->sessions, session_index)) + { + l2s_session_t *sess = sm->sessions + session_index; + u64 last_active; + if (session_is_alive (sm, sess, now, &last_active)) + { +#ifdef DEBUG_SESSIONS + clib_warning ("Restarting timer for session %d", (int) session_index); +#endif + /* Pretend we did this in the past, at last_active moment */ + timing_wheel_insert (&sm->timing_wheel, + last_active + session_get_timeout (sm, sess, + last_active), + session_index); + } + else + { +#ifdef DEBUG_SESSIONS + clib_warning ("Deleting session %d", (int) session_index); +#endif + delete_session (sm, sw_if_index, session_index); + } + } + } + _vec_len (sm->data_from_advancing_timing_wheel) = 0; + } +} + +static uword +l2sess_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + u32 n_left_from, *from, *to_next; + l2sess_next_t next_index; + u32 pkts_swapped = 0; + u32 cached_sw_if_index = (u32) ~ 0; + u32 cached_next_index = (u32) ~ 0; + u32 feature_bitmap0; + u32 trace_flags0; + + l2sess_main_t *sm = &l2sess_main; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Only a single loop for now for simplicity */ + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = L2SESS_NEXT_DROP; + u32 sw_if_index0; + //ethernet_header_t *en0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + //en0 = vlib_buffer_get_current (b0); + +/* + * The non-boilerplate is in the block below. + * Note first a magic macro block that sets up the behavior qualifiers: + * node_is_out : 1 = is output, 0 = is input + * node_is_ip6 : 1 = is ip6, 0 = is ip4 + * node_is_track : 1 = is a state tracking node, 0 - is a session addition node + * + * Subsequently the code adjusts its behavior depending on these variables. + * It's most probably not great performance wise but much easier to work with. + * + */ + { + int node_is_out = -1; + CLIB_UNUSED (int node_is_ip6) = -1; + CLIB_UNUSED (int node_is_track) = -1; + u32 node_index = 0; + u32 session_tables[2] = { ~0, ~0 }; + u32 session_nexts[2] = { ~0, ~0 }; + l2_output_next_nodes_st *next_nodes = 0; + u32 *input_feat_next_node_index; + u8 l4_proto; + u64 now = clib_cpu_time_now (); + +/* + * Set the variables according to which of the 8 nodes we are. + * Hopefully the compiler is smart enough to eliminate the extraneous. + */ +#define _(node_name, node_var, is_out, is_ip6, is_track) \ +if(node_var.index == node->node_index) \ + { \ + node_is_out = is_out; \ + node_is_ip6 = is_ip6; \ + node_is_track = is_track; \ + node_index = node_var.index; \ + next_nodes = &sm->node_var ## _next_nodes; \ + input_feat_next_node_index = sm->node_var ## _input_next_node_index; \ + } + foreach_l2sess_node +#undef _ + trace_flags0 = 0; + if (node_is_out) + { + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + } + else + { + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + } + /* potentially also remove the nodes here */ + feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; + + if (node_is_track) + { + u32 sess_index = vnet_buffer (b0)->l2_classify.opaque_index; + l2s_session_t *sess = sm->sessions + sess_index; + l4_proto = sess->l4_proto; + + if (session_is_alive (sm, sess, now, 0)) + { + if (6 == l4_proto) + { + tcp_session_account_buffer (b0, sess, node_is_out, + now); + } + else + { + udp_session_account_buffer (b0, sess, node_is_out, + now); + } + } + else + { + timing_wheel_delete (&sm->timing_wheel, sess_index); + delete_session (sm, sw_if_index0, sess_index); + /* FIXME: drop the packet that hit the obsolete node, for now. We really ought to recycle it. */ + next0 = 0; + } + } + else + { + /* + * "-add" node: take l2opaque which arrived to us, and deduce + * the tables out of that. ~0 means the topmost classifier table + * applied for this AF on the RX(for input)/TX(for output)) sw_if_index. + * Also add the mirrored session to the paired table. + */ + l2s_session_t *sess; + u32 sess_index; + + l4_proto = l2sess_get_l4_proto (b0, node_is_ip6); + + pool_get (sm->sessions, sess); + sess_index = sess - sm->sessions; + sess->create_time = now; + sess->side[node_is_out].active_time = now; + sess->side[1 - node_is_out].active_time = now; + sess->l4_proto = l4_proto; + sess->is_ip6 = node_is_ip6; + if (node_is_ip6) + { + session_store_ip6_l3l4_info (b0, sess, node_is_out); + } + else + { + session_store_ip4_l3l4_info (b0, sess, node_is_out); + } + + l2sess_get_session_tables (sm, sw_if_index0, node_is_out, + node_is_ip6, l4_proto, + session_tables); + l2sess_get_session_nexts (sm, sw_if_index0, node_is_out, + node_is_ip6, l4_proto, + session_nexts); + l2sess_flip_l3l4_fields (b0, node_is_ip6, l4_proto); + if (session_tables[1] != ~0) + { + l2sess_add_session (b0, node_is_out, node_is_ip6, + session_tables[1], session_nexts[1], + sess_index); + } + l2sess_flip_l3l4_fields (b0, node_is_ip6, l4_proto); + if (session_tables[0] != ~0) + { + l2sess_add_session (b0, node_is_out, node_is_ip6, + session_tables[0], session_nexts[0], + sess_index); + } + if (6 == sess->l4_proto) + { + tcp_session_account_buffer (b0, sess, node_is_out, now); + } + else + { + udp_session_account_buffer (b0, sess, node_is_out, now); + } + timing_wheel_insert (&sm->timing_wheel, + now + session_get_timeout (sm, sess, + now), + sess_index); + } + + if (now >= sm->timer_wheel_next_expiring_time) + { + check_idle_sessions (sm, sw_if_index0, now); + } + + if (node_is_out) + { + if (feature_bitmap0) + { + trace_flags0 |= 0x10; + } + if (sw_if_index0 == cached_sw_if_index) + { + trace_flags0 |= 0x20; + } + l2_output_dispatch (sm->vlib_main, + sm->vnet_main, + node, + node_index, + &cached_sw_if_index, + &cached_next_index, + next_nodes, + b0, sw_if_index0, feature_bitmap0, + &next0); + trace_flags0 |= 2; + + } + else + { + next0 = + feat_bitmap_get_next_node_index (input_feat_next_node_index, + feature_bitmap0); + trace_flags0 |= 4; + + } + + + + if (next0 >= node->n_next_nodes) + { + trace_flags0 |= 1; + } + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + l2sess_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->trace_flags = trace_flags0; + t->l4_proto = l4_proto; + t->session_tables[0] = session_tables[0]; + t->session_tables[1] = session_tables[1]; + t->session_nexts[0] = session_nexts[0]; + t->session_nexts[1] = session_nexts[1]; + } + + } + pkts_swapped += 1; + if (next0 >= node->n_next_nodes) + { + next0 = 0; + } + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + vlib_node_increment_counter (vm, node->node_index, + L2SESS_ERROR_SWAPPED, pkts_swapped); + return frame->n_vectors; +} + + +#define _(node_name, node_var, is_out, is_ip6, is_track) \ +static uword \ +node_var ## node_fn (vlib_main_t * vm, \ + vlib_node_runtime_t * node, \ + vlib_frame_t * frame) \ +{ \ + return l2sess_node_fn(vm, node, frame); \ +} \ +VLIB_REGISTER_NODE (node_var) = { \ + .function = node_var ## node_fn, \ + .name = node_name, \ + .vector_size = sizeof (u32), \ + .format_trace = format_ ## node_var ## _trace, \ + .type = VLIB_NODE_TYPE_INTERNAL, \ + \ + .n_errors = ARRAY_LEN(l2sess_error_strings), \ + .error_strings = l2sess_error_strings, \ + \ + .n_next_nodes = L2SESS_N_NEXT, \ + .next_nodes = { \ + [L2SESS_NEXT_DROP] = "error-drop", \ + }, \ +}; +foreach_l2sess_node +#undef _ diff --git a/src/plugins/acl/node_in.c b/src/plugins/acl/node_in.c new file mode 100644 index 00000000..2a5199a9 --- /dev/null +++ b/src/plugins/acl/node_in.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include "node_in.h" + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + u32 match_acl_index; + u32 match_rule_index; + u32 trace_bitmap; +} acl_in_trace_t; + +/* packet trace format function */ +static u8 * +format_acl_in_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + acl_in_trace_t *t = va_arg (*args, acl_in_trace_t *); + + s = + format (s, + "ACL_IN: sw_if_index %d, next index %d, match: inacl %d rule %d trace_bits %08x", + t->sw_if_index, t->next_index, t->match_acl_index, + t->match_rule_index, t->trace_bitmap); + return s; +} + +vlib_node_registration_t acl_in_node; + +#define foreach_acl_in_error \ +_(ACL_CHECK, "InACL check packets processed") + +typedef enum +{ +#define _(sym,str) ACL_IN_ERROR_##sym, + foreach_acl_in_error +#undef _ + ACL_IN_N_ERROR, +} acl_in_error_t; + +static char *acl_in_error_strings[] = { +#define _(sym,string) string, + foreach_acl_in_error +#undef _ +}; + +static uword +acl_in_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + u32 n_left_from, *from, *to_next; + acl_in_next_t next_index; + u32 pkts_acl_checked = 0; + u32 feature_bitmap0; + u32 trace_bitmap = 0; + u32 *input_feat_next_node_index = + acl_main.acl_in_node_input_next_node_index; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = ~0; + u32 sw_if_index0; + u32 next = ~0; + u32 match_acl_index = ~0; + u32 match_rule_index = ~0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; + + input_acl_packet_match (sw_if_index0, b0, &next, &match_acl_index, + &match_rule_index, &trace_bitmap); + if (next != ~0) + { + next0 = next; + } + if (next0 == ~0) + { + next0 = + feat_bitmap_get_next_node_index (input_feat_next_node_index, + feature_bitmap0); + } + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + acl_in_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->match_acl_index = match_acl_index; + t->match_rule_index = match_rule_index; + t->trace_bitmap = trace_bitmap; + } + + next0 = next0 < node->n_next_nodes ? next0 : 0; + + pkts_acl_checked += 1; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, acl_in_node.index, + ACL_IN_ERROR_ACL_CHECK, pkts_acl_checked); + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (acl_in_node) = +{ + .function = acl_in_node_fn,.name = "acl-plugin-in",.vector_size = + sizeof (u32),.format_trace = format_acl_in_trace,.type = + VLIB_NODE_TYPE_INTERNAL,.n_errors = + ARRAY_LEN (acl_in_error_strings),.error_strings = + acl_in_error_strings,.n_next_nodes = ACL_IN_N_NEXT, + /* edit / add dispositions here */ + .next_nodes = + { + [ACL_IN_ERROR_DROP] = "error-drop", + [ACL_IN_ETHERNET_INPUT] = "ethernet-input", + [ACL_IN_L2S_INPUT_IP4_ADD] = "aclp-l2s-input-ip4-add", + [ACL_IN_L2S_INPUT_IP6_ADD] = "aclp-l2s-input-ip6-add",} +,}; diff --git a/src/plugins/acl/node_in.h b/src/plugins/acl/node_in.h new file mode 100644 index 00000000..502bbf8d --- /dev/null +++ b/src/plugins/acl/node_in.h @@ -0,0 +1,12 @@ +#ifndef _NODE_IN_H_ +#define _NODE_IN_H_ + +typedef enum { + ACL_IN_ERROR_DROP, + ACL_IN_ETHERNET_INPUT, + ACL_IN_L2S_INPUT_IP4_ADD, + ACL_IN_L2S_INPUT_IP6_ADD, + ACL_IN_N_NEXT, +} acl_in_next_t; + +#endif diff --git a/src/plugins/acl/node_out.c b/src/plugins/acl/node_out.c new file mode 100644 index 00000000..50af3679 --- /dev/null +++ b/src/plugins/acl/node_out.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +#include "node_out.h" + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + u32 match_acl_index; + u32 match_rule_index; + u32 trace_bitmap; +} acl_out_trace_t; + +/* packet trace format function */ +static u8 * +format_acl_out_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + acl_out_trace_t *t = va_arg (*args, acl_out_trace_t *); + s = + format (s, + "ACL_OUT: sw_if_index %d, next index %d, match: outacl %d rule %d trace_bits %08x", + t->sw_if_index, t->next_index, t->match_acl_index, + t->match_rule_index, t->trace_bitmap); + return s; +} + +vlib_node_registration_t acl_out_node; + +#define foreach_acl_out_error \ +_(ACL_CHECK, "OutACL check packets processed") + +typedef enum +{ +#define _(sym,str) ACL_OUT_ERROR_##sym, + foreach_acl_out_error +#undef _ + ACL_OUT_N_ERROR, +} acl_out_error_t; + +static char *acl_out_error_strings[] = { +#define _(sym,string) string, + foreach_acl_out_error +#undef _ +}; + +static uword +acl_out_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + acl_main_t *am = &acl_main; + l2_output_next_nodes_st *next_nodes = &am->acl_out_output_next_nodes; + u32 n_left_from, *from, *to_next; + acl_out_next_t next_index; + u32 pkts_acl_checked = 0; + u32 feature_bitmap0; + u32 cached_sw_if_index = (u32) ~ 0; + u32 cached_next_index = (u32) ~ 0; + u32 match_acl_index = ~0; + u32 match_rule_index = ~0; + u32 trace_bitmap = 0; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = ~0; + u32 next = 0; + u32 sw_if_index0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; + + output_acl_packet_match (sw_if_index0, b0, &next, &match_acl_index, + &match_rule_index, &trace_bitmap); + if (next != ~0) + { + next0 = next; + } + if (next0 == ~0) + { + l2_output_dispatch (vm, + am->vnet_main, + node, + acl_out_node.index, + &cached_sw_if_index, + &cached_next_index, + next_nodes, + b0, sw_if_index0, feature_bitmap0, &next0); + } + + + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + acl_out_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->match_acl_index = match_acl_index; + t->match_rule_index = match_rule_index; + t->trace_bitmap = trace_bitmap; + } + + pkts_acl_checked += 1; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, acl_out_node.index, + ACL_OUT_ERROR_ACL_CHECK, pkts_acl_checked); + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (acl_out_node) = +{ + .function = acl_out_node_fn,.name = "acl-plugin-out",.vector_size = + sizeof (u32),.format_trace = format_acl_out_trace,.type = + VLIB_NODE_TYPE_INTERNAL,.n_errors = + ARRAY_LEN (acl_out_error_strings),.error_strings = + acl_out_error_strings,.n_next_nodes = ACL_OUT_N_NEXT, + /* edit / add dispositions here */ + .next_nodes = + { + [ACL_OUT_ERROR_DROP] = "error-drop", + [ACL_OUT_INTERFACE_OUTPUT] = "interface-output", + [ACL_OUT_L2S_OUTPUT_IP4_ADD] = "aclp-l2s-output-ip4-add", + [ACL_OUT_L2S_OUTPUT_IP6_ADD] = "aclp-l2s-output-ip6-add",} +,}; diff --git a/src/plugins/acl/node_out.h b/src/plugins/acl/node_out.h new file mode 100644 index 00000000..c919f3b7 --- /dev/null +++ b/src/plugins/acl/node_out.h @@ -0,0 +1,12 @@ +#ifndef _NODE_OUT_H_ +#define _NODE_OUT_H_ + +typedef enum { + ACL_OUT_ERROR_DROP, + ACL_OUT_INTERFACE_OUTPUT, + ACL_OUT_L2S_OUTPUT_IP4_ADD, + ACL_OUT_L2S_OUTPUT_IP6_ADD, + ACL_OUT_N_NEXT, +} acl_out_next_t; + +#endif diff --git a/src/plugins/acl/test/run-python b/src/plugins/acl/test/run-python new file mode 100755 index 00000000..215eb17a --- /dev/null +++ b/src/plugins/acl/test/run-python @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Do all the legwork to run a scapy shell with APIs available for load +# +CURR_DIR=`pwd` +ROOT_DIR=`git rev-parse --show-toplevel` +cd $ROOT_DIR +sudo apt-get install -y python-virtualenv +# uncomment the line below to enable build of plugins and api each time +# make plugins && make build-vpp-api || exit +virtualenv virtualenv +virtualenv/bin/pip install ipaddress +virtualenv/bin/pip install scapy +# install the python API into the virtualenv +cd $ROOT_DIR/vpp-api/python/ +$ROOT_DIR/virtualenv/bin/python setup.py install +# install the python ACL plugin API into the virtualenv +ACL_PLUGIN_SETUP_DIR=`find $ROOT_DIR/build-root -name acl-plugin` +cd $ACL_PLUGIN_SETUP_DIR; +$ROOT_DIR/virtualenv/bin/python setup.py install +cd $ROOT_DIR +# figure out the shared library path and start scapy +export LD_LIBRARY_PATH=`pwd`/`find . -name "libpneum.so" -exec dirname {} \; | grep lib64 | head -n 1` +cd $CURR_DIR +sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH $ROOT_DIR/virtualenv/bin/python $1 $2 $3 $4 $5 $6 $7 $8 $9 + + + diff --git a/src/plugins/acl/test/run-scapy b/src/plugins/acl/test/run-scapy new file mode 100755 index 00000000..266f07d1 --- /dev/null +++ b/src/plugins/acl/test/run-scapy @@ -0,0 +1,26 @@ +#!/bin/sh +# +# Do all the legwork to run a scapy shell with APIs available for load +# +ROOT_DIR=`git rev-parse --show-toplevel` +cd $ROOT_DIR +sudo apt-get install -y python-virtualenv +# uncomment the line below to enable the build of plugins and API each time.. +# make plugins && make build-vpp-api || exit +virtualenv virtualenv +virtualenv/bin/pip install ipaddress +virtualenv/bin/pip install scapy +# install the python API into the virtualenv +cd $ROOT_DIR/vpp-api/python/ +$ROOT_DIR/virtualenv/bin/python setup.py install +# install the python ACL plugin API into the virtualenv +ACL_PLUGIN_SETUP_DIR=`find $ROOT_DIR/build-root -name acl-plugin` +cd $ACL_PLUGIN_SETUP_DIR; +$ROOT_DIR/virtualenv/bin/python setup.py install +cd $ROOT_DIR +# figure out the shared library path and start scapy +export LD_LIBRARY_PATH=`pwd`/`find . -name "libpneum.so" -exec dirname {} \; | grep lib64 | head -n 1` +sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH virtualenv/bin/scapy + + + diff --git a/src/plugins/acl/test/test_acl_plugin.py b/src/plugins/acl/test/test_acl_plugin.py new file mode 100644 index 00000000..7fc72d67 --- /dev/null +++ b/src/plugins/acl/test/test_acl_plugin.py @@ -0,0 +1,118 @@ +from __future__ import print_function +import unittest, sys, time, threading, struct, logging, os +import vpp_papi +# import vpp_papi_plugins.acl +from ipaddress import * +papi_event = threading.Event() +print(vpp_papi.vpe.VL_API_SW_INTERFACE_SET_FLAGS) +def papi_event_handler(result): + if result.vl_msg_id == vpp_papi.vpe.VL_API_SW_INTERFACE_SET_FLAGS: + return + if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_INTERFACE_COUNTERS: + print('Interface counters', result) + return + if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_IP6_FIB_COUNTERS: + print('IPv6 FIB counters', result) + papi_event.set() + return + + print('Unknown message id:', result.vl_msg_id) + +import glob, subprocess +class TestAclPlugin(unittest.TestCase): + @classmethod + def setUpClass(cls): + print("Setup") + @classmethod + def tearDownClass(cls): + print("Teardown") + + def setUp(self): + print("Connecting API") + r = vpp_papi.connect("test_papi") + self.assertEqual(r, 0) + + def tearDown(self): + r = vpp_papi.disconnect() + self.assertEqual(r, 0) + + # + # The tests themselves + # + + # + # Basic request / reply + # + def test_show_version(self): + t = vpp_papi.show_version() + print('T', t); + program = t.program.decode().rstrip('\x00') + self.assertEqual('vpe', program) + + def x_test_acl_add(self): + print("Test ACL add") + self.assertEqual(1, 1) + + # + # Details / Dump + # + def x_test_details_dump(self): + t = vpp_papi.sw_interface_dump(0, b'') + print('Dump/details T', t) + + # + # Arrays + # + def x_test_arrays(self): + t = vpp_papi.vnet_get_summary_stats() + print('Summary stats', t) + print('Packets:', t.total_pkts[0]) + print('Packets:', t.total_pkts[1]) + # + # Variable sized arrays and counters + # + #@unittest.skip("stats") + def x_test_want_stats(self): + pid = 123 + vpp_papi.register_event_callback(papi_event_handler) + papi_event.clear() + + # Need to configure IPv6 to get som IPv6 FIB stats + t = vpp_papi.create_loopback('') + print(t) + self.assertEqual(t.retval, 0) + + ifindex = t.sw_if_index + addr = str(IPv6Address(u'1::1').packed) + t = vpp_papi.sw_interface_add_del_address(ifindex, 1, 1, 0, 16, addr) + print(t) + self.assertEqual(t.retval, 0) + + # Check if interface is up + # XXX: Add new API to query interface state based on ifindex, instead of dump all. + t = vpp_papi.sw_interface_set_flags(ifindex, 1, 1, 0) + self.assertEqual(t.retval, 0) + + t = vpp_papi.want_stats(True, pid) + + print (t) + + # + # Wait for some stats + # + self.assertEqual(papi_event.wait(15), True) + t = vpp_papi.want_stats(False, pid) + print (t) + + + # + # Plugins? + # + +if __name__ == '__main__' or __name__ == '__builtin__': + print("This is main") + suite = unittest.TestLoader().loadTestsFromTestCase(TestAclPlugin) + unittest.TextTestRunner(verbosity=2).run(suite) + #logging.basicConfig(level=logging.DEBUG) + # unittest.main() + diff --git a/src/plugins/ioam.am b/src/plugins/ioam.am new file mode 100644 index 00000000..a4984b18 --- /dev/null +++ b/src/plugins/ioam.am @@ -0,0 +1,150 @@ +# Copyright (c) 2015 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +######################################## +# iOAM Proof of Transit +######################################## + +ioam_pot_plugin_la_SOURCES = \ + ioam/lib-pot/pot_util.c \ + ioam/encap/ip6_ioam_pot.c \ + ioam/lib-pot/pot_util.h \ + ioam/lib-pot/math64.h \ + ioam/lib-pot/pot_api.c + +noinst_HEADERS += \ + ioam/lib-pot/pot_all_api_h.h \ + ioam/lib-pot/pot_msg_enum.h \ + ioam/lib-pot/pot.api.h \ + ioam/lib-pot/pot_util.h \ + ioam/lib-pot/math64.h + +API_FILES += ioam/lib-pot/pot.api + +ioam_pot_test_plugin_la_SOURCES = \ + ioam/lib-pot/pot_test.c \ + ioam/lib-pot/pot_plugin.api.h + +vppapitestplugins_LTLIBRARIES += ioam_pot_test_plugin.la +vppplugins_LTLIBRARIES += ioam_pot_plugin.la + +######################################## +# iOAM trace export for IPv6 +######################################## + +ioam_export_plugin_la_SOURCES = \ +ioam/export/ioam_export.c \ +ioam/export/node.c \ +ioam/export/ioam_export.api.h \ +ioam/export/ioam_export_thread.c + +noinst_HEADERS += \ + ioam/export/ioam_export_all_api_h.h \ + ioam/export/ioam_export_msg_enum.h \ + ioam/export/ioam_export.api.h + +API_FILES += ioam/export/ioam_export.api + +ioam_export_test_plugin_la_SOURCES = \ + ioam/export/ioam_export_test.c \ + ioam/export/ioam_export_plugin.api.h + +vppapitestplugins_LTLIBRARIES += ioam_export_test_plugin.la +vppplugins_LTLIBRARIES += ioam_export_plugin.la + +######################################## +# iOAM Trace +######################################## +libioam_trace_plugin_la_SOURCES = \ + ioam/lib-trace/trace_util.c \ + ioam/encap/ip6_ioam_trace.c \ + ioam/lib-trace/trace_util.h \ + ioam/lib-trace/trace_api.c + +noinst_HEADERS += \ + ioam/export/ioam_export_all_api_h.h \ + ioam/lib-trace/trace_all_api_h.h \ + ioam/lib-trace/trace_msg_enum.h \ + ioam/lib-trace/trace.api.h \ + ioam/lib-trace/trace_util.h + +API_FILES += ioam/lib-trace/trace.api + +ioam_trace_test_plugin_la_SOURCES = \ + ioam/lib-trace/trace_test.c \ + ioam/lib-trace/trace_plugin.api.h + +vppapitestplugins_LTLIBRARIES += ioam_trace_test_plugin.la +vppplugins_LTLIBRARIES += libioam_trace_plugin.la + +######################################## +# VxLAN-GPE +######################################## +libioam_vxlan_gpe_plugin_la_SOURCES = \ + ioam/lib-vxlan-gpe/ioam_encap.c \ + ioam/lib-vxlan-gpe/ioam_decap.c \ + ioam/lib-vxlan-gpe/ioam_transit.c \ + ioam/lib-vxlan-gpe/ioam_pop.c \ + ioam/lib-vxlan-gpe/vxlan_gpe_api.c \ + ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c \ + ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c \ + ioam/export-vxlan-gpe/vxlan_gpe_node.c \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api.h\ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c + +noinst_HEADERS += \ + ioam/export/ioam_export_all_api_h.h \ + ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h \ + ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h \ + ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api.h \ + ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h \ + ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h \ + ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api.h + +API_FILES += ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api +API_FILES += ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api + +ioam_vxlan_gpe_test_plugin_la_SOURCES = \ + ioam/lib-vxlan-gpe/vxlan_gpe_test.c \ + ioam/lib-vxlan-gpe/vxlan_gpe_plugin.api.h + +vppapitestplugins_LTLIBRARIES += ioam_vxlan_gpe_test_plugin.la +vppplugins_LTLIBRARIES += libioam_vxlan_gpe_plugin.la + +vxlan_gpe_ioam_export_test_plugin_la_SOURCES = \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c \ + ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_plugin.api.h + +vppapitestplugins_LTLIBRARIES += vxlan_gpe_ioam_export_test_plugin.la + +######################################## +# iOAM E2E plugin +######################################## + +ioam_e2e_plugin_la_SOURCES = \ + ioam/encap/ip6_ioam_e2e.c \ + ioam/encap/ip6_ioam_seqno.c \ + ioam/encap/ip6_ioam_seqno_analyse.c + +noinst_HEADERS += \ + ioam/encap/ip6_ioam_e2e.h \ + ioam/encap/ip6_ioam_seqno.h + +vppplugins_LTLIBRARIES += ioam_e2e_plugin.la + +# vi:syntax=automake diff --git a/src/plugins/ioam/dir.dox b/src/plugins/ioam/dir.dox new file mode 100644 index 00000000..f3389b52 --- /dev/null +++ b/src/plugins/ioam/dir.dox @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + @dir + @brief Inband OAM (iOAM) implementation +*/ diff --git a/src/plugins/ioam/encap/ip6_ioam_e2e.c b/src/plugins/ioam/encap/ip6_ioam_e2e.c new file mode 100644 index 00000000..0839cdce --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_e2e.c @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include + +#include "ip6_ioam_e2e.h" + +ioam_e2e_main_t ioam_e2e_main; + +static u8 * ioam_e2e_trace_handler (u8 * s, + ip6_hop_by_hop_option_t *opt) +{ + ioam_e2e_option_t * e2e = (ioam_e2e_option_t *)opt; + u32 seqno = 0; + + if (e2e) + { + seqno = clib_net_to_host_u32 (e2e->e2e_data); + } + + s = format (s, "SeqNo = 0x%Lx", seqno); + return s; +} + +int +ioam_e2e_config_handler (void *data, u8 disable) +{ + int *analyse = data; + + /* Register hanlders if enabled */ + if (!disable) + { + /* If encap node register for encap handler */ + if (0 == *analyse) + { + if (ip6_hbh_register_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, + ioam_seqno_encap_handler, + ioam_e2e_trace_handler) < 0) + { + return (-1); + } + } + /* If analyze node then register for decap handler */ + else + { + if (ip6_hbh_pop_register_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, + ioam_seqno_decap_handler) < 0) + { + return (-1); + } + } + return 0; + } + + /* UnRegister handlers */ + (void) ip6_hbh_unregister_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE); + (void) ip6_hbh_pop_unregister_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE); + return 0; +} + +int +ioam_e2e_rewrite_handler (u8 *rewrite_string, + u8 *rewrite_size) +{ + ioam_e2e_option_t *e2e_option; + + if (rewrite_string && *rewrite_size == sizeof(ioam_e2e_option_t)) + { + e2e_option = (ioam_e2e_option_t *)rewrite_string; + e2e_option->hdr.type = HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE + | HBH_OPTION_TYPE_SKIP_UNKNOWN; + e2e_option->hdr.length = sizeof (ioam_e2e_option_t) - + sizeof (ip6_hop_by_hop_option_t); + return(0); + } + return(-1); +} + +u32 +ioam_e2e_flow_handler (u32 ctx, u8 add) +{ + ioam_e2e_data_t *data; + u16 i; + + if (add) + { + pool_get(ioam_e2e_main.e2e_data, data); + data->flow_ctx = ctx; + ioam_seqno_init_bitmap(&data->seqno_data); + return ((u32) (data - ioam_e2e_main.e2e_data)); + } + + /* Delete case */ + for (i = 0; i < vec_len(ioam_e2e_main.e2e_data); i++) + { + if (pool_is_free_index(ioam_e2e_main.e2e_data, i)) + continue; + + data = pool_elt_at_index(ioam_e2e_main.e2e_data, i); + if (data && (data->flow_ctx == ctx)) + { + pool_put_index(ioam_e2e_main.e2e_data, i); + return (0); + } + } + return 0; +} + +static clib_error_t * +ioam_show_e2e_cmd_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + ioam_e2e_data_t *e2e_data; + u8 *s = 0; + int i; + + vec_reset_length(s); + + s = format(0, "IOAM E2E information: \n"); + for (i = 0; i < vec_len(ioam_e2e_main.e2e_data); i++) + { + if (pool_is_free_index(ioam_e2e_main.e2e_data, i)) + continue; + + e2e_data = pool_elt_at_index(ioam_e2e_main.e2e_data, i); + s = format(s, "Flow name: %s\n", get_flow_name_from_flow_ctx(e2e_data->flow_ctx)); + + s = show_ioam_seqno_cmd_fn(s, + &e2e_data->seqno_data, + !IOAM_DEAP_ENABLED(e2e_data->flow_ctx)); + } + + vlib_cli_output(vm, "%v", s); + return 0; +} + + +VLIB_CLI_COMMAND (ioam_show_e2e_cmd, static) = { + .path = "show ioam e2e ", + .short_help = "show ioam e2e information", + .function = ioam_show_e2e_cmd_fn, +}; + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + clib_error_t * error = 0; + + ioam_e2e_main.vlib_main = vm; + ioam_e2e_main.vnet_main = h->vnet_main; + return error; +} + +/* + * Init handler E2E headet handling. + * Init hanlder registers encap, decap, trace and Rewrite handlers. + */ +static clib_error_t * +ioam_e2e_init (vlib_main_t * vm) +{ + clib_error_t * error; + + if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init))) + { + return(error); + } + + /* + * As of now we have only PPC under E2E header. + */ + if (ip6_hbh_config_handler_register(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, + ioam_e2e_config_handler) < 0) + { + return (clib_error_create("Registration of " + "HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE for rewrite failed")); + } + + if (ip6_hbh_add_register_option(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, + sizeof(ioam_e2e_option_t), + ioam_e2e_rewrite_handler) < 0) + { + return (clib_error_create("Registration of " + "HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE for rewrite failed")); + } + + if (ip6_hbh_flow_handler_register(HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE, + ioam_e2e_flow_handler) < 0) + { + return (clib_error_create("Registration of " + "HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE Flow handler failed")); + } + + return (0); +} + +/* + * Init function for the E2E lib. + * ip6_hop_by_hop_ioam_e2e_init gets called during init. + */ +VLIB_INIT_FUNCTION (ioam_e2e_init); diff --git a/src/plugins/ioam/encap/ip6_ioam_e2e.h b/src/plugins/ioam/encap/ip6_ioam_e2e.h new file mode 100644 index 00000000..18f35f80 --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_e2e.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __included_ip6_ioam_e2e_h__ +#define __included_ip6_ioam_e2e_h__ + +#include "ip6_ioam_seqno.h" + +typedef struct ioam_e2e_data_t_ { + u32 flow_ctx; + u32 pad; + ioam_seqno_data seqno_data; +} ioam_e2e_data_t; + +typedef struct { + ioam_e2e_data_t *e2e_data; + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; +} ioam_e2e_main_t; + +extern ioam_e2e_main_t ioam_e2e_main; + +static inline ioam_seqno_data * +ioam_e2ec_get_seqno_data_from_flow_ctx (u32 flow_ctx) +{ + ioam_e2e_data_t *data = NULL; + u32 index; + + index = get_flow_data_from_flow_ctx(flow_ctx, + HBH_OPTION_TYPE_IOAM_EDGE_TO_EDGE); + data = &ioam_e2e_main.e2e_data[index]; + return &(data->seqno_data); +} + +#endif /* __included_ioam_e2e_h__ */ diff --git a/src/plugins/ioam/encap/ip6_ioam_pot.c b/src/plugins/ioam/encap/ip6_ioam_pot.c new file mode 100644 index 00000000..05f42c91 --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_pot.c @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +typedef CLIB_PACKED(struct { + ip6_hop_by_hop_option_t hdr; + u8 pot_type; +#define PROFILE_ID_MASK 0xF + u8 reserved_profile_id; /* 4 bits reserved, 4 bits to carry profile id */ + u64 random; + u64 cumulative; +}) ioam_pot_option_t; + +#define foreach_ip6_hop_by_hop_ioam_pot_stats \ + _(PROCESSED, "Pkts with ip6 hop-by-hop pot options") \ + _(PROFILE_MISS, "Pkts with ip6 hop-by-hop pot options but no profile set") \ + _(PASSED, "Pkts with POT in Policy") \ + _(FAILED, "Pkts with POT out of Policy") + +static char * ip6_hop_by_hop_ioam_pot_stats_strings[] = { +#define _(sym,string) string, + foreach_ip6_hop_by_hop_ioam_pot_stats +#undef _ +}; + +typedef enum { +#define _(sym,str) IP6_IOAM_POT_##sym, + foreach_ip6_hop_by_hop_ioam_pot_stats +#undef _ + IP6_IOAM_POT_N_STATS, +} ip6_ioam_pot_stats_t; + +typedef struct { + /* stats */ + u64 counters[ARRAY_LEN(ip6_hop_by_hop_ioam_pot_stats_strings)]; + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; +} ip6_hop_by_hop_ioam_pot_main_t; + +ip6_hop_by_hop_ioam_pot_main_t ip6_hop_by_hop_ioam_pot_main; + +always_inline void +ip6_ioam_stats_increment_counter (u32 counter_index, u64 increment) +{ + ip6_hop_by_hop_ioam_pot_main_t *hm = &ip6_hop_by_hop_ioam_pot_main; + + hm->counters[counter_index] += increment; +} + + +static u8 * format_ioam_pot (u8 * s, va_list * args) +{ + ioam_pot_option_t * pot0 = va_arg (*args, ioam_pot_option_t *); + u64 random, cumulative; + random = cumulative = 0; + if (pot0) + { + random = clib_net_to_host_u64 (pot0->random); + cumulative = clib_net_to_host_u64 (pot0->cumulative); + } + + s = format (s, "random = 0x%Lx, Cumulative = 0x%Lx, Index = 0x%x", + random, cumulative, pot0 ? pot0->reserved_profile_id : ~0); + return s; +} + +u8 * +ip6_hbh_ioam_proof_of_transit_trace_handler (u8 *s, ip6_hop_by_hop_option_t *opt) +{ + ioam_pot_option_t *pot; + + s = format (s, " POT opt present\n"); + pot = (ioam_pot_option_t *) opt; + s = format (s, " %U\n", format_ioam_pot, pot); + return (s); +} + +int +ip6_hbh_ioam_proof_of_transit_handler (vlib_buffer_t *b, + ip6_header_t *ip, + ip6_hop_by_hop_option_t *opt0) +{ + ioam_pot_option_t * pot0; + u64 random = 0, cumulative = 0; + int rv = 0; + u8 pot_profile_index; + pot_profile *pot_profile = 0, *new_profile = 0; + u8 pot_encap = 0; + + pot0 = (ioam_pot_option_t *) opt0; + pot_encap = (pot0->random == 0); + pot_profile_index = pot_profile_get_active_id(); + pot_profile = pot_profile_get_active(); + if (pot_encap && PREDICT_FALSE(!pot_profile)) + { + ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PROFILE_MISS, 1); + return(-1); + } + if (pot_encap) + { + pot0->reserved_profile_id = + pot_profile_index & PROFILE_ID_MASK; + pot_profile_incr_usage_stats(pot_profile); + } + else + { /* Non encap node */ + if (PREDICT_FALSE(pot0->reserved_profile_id != + pot_profile_index || pot_profile == 0)) + { + /* New profile announced by encap node. */ + new_profile = + pot_profile_find(pot0->reserved_profile_id); + if (PREDICT_FALSE(new_profile == 0 || + new_profile->valid == 0)) + { + ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PROFILE_MISS, 1); + return(-1); + } + else + { + pot_profile_index = pot0->reserved_profile_id; + pot_profile = new_profile; + pot_profile_set_active(pot_profile_index); + pot_profile_reset_usage_stats(pot_profile); + } + } + pot_profile_incr_usage_stats(pot_profile); + } + + if (pot0->random == 0) + { + pot0->random = clib_host_to_net_u64(pot_generate_random(pot_profile)); + pot0->cumulative = 0; + } + random = clib_net_to_host_u64(pot0->random); + cumulative = clib_net_to_host_u64(pot0->cumulative); + pot0->cumulative = clib_host_to_net_u64( + pot_update_cumulative(pot_profile, + cumulative, + random)); + ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PROCESSED, 1); + + return (rv); +} + +int +ip6_hbh_ioam_proof_of_transit_pop_handler (vlib_buffer_t *b, ip6_header_t *ip, + ip6_hop_by_hop_option_t *opt0) +{ + ioam_pot_option_t * pot0; + u64 random = 0; + u64 cumulative = 0; + int rv = 0; + pot_profile *pot_profile = 0; + u8 result = 0; + + pot0 = (ioam_pot_option_t *) opt0; + random = clib_net_to_host_u64(pot0->random); + cumulative = clib_net_to_host_u64(pot0->cumulative); + pot_profile = pot_profile_get_active(); + result = pot_validate (pot_profile, + cumulative, random); + + if (result == 1) + { + ip6_ioam_stats_increment_counter (IP6_IOAM_POT_PASSED, 1); + } + else + { + ip6_ioam_stats_increment_counter (IP6_IOAM_POT_FAILED, 1); + } + return (rv); +} + +int ip6_hop_by_hop_ioam_pot_rewrite_handler (u8 *rewrite_string, u8 *rewrite_size) +{ + ioam_pot_option_t * pot_option; + if (rewrite_string && *rewrite_size == sizeof(ioam_pot_option_t)) + { + pot_option = (ioam_pot_option_t *)rewrite_string; + pot_option->hdr.type = HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT + | HBH_OPTION_TYPE_DATA_CHANGE_ENROUTE; + pot_option->hdr.length = sizeof (ioam_pot_option_t) - + sizeof (ip6_hop_by_hop_option_t); + return(0); + } + return(-1); +} + +static clib_error_t * +ip6_show_ioam_pot_cmd_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + ip6_hop_by_hop_ioam_pot_main_t *hm = &ip6_hop_by_hop_ioam_pot_main; + u8 *s = 0; + int i = 0; + + for ( i = 0; i < IP6_IOAM_POT_N_STATS; i++) + { + s = format(s, " %s - %lu\n", ip6_hop_by_hop_ioam_pot_stats_strings[i], + hm->counters[i]); + } + + vlib_cli_output(vm, "%v", s); + vec_free(s); + return 0; +} + + +VLIB_CLI_COMMAND (ip6_show_ioam_pot_cmd, static) = { + .path = "show ioam pot", + .short_help = "iOAM pot statistics", + .function = ip6_show_ioam_pot_cmd_fn, +}; + + +static clib_error_t * +ip6_hop_by_hop_ioam_pot_init (vlib_main_t * vm) +{ + ip6_hop_by_hop_ioam_pot_main_t * hm = &ip6_hop_by_hop_ioam_pot_main; + clib_error_t * error; + + if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init))) + return(error); + + hm->vlib_main = vm; + hm->vnet_main = vnet_get_main(); + memset(hm->counters, 0, sizeof(hm->counters)); + + if (ip6_hbh_register_option(HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT, ip6_hbh_ioam_proof_of_transit_handler, + ip6_hbh_ioam_proof_of_transit_trace_handler) < 0) + return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT failed")); + + if (ip6_hbh_add_register_option(HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT, + sizeof(ioam_pot_option_t), + ip6_hop_by_hop_ioam_pot_rewrite_handler) < 0) + return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT for rewrite failed")); + + if (ip6_hbh_pop_register_option(HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT, + ip6_hbh_ioam_proof_of_transit_pop_handler) < 0) + return (clib_error_create("registration of HBH_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT POP failed")); + + return (0); +} + +VLIB_INIT_FUNCTION (ip6_hop_by_hop_ioam_pot_init); + + diff --git a/src/plugins/ioam/encap/ip6_ioam_seqno.c b/src/plugins/ioam/encap/ip6_ioam_seqno.c new file mode 100644 index 00000000..0b4d4192 --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_seqno.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "ip6_ioam_seqno.h" +#include "ip6_ioam_e2e.h" + +ioam_seqno_data_main_t ioam_seqno_main; + +void ioam_seqno_init_bitmap (ioam_seqno_data *data) +{ + seqno_bitmap *bitmap = &data->seqno_rx.bitmap; + bitmap->window_size = SEQNO_WINDOW_SIZE; + bitmap->array_size = SEQNO_WINDOW_ARRAY_SIZE; + bitmap->mask = 32 * SEQNO_WINDOW_ARRAY_SIZE - 1; + bitmap->array[0] = 0x00000000;/* pretend we haven seen sequence numbers 0*/ + bitmap->highest = 0; + + data->seq_num = 0; + return ; +} + +/* + * This Routine gets called from IPv6 hop-by-hop option handling. + * Only if we are encap node, then add PPC data. + * On a Transit(MID) node we dont do anything with E2E headers. + * On decap node decap is handled by seperate function. + */ +int +ioam_seqno_encap_handler (vlib_buffer_t *b, ip6_header_t *ip, + ip6_hop_by_hop_option_t *opt) +{ + u32 opaque_index = vnet_buffer(b)->l2_classify.opaque_index; + ioam_e2e_option_t * e2e; + int rv = 0; + ioam_seqno_data *data; + + data = ioam_e2ec_get_seqno_data_from_flow_ctx(opaque_index); + e2e = (ioam_e2e_option_t *) opt; + e2e->e2e_data = clib_host_to_net_u32(++data->seq_num); + + return (rv); +} + +/* + * This Routine gets called on POP/Decap node. + */ +int +ioam_seqno_decap_handler (vlib_buffer_t *b, ip6_header_t *ip, + ip6_hop_by_hop_option_t *opt) +{ + u32 opaque_index = vnet_buffer(b)->l2_classify.opaque_index; + ioam_e2e_option_t * e2e; + int rv = 0; + ioam_seqno_data *data; + + data = ioam_e2ec_get_seqno_data_from_flow_ctx(opaque_index); + e2e = (ioam_e2e_option_t *) opt; + ioam_analyze_seqno(&data->seqno_rx, (u64) clib_net_to_host_u32(e2e->e2e_data)); + + return (rv); +} + +u8 * +show_ioam_seqno_cmd_fn (u8 *s, ioam_seqno_data *seqno_data, u8 enc) +{ + seqno_rx_info *rx; + + s = format(s, "SeqNo Data:\n"); + if (enc) + { + s = format(s, " Current Seq. Number : %llu\n", seqno_data->seq_num); + } + else + { + rx = &seqno_data->seqno_rx; + s = format(s, " Highest Seq. Number : %llu\n", rx->bitmap.highest); + s = format(s, " Packets received : %llu\n", rx->rx_packets); + s = format(s, " Lost packets : %llu\n", rx->lost_packets); + s = format(s, " Reordered packets : %llu\n", rx->reordered_packets); + s = format(s, " Duplicate packets : %llu\n", rx->dup_packets); + } + + format(s, "\n"); + return s; +} diff --git a/src/plugins/ioam/encap/ip6_ioam_seqno.h b/src/plugins/ioam/encap/ip6_ioam_seqno.h new file mode 100644 index 00000000..13a84db0 --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_seqno.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __included_ip6_ioam_seqno_h__ +#define __included_ip6_ioam_seqno_h__ + +#include +#include + +#define SEQ_CHECK_VALUE 0x80000000 /* for seq number wraparound detection */ + +#define SEQNO_WINDOW_SIZE 2048 +#define SEQNO_WINDOW_ARRAY_SIZE 64 + +typedef struct seqno_bitmap_ { + u32 window_size; + u32 array_size; + u32 mask; + u32 pad; + u64 highest; + u64 array[SEQNO_WINDOW_ARRAY_SIZE]; /* Will be alloc to array_size */ +} seqno_bitmap; + +typedef struct seqno_rx_info_ { + u64 rx_packets; + u64 lost_packets; + u64 reordered_packets; + u64 dup_packets; + seqno_bitmap bitmap; +} seqno_rx_info; + +/* This structure is 64-byte aligned */ +typedef struct ioam_seqno_data_ { + union { + u32 seq_num; /* Useful only for encap node */ + seqno_rx_info seqno_rx; + }; +} ioam_seqno_data; + +typedef struct ioam_seqno_data_main_t_ { + ioam_seqno_data *seqno_data; +} ioam_seqno_data_main_t; + +void ioam_seqno_init_bitmap(ioam_seqno_data *data); + +int ioam_seqno_encap_handler(vlib_buffer_t *b, ip6_header_t *ip, + ip6_hop_by_hop_option_t *opt); + +int +ioam_seqno_decap_handler(vlib_buffer_t *b, ip6_header_t *ip, + ip6_hop_by_hop_option_t *opt); + +void ioam_analyze_seqno(seqno_rx_info *ppc_rx, u64 seqno); + +u8 * +show_ioam_seqno_cmd_fn(u8 *s, ioam_seqno_data *seqno_data, u8 enc); + +#endif diff --git a/src/plugins/ioam/encap/ip6_ioam_seqno_analyse.c b/src/plugins/ioam/encap/ip6_ioam_seqno_analyse.c new file mode 100644 index 00000000..4638871c --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_seqno_analyse.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "ip6_ioam_seqno.h" + +static inline void BIT_SET (u64 *p, u32 n) +{ + p[ n>>5 ] |= (1 << (n&31)); +} + +static inline int BIT_TEST (u64 *p, u32 n) +{ + return p[ n>>5 ] & (1 << (n&31)); +} + +static void BIT_CLEAR (u64 *p, u64 start, int num_bits, u32 mask) +{ + int n, t; + int start_index = (start >> 5); + int mask_index = (mask >> 5); + + start_index &= mask_index; + if (start & 0x1f) + { + int start_bit = (start & 0x1f); + + n = (1 << start_bit)-1; + t = start_bit + num_bits; + if (t < 32) + { + n |= ~((1 << t)-1); + p[ start_index ] &= n; + return; + } + p[ start_index ] &= n; + start_index = (start_index + 1) & mask_index; + num_bits -= (32 - start_bit); + } + while (num_bits >= 32) + { + p[ start_index ] = 0; + start_index = (start_index + 1) & mask_index; + num_bits -= 32; + } + n = ~((1 << num_bits) - 1); + p[ start_index ] &= n; +} + +static inline u8 seqno_check_wraparound(u32 a, u32 b) +{ + if ((a != b) && (a > b) && ((a - b) > SEQ_CHECK_VALUE)) + { + return 1; + } + return 0; +} + +/* + * Function to analyze the PPC value recevied. + * - Updates the bitmap with received sequence number + * - counts the received/lost/duplicate/reordered packets + */ +void ioam_analyze_seqno (seqno_rx_info *seqno_rx, u64 seqno) +{ + int diff; + static int peer_dead_count; + seqno_bitmap *bitmap = &seqno_rx->bitmap; + + seqno_rx->rx_packets++; + + if (seqno > bitmap->highest) + { /* new larger sequence number */ + peer_dead_count = 0; + diff = seqno - bitmap->highest; + if (diff < bitmap->window_size) + { + if (diff > 1) + { /* diff==1 is *such* a common case it's a win to optimize it */ + BIT_CLEAR(bitmap->array, bitmap->highest+1, diff-1, bitmap->mask); + seqno_rx->lost_packets += diff -1; + } + } + else + { + seqno_rx->lost_packets += diff -1; + memset( bitmap->array, 0, bitmap->array_size * sizeof(u64) ); + } + BIT_SET(bitmap->array, seqno & bitmap->mask); + bitmap->highest = seqno; + return; + } + + /* we've seen a bigger seq number before */ + diff = bitmap->highest - seqno; + if (diff >= bitmap->window_size) + { + if (seqno_check_wraparound(bitmap->highest, seqno)) + { + memset( bitmap->array, 0, bitmap->array_size * sizeof(u64)); + BIT_SET(bitmap->array, seqno & bitmap->mask); + bitmap->highest = seqno; + return; + } + else + { + peer_dead_count++; + if (peer_dead_count > 25) + { + peer_dead_count = 0; + memset( bitmap->array, 0, bitmap->array_size * sizeof(u64) ); + BIT_SET(bitmap->array, seqno & bitmap->mask); + bitmap->highest = seqno; + } + //ppc_rx->reordered_packets++; + } + return; + } + + if (BIT_TEST(bitmap->array, seqno & bitmap->mask)) + { + seqno_rx->dup_packets++; + return; /* Already seen */ + } + seqno_rx->reordered_packets++; + seqno_rx->lost_packets--; + BIT_SET(bitmap->array, seqno & bitmap->mask); + return; +} diff --git a/src/plugins/ioam/encap/ip6_ioam_trace.c b/src/plugins/ioam/encap/ip6_ioam_trace.c new file mode 100644 index 00000000..e63db6e4 --- /dev/null +++ b/src/plugins/ioam/encap/ip6_ioam_trace.c @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +/* Timestamp precision multipliers for seconds, milliseconds, microseconds + * and nanoseconds respectively. + */ +static f64 trace_tsp_mul[4] = { 1, 1e3, 1e6, 1e9 }; + +typedef union +{ + u64 as_u64; + u32 as_u32[2]; +} time_u64_t; + +/* *INDENT-OFF* */ +typedef CLIB_PACKED(struct { + ip6_hop_by_hop_option_t hdr; + u8 ioam_trace_type; + u8 data_list_elts_left; + u32 elts[0]; /* Variable type. So keep it generic */ +}) ioam_trace_option_t; +/* *INDENT-ON* */ + + +extern ip6_hop_by_hop_ioam_main_t ip6_hop_by_hop_ioam_main; +extern ip6_main_t ip6_main; + +#define foreach_ip6_hop_by_hop_ioam_trace_stats \ + _(PROCESSED, "Pkts with ip6 hop-by-hop trace options") \ + _(PROFILE_MISS, "Pkts with ip6 hop-by-hop trace options but no profile set") \ + _(UPDATED, "Pkts with trace updated") \ + _(FULL, "Pkts with trace options but no space") + +static char *ip6_hop_by_hop_ioam_trace_stats_strings[] = { +#define _(sym,string) string, + foreach_ip6_hop_by_hop_ioam_trace_stats +#undef _ +}; + +typedef enum +{ +#define _(sym,str) IP6_IOAM_TRACE_##sym, + foreach_ip6_hop_by_hop_ioam_trace_stats +#undef _ + IP6_IOAM_TRACE_N_STATS, +} ip6_ioam_trace_stats_t; + + +typedef struct +{ + /* stats */ + u64 counters[ARRAY_LEN (ip6_hop_by_hop_ioam_trace_stats_strings)]; + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; +} ip6_hop_by_hop_ioam_trace_main_t; + +ip6_hop_by_hop_ioam_trace_main_t ip6_hop_by_hop_ioam_trace_main; + +always_inline void +ip6_ioam_trace_stats_increment_counter (u32 counter_index, u64 increment) +{ + ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main; + + hm->counters[counter_index] += increment; +} + + +static u8 * +format_ioam_data_list_element (u8 * s, va_list * args) +{ + u32 *elt = va_arg (*args, u32 *); + u8 *trace_type_p = va_arg (*args, u8 *); + u8 trace_type = *trace_type_p; + + + if (trace_type & BIT_TTL_NODEID) + { + u32 ttl_node_id_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "ttl 0x%x node id 0x%x ", + ttl_node_id_host_byte_order >> 24, + ttl_node_id_host_byte_order & 0x00FFFFFF); + + elt++; + } + + if (trace_type & BIT_ING_INTERFACE && trace_type & BIT_ING_INTERFACE) + { + u32 ingress_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "ingress 0x%x egress 0x%x ", + ingress_host_byte_order >> 16, + ingress_host_byte_order & 0xFFFF); + elt++; + } + + if (trace_type & BIT_TIMESTAMP) + { + u32 ts_in_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "ts 0x%x \n", ts_in_host_byte_order); + elt++; + } + + if (trace_type & BIT_APPDATA) + { + u32 appdata_in_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "app 0x%x ", appdata_in_host_byte_order); + elt++; + } + + return s; +} + + +int +ip6_ioam_trace_get_sizeof_handler (u32 * result) +{ + u16 size = 0; + u8 trace_data_size = 0; + trace_profile *profile = NULL; + + *result = 0; + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); + return (-1); + } + + trace_data_size = fetch_trace_data_size (profile->trace_type); + if (PREDICT_FALSE (trace_data_size == 0)) + return VNET_API_ERROR_INVALID_VALUE; + + if (PREDICT_FALSE (profile->num_elts * trace_data_size > 254)) + return VNET_API_ERROR_INVALID_VALUE; + + size += + sizeof (ioam_trace_option_t) + (profile->num_elts * trace_data_size); + *result = size; + + return 0; +} + + + +int +ip6_hop_by_hop_ioam_trace_rewrite_handler (u8 * rewrite_string, + u8 * rewrite_size) +{ + ioam_trace_option_t *trace_option = NULL; + u8 trace_data_size = 0; + u8 trace_option_elts = 0; + trace_profile *profile = NULL; + + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); + return (-1); + } + + if (PREDICT_FALSE (!rewrite_string)) + return -1; + + trace_option_elts = profile->num_elts; + trace_data_size = fetch_trace_data_size (profile->trace_type); + trace_option = (ioam_trace_option_t *) rewrite_string; + trace_option->hdr.type = HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST | + HBH_OPTION_TYPE_DATA_CHANGE_ENROUTE; + trace_option->hdr.length = 2 /*ioam_trace_type,data_list_elts_left */ + + trace_option_elts * trace_data_size; + trace_option->ioam_trace_type = profile->trace_type & TRACE_TYPE_MASK; + trace_option->data_list_elts_left = trace_option_elts; + *rewrite_size = + sizeof (ioam_trace_option_t) + (trace_option_elts * trace_data_size); + + return 0; +} + + +int +ip6_hbh_ioam_trace_data_list_handler (vlib_buffer_t * b, ip6_header_t * ip, + ip6_hop_by_hop_option_t * opt) +{ + ip6_main_t *im = &ip6_main; + ip_lookup_main_t *lm = &im->lookup_main; + ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main; + u8 elt_index = 0; + ioam_trace_option_t *trace = (ioam_trace_option_t *) opt; + u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; + ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index); + time_u64_t time_u64; + u32 *elt; + int rv = 0; + trace_profile *profile = NULL; + + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); + return (-1); + } + + + time_u64.as_u64 = 0; + + if (PREDICT_TRUE (trace->data_list_elts_left)) + { + trace->data_list_elts_left--; + /* fetch_trace_data_size returns in bytes. Convert it to 4-bytes + * to skip to this node's location. + */ + elt_index = + trace->data_list_elts_left * + fetch_trace_data_size (trace->ioam_trace_type) / 4; + elt = &trace->elts[elt_index]; + if (trace->ioam_trace_type & BIT_TTL_NODEID) + { + *elt = + clib_host_to_net_u32 ((ip->hop_limit << 24) | profile->node_id); + elt++; + } + + if (trace->ioam_trace_type & BIT_ING_INTERFACE) + { + *elt = + (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 | + (adj->rewrite_header.sw_if_index & 0xFFFF); + *elt = clib_host_to_net_u32 (*elt); + elt++; + } + + if (trace->ioam_trace_type & BIT_TIMESTAMP) + { + /* Send least significant 32 bits */ + f64 time_f64 = + (f64) (((f64) hm->unix_time_0) + + (vlib_time_now (hm->vlib_main) - hm->vlib_time_0)); + + time_u64.as_u64 = time_f64 * trace_tsp_mul[profile->trace_tsp]; + *elt = clib_host_to_net_u32 (time_u64.as_u32[0]); + elt++; + } + + if (trace->ioam_trace_type & BIT_APPDATA) + { + /* $$$ set elt0->app_data */ + *elt = clib_host_to_net_u32 (profile->app_data); + elt++; + } + ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_UPDATED, 1); + } + else + { + ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_FULL, 1); + } + return (rv); +} + +u8 * +ip6_hbh_ioam_trace_data_list_trace_handler (u8 * s, + ip6_hop_by_hop_option_t * opt) +{ + ioam_trace_option_t *trace; + u8 trace_data_size_in_words = 0; + u32 *elt; + int elt_index = 0; + + trace = (ioam_trace_option_t *) opt; + s = + format (s, " Trace Type 0x%x , %d elts left\n", trace->ioam_trace_type, + trace->data_list_elts_left); + trace_data_size_in_words = + fetch_trace_data_size (trace->ioam_trace_type) / 4; + elt = &trace->elts[0]; + while ((u8 *) elt < ((u8 *) (&trace->elts[0]) + trace->hdr.length - 2 + /* -2 accounts for ioam_trace_type,elts_left */ )) + { + s = format (s, " [%d] %U\n", elt_index, + format_ioam_data_list_element, + elt, &trace->ioam_trace_type); + elt_index++; + elt += trace_data_size_in_words; + } + return (s); +} + + +static clib_error_t * +ip6_show_ioam_trace_cmd_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main; + u8 *s = 0; + int i = 0; + + for (i = 0; i < IP6_IOAM_TRACE_N_STATS; i++) + { + s = + format (s, " %s - %lu\n", ip6_hop_by_hop_ioam_trace_stats_strings[i], + hm->counters[i]); + } + + vlib_cli_output (vm, "%v", s); + vec_free (s); + return 0; +} + + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (ip6_show_ioam_trace_cmd, static) = { + .path = "show ioam trace", + .short_help = "iOAM trace statistics", + .function = ip6_show_ioam_trace_cmd_fn, +}; +/* *INDENT-ON* */ + + +static clib_error_t * +ip6_hop_by_hop_ioam_trace_init (vlib_main_t * vm) +{ + ip6_hop_by_hop_ioam_trace_main_t *hm = &ip6_hop_by_hop_ioam_trace_main; + clib_error_t *error; + + if ((error = vlib_call_init_function (vm, ip_main_init))) + return (error); + + if ((error = vlib_call_init_function (vm, ip6_lookup_init))) + return error; + + if ((error = vlib_call_init_function (vm, ip6_hop_by_hop_ioam_init))) + return (error); + + hm->vlib_main = vm; + hm->vnet_main = vnet_get_main (); + memset (hm->counters, 0, sizeof (hm->counters)); + + + if (ip6_hbh_register_option + (HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST, + ip6_hbh_ioam_trace_data_list_handler, + ip6_hbh_ioam_trace_data_list_trace_handler) < 0) + return (clib_error_create + ("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST failed")); + + + if (ip6_hbh_add_register_option (HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST, + sizeof (ioam_trace_option_t), + ip6_hop_by_hop_ioam_trace_rewrite_handler) + < 0) + return (clib_error_create + ("registration of HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST for rewrite failed")); + + + return (0); +} + +int +ip6_trace_profile_cleanup (void) +{ + ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main; + + hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] = 0; + + return 0; + +} + + +int +ip6_trace_profile_setup (void) +{ + u32 trace_size = 0; + ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main; + + trace_profile *profile = NULL; + + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + ip6_ioam_trace_stats_increment_counter (IP6_IOAM_TRACE_PROFILE_MISS, 1); + return (-1); + } + + + if (ip6_ioam_trace_get_sizeof_handler (&trace_size) < 0) + return (-1); + + hm->options_size[HBH_OPTION_TYPE_IOAM_TRACE_DATA_LIST] = trace_size; + + return (0); +} + + +VLIB_INIT_FUNCTION (ip6_hop_by_hop_ioam_trace_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h new file mode 100644 index 00000000..fbd86180 --- /dev/null +++ b/src/plugins/ioam/export-common/ioam_export.h @@ -0,0 +1,616 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_ioam_export_h__ +#define __included_ioam_export_h__ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +typedef struct ioam_export_buffer +{ + /* Allocated buffer */ + u32 buffer_index; + u64 touched_at; + u8 records_in_this_buffer; +} ioam_export_buffer_t; + + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + + /* TODO: to support multiple collectors all this has to be grouped and create a vector here */ + u8 *record_header; + u32 sequence_number; + u32 domain_id; + + /* ipfix collector, our ip address */ + ip4_address_t ipfix_collector; + ip4_address_t src_address; + + /* Pool of ioam_export_buffer_t */ + ioam_export_buffer_t *buffer_pool; + /* Vector of per thread ioam_export_buffer_t to buffer pool index */ + u32 *buffer_per_thread; + /* Lock per thread to swap buffers between worker and timer process */ + volatile u32 **lockp; + + /* time scale transform */ + u32 unix_time_0; + f64 vlib_time_0; + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; + u32 ip4_lookup_node_index; + + uword my_hbh_slot; + u32 export_process_node_index; +} ioam_export_main_t; + +ioam_export_main_t ioam_export_main; +ioam_export_main_t vxlan_gpe_ioam_export_main; + +extern vlib_node_registration_t export_node; + +#define DEFAULT_EXPORT_SIZE (3 * CLIB_CACHE_LINE_BYTES) +/* + * Number of records in a buffer + * ~(MTU (1500) - [ip hdr(40) + UDP(8) + ipfix (24)]) / DEFAULT_EXPORT_SIZE + */ +#define DEFAULT_EXPORT_RECORDS 7 + +always_inline ioam_export_buffer_t * +ioam_export_get_my_buffer (ioam_export_main_t * em, u32 thread_id) +{ + + if (vec_len (em->buffer_per_thread) > thread_id) + return (pool_elt_at_index + (em->buffer_pool, em->buffer_per_thread[thread_id])); + return (0); +} + +inline static int +ioam_export_buffer_add_header (ioam_export_main_t * em, vlib_buffer_t * b0) +{ + clib_memcpy (b0->data, em->record_header, vec_len (em->record_header)); + b0->current_data = 0; + b0->current_length = vec_len (em->record_header); + b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + return (1); +} + +inline static int +ioam_export_init_buffer (ioam_export_main_t * em, vlib_main_t * vm, + ioam_export_buffer_t * eb) +{ + vlib_buffer_t *b = 0; + + if (!eb) + return (-1); + /* TODO: Perhaps buffer init from template here */ + if (vlib_buffer_alloc (vm, &(eb->buffer_index), 1) != 1) + return (-2); + eb->records_in_this_buffer = 0; + eb->touched_at = vlib_time_now (vm); + b = vlib_get_buffer (vm, eb->buffer_index); + (void) ioam_export_buffer_add_header (em, b); + vnet_buffer (b)->sw_if_index[VLIB_RX] = 0; + vnet_buffer (b)->sw_if_index[VLIB_TX] = ~0; + return (1); +} + +inline static void +ioam_export_thread_buffer_free (ioam_export_main_t * em) +{ + vlib_main_t *vm = em->vlib_main; + ioam_export_buffer_t *eb = 0; + int i; + for (i = 0; i < vec_len (em->buffer_per_thread); i++) + { + eb = pool_elt_at_index (em->buffer_pool, em->buffer_per_thread[i]); + if (eb) + vlib_buffer_free (vm, &(eb->buffer_index), 1); + } + for (i = 0; i < vec_len (em->lockp); i++) + clib_mem_free ((void *) em->lockp[i]); + vec_free (em->buffer_per_thread); + pool_free (em->buffer_pool); + vec_free (em->lockp); + em->buffer_per_thread = 0; + em->buffer_pool = 0; + em->lockp = 0; +} + +inline static int +ioam_export_thread_buffer_init (ioam_export_main_t * em, vlib_main_t * vm) +{ + int no_of_threads = vec_len (vlib_worker_threads); + int i; + ioam_export_buffer_t *eb = 0; + vlib_node_t *ip4_lookup_node; + + pool_alloc_aligned (em->buffer_pool, + no_of_threads - 1, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (em->buffer_per_thread, + no_of_threads - 1, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (em->lockp, no_of_threads - 1, CLIB_CACHE_LINE_BYTES); + ip4_lookup_node = vlib_get_node_by_name (vm, (u8 *) "ip4-lookup"); + em->ip4_lookup_node_index = ip4_lookup_node->index; + if (!em->buffer_per_thread || !em->buffer_pool || !em->lockp) + { + return (-1); + } + for (i = 0; i < no_of_threads; i++) + { + eb = 0; + pool_get_aligned (em->buffer_pool, eb, CLIB_CACHE_LINE_BYTES); + memset (eb, 0, sizeof (*eb)); + em->buffer_per_thread[i] = eb - em->buffer_pool; + if (ioam_export_init_buffer (em, vm, eb) != 1) + { + ioam_export_thread_buffer_free (em); + return (-2); + } + em->lockp[i] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, + CLIB_CACHE_LINE_BYTES); + memset ((void *) em->lockp[i], 0, CLIB_CACHE_LINE_BYTES); + } + return (1); +} + +#define IPFIX_IOAM_EXPORT_ID 272 + +/* Used to build the rewrite */ +/* data set packet */ +typedef struct +{ + ipfix_message_header_t h; + ipfix_set_header_t s; +} ipfix_data_packet_t; + +typedef struct +{ + ip4_header_t ip4; + udp_header_t udp; + ipfix_data_packet_t ipfix; +} ip4_ipfix_data_packet_t; + + +inline static void +ioam_export_header_cleanup (ioam_export_main_t * em, + ip4_address_t * collector_address, + ip4_address_t * src_address) +{ + vec_free (em->record_header); + em->record_header = 0; +} + +inline static int +ioam_export_header_create (ioam_export_main_t * em, + ip4_address_t * collector_address, + ip4_address_t * src_address) +{ + ip4_header_t *ip; + udp_header_t *udp; + ipfix_message_header_t *h; + ipfix_set_header_t *s; + u8 *rewrite = 0; + ip4_ipfix_data_packet_t *tp; + + + /* allocate rewrite space */ + vec_validate_aligned (rewrite, + sizeof (ip4_ipfix_data_packet_t) - 1, + CLIB_CACHE_LINE_BYTES); + + tp = (ip4_ipfix_data_packet_t *) rewrite; + ip = (ip4_header_t *) & tp->ip4; + udp = (udp_header_t *) (ip + 1); + h = (ipfix_message_header_t *) (udp + 1); + s = (ipfix_set_header_t *) (h + 1); + + ip->ip_version_and_header_length = 0x45; + ip->ttl = 254; + ip->protocol = IP_PROTOCOL_UDP; + ip->src_address.as_u32 = src_address->as_u32; + ip->dst_address.as_u32 = collector_address->as_u32; + udp->src_port = clib_host_to_net_u16 (4939 /* $$FIXME */ ); + udp->dst_port = clib_host_to_net_u16 (4939); + /* FIXUP: UDP length */ + udp->length = clib_host_to_net_u16 (vec_len (rewrite) + + (DEFAULT_EXPORT_RECORDS * + DEFAULT_EXPORT_SIZE) - sizeof (*ip)); + + /* FIXUP: message header export_time */ + /* FIXUP: message header sequence_number */ + h->domain_id = clib_host_to_net_u32 (em->domain_id); + + /*FIXUP: Setid length in octets if records exported are not default */ + s->set_id_length = ipfix_set_id_length (IPFIX_IOAM_EXPORT_ID, + (sizeof (*s) + + (DEFAULT_EXPORT_RECORDS * + DEFAULT_EXPORT_SIZE))); + + /* FIXUP: h version and length length in octets if records exported are not default */ + h->version_length = version_length (sizeof (*h) + + (sizeof (*s) + + (DEFAULT_EXPORT_RECORDS * + DEFAULT_EXPORT_SIZE))); + + /* FIXUP: ip length if records exported are not default */ + /* FIXUP: ip checksum if records exported are not default */ + ip->length = clib_host_to_net_u16 (vec_len (rewrite) + + (DEFAULT_EXPORT_RECORDS * + DEFAULT_EXPORT_SIZE)); + ip->checksum = ip4_header_checksum (ip); + _vec_len (rewrite) = sizeof (ip4_ipfix_data_packet_t); + em->record_header = rewrite; + return (1); +} + +inline static int +ioam_export_send_buffer (ioam_export_main_t * em, vlib_main_t * vm, + ioam_export_buffer_t * eb) +{ + ip4_header_t *ip; + udp_header_t *udp; + ipfix_message_header_t *h; + ipfix_set_header_t *s; + ip4_ipfix_data_packet_t *tp; + vlib_buffer_t *b0; + u16 new_l0, old_l0; + ip_csum_t sum0; + vlib_frame_t *nf = 0; + u32 *to_next; + + b0 = vlib_get_buffer (vm, eb->buffer_index); + tp = vlib_buffer_get_current (b0); + ip = (ip4_header_t *) & tp->ip4; + udp = (udp_header_t *) (ip + 1); + h = (ipfix_message_header_t *) (udp + 1); + s = (ipfix_set_header_t *) (h + 1); + + /* FIXUP: message header export_time */ + h->export_time = clib_host_to_net_u32 ((u32) + (((f64) em->unix_time_0) + + (vlib_time_now (em->vlib_main) - + em->vlib_time_0))); + + /* FIXUP: message header sequence_number */ + h->sequence_number = clib_host_to_net_u32 (em->sequence_number++); + + /* FIXUP: lengths if different from default */ + if (PREDICT_FALSE (eb->records_in_this_buffer != DEFAULT_EXPORT_RECORDS)) + { + s->set_id_length = + ipfix_set_id_length (IPFIX_IOAM_EXPORT_ID /* set_id */ , + b0->current_length - (sizeof (*ip) + + sizeof (*udp) + + sizeof (*h))); + h->version_length = + version_length (b0->current_length - (sizeof (*ip) + sizeof (*udp))); + sum0 = ip->checksum; + old_l0 = ip->length; + new_l0 = clib_host_to_net_u16 ((u16) b0->current_length); + sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t, + length /* changed member */ ); + ip->checksum = ip_csum_fold (sum0); + ip->length = new_l0; + udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip)); + } + + /* Enqueue pkts to ip4-lookup */ + + nf = vlib_get_frame_to_node (vm, em->ip4_lookup_node_index); + nf->n_vectors = 0; + to_next = vlib_frame_vector_args (nf); + nf->n_vectors = 1; + to_next[0] = eb->buffer_index; + vlib_put_frame_to_node (vm, em->ip4_lookup_node_index, nf); + return (1); + +} + +#define EXPORT_TIMEOUT (20.0) +#define THREAD_PERIOD (30.0) +inline static uword +ioam_export_process_common (ioam_export_main_t * em, vlib_main_t * vm, + vlib_node_runtime_t * rt, vlib_frame_t * f, + u32 index) +{ + f64 now; + f64 timeout = 30.0; + uword event_type; + uword *event_data = 0; + int i; + ioam_export_buffer_t *eb = 0, *new_eb = 0; + u32 *vec_buffer_indices = 0; + u32 *vec_buffer_to_be_sent = 0; + u32 *thread_index = 0; + u32 new_pool_index = 0; + + em->export_process_node_index = index; + /* Wait for Godot... */ + vlib_process_wait_for_event_or_clock (vm, 1e9); + event_type = vlib_process_get_events (vm, &event_data); + if (event_type != 1) + clib_warning ("bogus kickoff event received, %d", event_type); + vec_reset_length (event_data); + + while (1) + { + vlib_process_wait_for_event_or_clock (vm, timeout); + event_type = vlib_process_get_events (vm, &event_data); + switch (event_type) + { + case 2: /* Stop and Wait for kickoff again */ + timeout = 1e9; + break; + case 1: /* kickoff : Check for unsent buffers */ + timeout = THREAD_PERIOD; + break; + case ~0: /* timeout */ + break; + } + vec_reset_length (event_data); + now = vlib_time_now (vm); + /* + * Create buffers for threads that are not active enough + * to send out the export records + */ + for (i = 0; i < vec_len (em->buffer_per_thread); i++) + { + /* If the worker thread is processing export records ignore further checks */ + if (*em->lockp[i] == 1) + continue; + eb = pool_elt_at_index (em->buffer_pool, em->buffer_per_thread[i]); + if (eb->records_in_this_buffer > 0 + && now > (eb->touched_at + EXPORT_TIMEOUT)) + { + pool_get_aligned (em->buffer_pool, new_eb, + CLIB_CACHE_LINE_BYTES); + memset (new_eb, 0, sizeof (*new_eb)); + if (ioam_export_init_buffer (em, vm, new_eb) == 1) + { + new_pool_index = new_eb - em->buffer_pool; + vec_add (vec_buffer_indices, &new_pool_index, 1); + vec_add (vec_buffer_to_be_sent, &em->buffer_per_thread[i], + 1); + vec_add (thread_index, &i, 1); + } + else + { + pool_put (em->buffer_pool, new_eb); + /*Give up */ + goto CLEANUP; + } + } + } + if (vec_len (thread_index) != 0) + { + /* + * Now swap the buffers out + */ + for (i = 0; i < vec_len (thread_index); i++) + { + while (__sync_lock_test_and_set (em->lockp[thread_index[i]], 1)) + ; + em->buffer_per_thread[thread_index[i]] = + vec_pop (vec_buffer_indices); + *em->lockp[thread_index[i]] = 0; + } + + /* Send the buffers */ + for (i = 0; i < vec_len (vec_buffer_to_be_sent); i++) + { + eb = + pool_elt_at_index (em->buffer_pool, vec_buffer_to_be_sent[i]); + ioam_export_send_buffer (em, vm, eb); + pool_put (em->buffer_pool, eb); + } + } + + CLEANUP: + /* Free any leftover/unused buffers and everything that was allocated */ + for (i = 0; i < vec_len (vec_buffer_indices); i++) + { + new_eb = pool_elt_at_index (em->buffer_pool, vec_buffer_indices[i]); + vlib_buffer_free (vm, &new_eb->buffer_index, 1); + pool_put (em->buffer_pool, new_eb); + } + vec_free (vec_buffer_indices); + vec_free (vec_buffer_to_be_sent); + vec_free (thread_index); + } + return 0; /* not so much */ +} + +#define ioam_export_node_common(EM, VM, N, F, HTYPE, L, V, NEXT) \ +do { \ + u32 n_left_from, *from, *to_next; \ + export_next_t next_index; \ + u32 pkts_recorded = 0; \ + ioam_export_buffer_t *my_buf = 0; \ + vlib_buffer_t *eb0 = 0; \ + u32 ebi0 = 0; \ + from = vlib_frame_vector_args (F); \ + n_left_from = (F)->n_vectors; \ + next_index = (N)->cached_next_index; \ + while (__sync_lock_test_and_set ((EM)->lockp[(VM)->cpu_index], 1)); \ + my_buf = ioam_export_get_my_buffer (EM, (VM)->cpu_index); \ + my_buf->touched_at = vlib_time_now (VM); \ + while (n_left_from > 0) \ + { \ + u32 n_left_to_next; \ + vlib_get_next_frame (VM, N, next_index, to_next, n_left_to_next); \ + while (n_left_from >= 4 && n_left_to_next >= 2) \ + { \ + u32 next0 = NEXT; \ + u32 next1 = NEXT; \ + u32 bi0, bi1; \ + HTYPE *ip0, *ip1; \ + vlib_buffer_t *p0, *p1; \ + u32 ip_len0, ip_len1; \ + { \ + vlib_buffer_t *p2, *p3; \ + p2 = vlib_get_buffer (VM, from[2]); \ + p3 = vlib_get_buffer (VM, from[3]); \ + vlib_prefetch_buffer_header (p2, LOAD); \ + vlib_prefetch_buffer_header (p3, LOAD); \ + CLIB_PREFETCH (p2->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD); \ + CLIB_PREFETCH (p3->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD); \ + } \ + to_next[0] = bi0 = from[0]; \ + to_next[1] = bi1 = from[1]; \ + from += 2; \ + to_next += 2; \ + n_left_from -= 2; \ + n_left_to_next -= 2; \ + p0 = vlib_get_buffer (VM, bi0); \ + p1 = vlib_get_buffer (VM, bi1); \ + ip0 = vlib_buffer_get_current (p0); \ + ip1 = vlib_buffer_get_current (p1); \ + ip_len0 = \ + clib_net_to_host_u16 (ip0->L) + sizeof (HTYPE); \ + ip_len1 = \ + clib_net_to_host_u16 (ip1->L) + sizeof (HTYPE); \ + ebi0 = my_buf->buffer_index; \ + eb0 = vlib_get_buffer (VM, ebi0); \ + if (PREDICT_FALSE (eb0 == 0)) \ + goto NO_BUFFER1; \ + ip_len0 = \ + ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0; \ + ip_len1 = \ + ip_len1 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len1; \ + copy3cachelines (eb0->data + eb0->current_length, ip0, ip_len0); \ + eb0->current_length += DEFAULT_EXPORT_SIZE; \ + my_buf->records_in_this_buffer++; \ + if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS) \ + { \ + ioam_export_send_buffer (EM, VM, my_buf); \ + ioam_export_init_buffer (EM, VM, my_buf); \ + } \ + ebi0 = my_buf->buffer_index; \ + eb0 = vlib_get_buffer (VM, ebi0); \ + if (PREDICT_FALSE (eb0 == 0)) \ + goto NO_BUFFER1; \ + copy3cachelines (eb0->data + eb0->current_length, ip1, ip_len1); \ + eb0->current_length += DEFAULT_EXPORT_SIZE; \ + my_buf->records_in_this_buffer++; \ + if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS) \ + { \ + ioam_export_send_buffer (EM, VM, my_buf); \ + ioam_export_init_buffer (EM, VM, my_buf); \ + } \ + pkts_recorded += 2; \ + if (PREDICT_FALSE (((node)->flags & VLIB_NODE_FLAG_TRACE))) \ + { \ + if (p0->flags & VLIB_BUFFER_IS_TRACED) \ + { \ + export_trace_t *t = \ + vlib_add_trace (VM, node, p0, sizeof (*t)); \ + t->flow_label = \ + clib_net_to_host_u32 (ip0->V); \ + t->next_index = next0; \ + } \ + if (p1->flags & VLIB_BUFFER_IS_TRACED) \ + { \ + export_trace_t *t = \ + vlib_add_trace (VM, N, p1, sizeof (*t)); \ + t->flow_label = \ + clib_net_to_host_u32 (ip1->V); \ + t->next_index = next1; \ + } \ + } \ + NO_BUFFER1: \ + vlib_validate_buffer_enqueue_x2 (VM, N, next_index, \ + to_next, n_left_to_next, \ + bi0, bi1, next0, next1); \ + } \ + while (n_left_from > 0 && n_left_to_next > 0) \ + { \ + u32 bi0; \ + vlib_buffer_t *p0; \ + u32 next0 = NEXT; \ + HTYPE *ip0; \ + u32 ip_len0; \ + bi0 = from[0]; \ + to_next[0] = bi0; \ + from += 1; \ + to_next += 1; \ + n_left_from -= 1; \ + n_left_to_next -= 1; \ + p0 = vlib_get_buffer (VM, bi0); \ + ip0 = vlib_buffer_get_current (p0); \ + ip_len0 = \ + clib_net_to_host_u16 (ip0->L) + sizeof (HTYPE); \ + ebi0 = my_buf->buffer_index; \ + eb0 = vlib_get_buffer (VM, ebi0); \ + if (PREDICT_FALSE (eb0 == 0)) \ + goto NO_BUFFER; \ + ip_len0 = \ + ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0; \ + copy3cachelines (eb0->data + eb0->current_length, ip0, ip_len0); \ + eb0->current_length += DEFAULT_EXPORT_SIZE; \ + my_buf->records_in_this_buffer++; \ + if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS) \ + { \ + ioam_export_send_buffer (EM, VM, my_buf); \ + ioam_export_init_buffer (EM, VM, my_buf); \ + } \ + if (PREDICT_FALSE (((N)->flags & VLIB_NODE_FLAG_TRACE) \ + && (p0->flags & VLIB_BUFFER_IS_TRACED))) \ + { \ + export_trace_t *t = vlib_add_trace (VM, (N), p0, sizeof (*t)); \ + t->flow_label = \ + clib_net_to_host_u32 (ip0->V); \ + t->next_index = next0; \ + } \ + pkts_recorded += 1; \ + NO_BUFFER: \ + vlib_validate_buffer_enqueue_x1 (VM, N, next_index, \ + to_next, n_left_to_next, \ + bi0, next0); \ + } \ + vlib_put_next_frame (VM, N, next_index, n_left_to_next); \ + } \ + vlib_node_increment_counter (VM, export_node.index, \ + EXPORT_ERROR_RECORDED, pkts_recorded); \ + *(EM)->lockp[(VM)->cpu_index] = 0; \ +} while(0) + +#endif /* __included_ioam_export_h__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api new file mode 100644 index 00000000..7b17c3f7 --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api @@ -0,0 +1,42 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Define a simple binary API to control the feature */ + +define vxlan_gpe_ioam_export_enable_disable { + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + + /* Enable / disable the feature */ + u8 is_disable; + + /* Collector ip address */ + u8 collector_address[4]; + u8 src_address[4]; + + /* Src ip address */ +}; + +define vxlan_gpe_ioam_export_enable_disable_reply { + /* From the request */ + u32 context; + + /* Return value, zero means all OK */ + i32 retval; +}; \ No newline at end of file diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c new file mode 100644 index 00000000..bab8d977 --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.c @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * vxlan_gpe_ioam_export.c - ioam export API / debug CLI handling + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + + +/* List of message types that this plugin understands */ + + +#define foreach_vxlan_gpe_ioam_export_plugin_api_msg \ +_(VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE, vxlan_gpe_ioam_export_enable_disable) + +extern void vxlan_gpe_set_next_override (uword next); +/* Action function shared between message handler and debug CLI */ +int +vxlan_gpe_ioam_export_enable_disable (ioam_export_main_t * em, + u8 is_disable, + ip4_address_t * collector_address, + ip4_address_t * src_address) +{ + vlib_main_t *vm = em->vlib_main; + u32 node_index = export_node.index; + vlib_node_t *vxlan_gpe_decap_ioam_node = NULL; + + if (is_disable == 0) + { + if (em->my_hbh_slot == ~0) + { + /* Hook this export node to vxlan-gpe-decap-ioam-v4 */ + vxlan_gpe_decap_ioam_node = + vlib_get_node_by_name (vm, (u8 *) "vxlan-gpe-decap-ioam-v4"); + if (!vxlan_gpe_decap_ioam_node) + { + /* node does not exist give up */ + return (-1); + } + em->my_hbh_slot = + vlib_node_add_next (vm, vxlan_gpe_decap_ioam_node->index, + node_index); + } + if (1 == ioam_export_header_create (em, collector_address, src_address)) + { + ioam_export_thread_buffer_init (em, vm); + vxlan_gpe_set_next_override (em->my_hbh_slot); + /* Turn on the export buffer check process */ + vlib_process_signal_event (vm, em->export_process_node_index, 1, 0); + + } + else + { + return (-2); + } + } + else + { + vxlan_gpe_set_next_override (VXLAN_GPE_DECAP_IOAM_V4_NEXT_POP); + ioam_export_header_cleanup (em, collector_address, src_address); + ioam_export_thread_buffer_free (em); + /* Turn off the export buffer check process */ + vlib_process_signal_event (vm, em->export_process_node_index, 2, 0); + + } + + return 0; +} + +/* API message handler */ +static void vl_api_vxlan_gpe_ioam_export_enable_disable_t_handler + (vl_api_vxlan_gpe_ioam_export_enable_disable_t * mp) +{ + vl_api_vxlan_gpe_ioam_export_enable_disable_reply_t *rmp; + ioam_export_main_t *sm = &vxlan_gpe_ioam_export_main; + int rv; + + rv = vxlan_gpe_ioam_export_enable_disable (sm, (int) (mp->is_disable), + (ip4_address_t *) + mp->collector_address, + (ip4_address_t *) + mp->src_address); + + REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE_REPLY); +} /* API message handler */ + + + +/* Set up the API message handling tables */ +static clib_error_t * +vxlan_gpe_ioam_export_plugin_api_hookup (vlib_main_t * vm) +{ + ioam_export_main_t *sm = &vxlan_gpe_ioam_export_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vxlan_gpe_ioam_export_plugin_api_msg; +#undef _ + + return 0; +} + + +static clib_error_t * +set_vxlan_gpe_ioam_export_ipfix_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + ioam_export_main_t *em = &vxlan_gpe_ioam_export_main; + ip4_address_t collector, src; + u8 is_disable = 0; + + collector.as_u32 = 0; + src.as_u32 = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "collector %U", unformat_ip4_address, &collector)) + ; + else if (unformat (input, "src %U", unformat_ip4_address, &src)) + ; + else if (unformat (input, "disable")) + is_disable = 1; + else + break; + } + + if (collector.as_u32 == 0) + return clib_error_return (0, "collector address required"); + + if (src.as_u32 == 0) + return clib_error_return (0, "src address required"); + + em->ipfix_collector.as_u32 = collector.as_u32; + em->src_address.as_u32 = src.as_u32; + + vlib_cli_output (vm, "Collector %U, src address %U", + format_ip4_address, &em->ipfix_collector, + format_ip4_address, &em->src_address); + + /* Turn on the export timer process */ + // vlib_process_signal_event (vm, flow_report_process_node.index, + //1, 0); + if (0 != + vxlan_gpe_ioam_export_enable_disable (em, is_disable, &collector, &src)) + { + return clib_error_return (0, "Unable to set ioam vxlan-gpe export"); + } + + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (set_vxlan_gpe_ioam_ipfix_command, static) = +{ +.path = "set vxlan-gpe-ioam export ipfix", +.short_help = "set vxlan-gpe-ioam export ipfix collector src ", +.function = set_vxlan_gpe_ioam_export_ipfix_command_fn, +}; +/* *INDENT-ON* */ + + +static clib_error_t * +vxlan_gpe_ioam_export_init (vlib_main_t * vm) +{ + ioam_export_main_t *em = &vxlan_gpe_ioam_export_main; + clib_error_t *error = 0; + u8 *name; + + name = format (0, "vxlan_gpe_ioam_export_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + em->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + em->unix_time_0 = (u32) time (0); /* Store starting time */ + em->vlib_time_0 = vlib_time_now (vm); + + error = vxlan_gpe_ioam_export_plugin_api_hookup (vm); + em->my_hbh_slot = ~0; + em->vlib_main = vm; + em->vnet_main = vnet_get_main (); + vec_free (name); + + return error; +} + +VLIB_INIT_FUNCTION (vxlan_gpe_ioam_export_init); + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h new file mode 100644 index 00000000..6d93f093 --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_all_api_h.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h new file mode 100644 index 00000000..cc5698de --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_vxlan_gpe_ioam_export_msg_enum_h +#define included_vxlan_gpe_ioam_export_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_vxlan_gpe_ioam_export_msg_enum_h */ diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c new file mode 100644 index 00000000..494263d9 --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_test.c @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * vxlan_gpe_ioam_export_test.c - test harness plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include + + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} export_test_main_t; + +export_test_main_t export_test_main; + +#define foreach_standard_reply_retval_handler \ +_(vxlan_gpe_ioam_export_enable_disable_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = export_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE_REPLY, vxlan_gpe_ioam_export_enable_disable_reply) + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int +api_vxlan_gpe_ioam_export_enable_disable (vat_main_t * vam) +{ + export_test_main_t *sm = &export_test_main; + unformat_input_t *i = vam->input; + f64 timeout; + int is_disable = 0; + vl_api_vxlan_gpe_ioam_export_enable_disable_t *mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "disable")) + is_disable = 1; + else + break; + } + + /* Construct the API message */ + M (VXLAN_GPE_IOAM_EXPORT_ENABLE_DISABLE, + vxlan_gpe_ioam_export_enable_disable); + mp->is_disable = is_disable; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(vxlan_gpe_ioam_export_enable_disable, " [disable]") + +void +vat_api_hookup (vat_main_t * vam) +{ + export_test_main_t *sm = &export_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * +vat_plugin_register (vat_main_t * vam) +{ + export_test_main_t *sm = &export_test_main; + u8 *name; + + sm->vat_main = vam; + + name = format (0, "vxlan_gpe_ioam_export_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~ 0) + vat_api_hookup (vam); + + vec_free (name); + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c new file mode 100644 index 00000000..58508ebf --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export_thread.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * ioam_export_thread.c + */ +#include +#include +#include + +static vlib_node_registration_t vxlan_gpe_ioam_export_process_node; + +static uword +vxlan_gpe_ioam_export_process (vlib_main_t * vm, + vlib_node_runtime_t * rt, vlib_frame_t * f) +{ + return (ioam_export_process_common (&vxlan_gpe_ioam_export_main, + vm, rt, f, + vxlan_gpe_ioam_export_process_node.index)); +} + + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (vxlan_gpe_ioam_export_process_node, static) = +{ + .function = vxlan_gpe_ioam_export_process, + .type = VLIB_NODE_TYPE_PROCESS, + .name = "vxlan-gpe-ioam-export-process", +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_node.c b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_node.c new file mode 100644 index 00000000..722c2b06 --- /dev/null +++ b/src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_node.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct +{ + u32 next_index; + u32 flow_label; +} export_trace_t; + +/* packet trace format function */ +static u8 * +format_export_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + export_trace_t *t = va_arg (*args, export_trace_t *); + + s = format (s, "EXPORT: flow_label %d, next index %d", + t->flow_label, t->next_index); + return s; +} + +vlib_node_registration_t export_node; + +#define foreach_export_error \ +_(RECORDED, "Packets recorded for export") + +typedef enum +{ +#define _(sym,str) EXPORT_ERROR_##sym, + foreach_export_error +#undef _ + EXPORT_N_ERROR, +} export_error_t; + +static char *export_error_strings[] = { +#define _(sym,string) string, + foreach_export_error +#undef _ +}; + +typedef enum +{ + EXPORT_NEXT_VXLAN_GPE_INPUT, + EXPORT_N_NEXT, +} export_next_t; + +always_inline void +copy3cachelines (void *dst, const void *src, size_t n) +{ +#if 0 + if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) + { + /* Copy only the first 1/2 cache lines whatever is available */ + if (n >= 64) + clib_mov64 ((u8 *) dst, (const u8 *) src); + if (n >= 128) + clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); + return; + } + clib_mov64 ((u8 *) dst, (const u8 *) src); + clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); + clib_mov64 ((u8 *) dst + 128, (const u8 *) src + 128); +#endif +#if 1 + + u64 *copy_dst, *copy_src; + int i; + copy_dst = (u64 *) dst; + copy_src = (u64 *) src; + if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) + { + for (i = 0; i < n / 64; i++) + { + copy_dst[0] = copy_src[0]; + copy_dst[1] = copy_src[1]; + copy_dst[2] = copy_src[2]; + copy_dst[3] = copy_src[3]; + copy_dst[4] = copy_src[4]; + copy_dst[5] = copy_src[5]; + copy_dst[6] = copy_src[6]; + copy_dst[7] = copy_src[7]; + copy_dst += 8; + copy_src += 8; + } + return; + } + for (i = 0; i < 3; i++) + { + copy_dst[0] = copy_src[0]; + copy_dst[1] = copy_src[1]; + copy_dst[2] = copy_src[2]; + copy_dst[3] = copy_src[3]; + copy_dst[4] = copy_src[4]; + copy_dst[5] = copy_src[5]; + copy_dst[6] = copy_src[6]; + copy_dst[7] = copy_src[7]; + copy_dst += 8; + copy_src += 8; + } +#endif +} + + +static uword +vxlan_gpe_export_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + ioam_export_main_t *em = &vxlan_gpe_ioam_export_main; + ioam_export_node_common (em, vm, node, frame, ip4_header_t, length, + ip_version_and_header_length, + EXPORT_NEXT_VXLAN_GPE_INPUT); + return frame->n_vectors; +} + +/* + * Node for VXLAN-GPE export + */ +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (export_node) = +{ + .function = vxlan_gpe_export_node_fn, + .name = "vxlan-gpe-ioam-export", + .vector_size = sizeof (u32), + .format_trace = format_export_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN (export_error_strings), + .error_strings = export_error_strings, + .n_next_nodes = EXPORT_N_NEXT, + /* edit / add dispositions here */ + .next_nodes = + {[EXPORT_NEXT_VXLAN_GPE_INPUT] = "vxlan-gpe-pop-ioam-v4"}, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export/ioam_export.api b/src/plugins/ioam/export/ioam_export.api new file mode 100644 index 00000000..f22d9fc8 --- /dev/null +++ b/src/plugins/ioam/export/ioam_export.api @@ -0,0 +1,42 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Define a simple binary API to control the feature */ + +define ioam_export_ip6_enable_disable { + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + + /* Enable / disable the feature */ + u8 is_disable; + + /* Collector ip address */ + u8 collector_address[4]; + u8 src_address[4]; + + /* Src ip address */ +}; + +define ioam_export_ip6_enable_disable_reply { + /* From the request */ + u32 context; + + /* Return value, zero means all OK */ + i32 retval; +}; diff --git a/src/plugins/ioam/export/ioam_export.c b/src/plugins/ioam/export/ioam_export.c new file mode 100644 index 00000000..b122e445 --- /dev/null +++ b/src/plugins/ioam/export/ioam_export.c @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * ioam_export.c - ioam export API / debug CLI handling + *------------------------------------------------------------------ + */ + +#include +#include +#include + +#include +#include +#include +#include + + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + + +/* List of message types that this plugin understands */ + +#define foreach_ioam_export_plugin_api_msg \ +_(IOAM_EXPORT_IP6_ENABLE_DISABLE, ioam_export_ip6_enable_disable) + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + ioam_export_main_t *em = &ioam_export_main; + clib_error_t *error = 0; + + em->vlib_main = vm; + em->vnet_main = h->vnet_main; + + return error; +} + +/* Action function shared between message handler and debug CLI */ + +int +ioam_export_ip6_enable_disable (ioam_export_main_t * em, + u8 is_disable, + ip4_address_t * collector_address, + ip4_address_t * src_address) +{ + vlib_main_t *vm = em->vlib_main; + + if (is_disable == 0) + { + if (1 == ioam_export_header_create (em, collector_address, src_address)) + { + ioam_export_thread_buffer_init (em, vm); + ip6_hbh_set_next_override (em->my_hbh_slot); + /* Turn on the export buffer check process */ + vlib_process_signal_event (vm, em->export_process_node_index, 1, 0); + + } + else + { + return (-2); + } + } + else + { + ip6_hbh_set_next_override (IP6_LOOKUP_NEXT_POP_HOP_BY_HOP); + ioam_export_header_cleanup (em, collector_address, src_address); + ioam_export_thread_buffer_free (em); + /* Turn off the export buffer check process */ + vlib_process_signal_event (vm, em->export_process_node_index, 2, 0); + + } + + return 0; +} + +/* API message handler */ +static void vl_api_ioam_export_ip6_enable_disable_t_handler + (vl_api_ioam_export_ip6_enable_disable_t * mp) +{ + vl_api_ioam_export_ip6_enable_disable_reply_t *rmp; + ioam_export_main_t *sm = &ioam_export_main; + int rv; + + rv = ioam_export_ip6_enable_disable (sm, (int) (mp->is_disable), + (ip4_address_t *) + mp->collector_address, + (ip4_address_t *) mp->src_address); + + REPLY_MACRO (VL_API_IOAM_EXPORT_IP6_ENABLE_DISABLE_REPLY); +} + +/* Set up the API message handling tables */ +static clib_error_t * +ioam_export_plugin_api_hookup (vlib_main_t * vm) +{ + ioam_export_main_t *sm = &ioam_export_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_ioam_export_plugin_api_msg; +#undef _ + + return 0; +} + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (ioam_export_main_t * sm, api_main_t * am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); + foreach_vl_msg_name_crc_ioam_export; +#undef _ +} + +static clib_error_t * +set_ioam_export_ipfix_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + ioam_export_main_t *em = &ioam_export_main; + ip4_address_t collector, src; + u8 is_disable = 0; + + collector.as_u32 = 0; + src.as_u32 = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "collector %U", unformat_ip4_address, &collector)) + ; + else if (unformat (input, "src %U", unformat_ip4_address, &src)) + ; + else if (unformat (input, "disable")) + is_disable = 1; + else + break; + } + + if (collector.as_u32 == 0) + return clib_error_return (0, "collector address required"); + + if (src.as_u32 == 0) + return clib_error_return (0, "src address required"); + + em->ipfix_collector.as_u32 = collector.as_u32; + em->src_address.as_u32 = src.as_u32; + + vlib_cli_output (vm, "Collector %U, src address %U", + format_ip4_address, &em->ipfix_collector, + format_ip4_address, &em->src_address); + + /* Turn on the export timer process */ + // vlib_process_signal_event (vm, flow_report_process_node.index, + //1, 0); + ioam_export_ip6_enable_disable (em, is_disable, &collector, &src); + + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (set_ipfix_command, static) = +{ +.path = "set ioam export ipfix",.short_help = + "set ioam export ipfix collector src ",. + function = set_ioam_export_ipfix_command_fn,}; +/* *INDENT-ON* */ + + +static clib_error_t * +ioam_export_init (vlib_main_t * vm) +{ + ioam_export_main_t *em = &ioam_export_main; + clib_error_t *error = 0; + u8 *name; + u32 node_index = export_node.index; + vlib_node_t *ip6_hbyh_node = NULL; + + name = format (0, "ioam_export_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + em->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + em->unix_time_0 = (u32) time (0); /* Store starting time */ + em->vlib_time_0 = vlib_time_now (vm); + + error = ioam_export_plugin_api_hookup (vm); + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (em, &api_main); + + /* Hook this export node to ip6-hop-by-hop */ + ip6_hbyh_node = vlib_get_node_by_name (vm, (u8 *) "ip6-hop-by-hop"); + em->my_hbh_slot = vlib_node_add_next (vm, ip6_hbyh_node->index, node_index); + vec_free (name); + + return error; +} + +VLIB_INIT_FUNCTION (ioam_export_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/export/ioam_export_all_api_h.h b/src/plugins/ioam/export/ioam_export_all_api_h.h new file mode 100644 index 00000000..bc4368f2 --- /dev/null +++ b/src/plugins/ioam/export/ioam_export_all_api_h.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/plugins/ioam/export/ioam_export_msg_enum.h b/src/plugins/ioam/export/ioam_export_msg_enum.h new file mode 100644 index 00000000..c2de7988 --- /dev/null +++ b/src/plugins/ioam/export/ioam_export_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_ioam_export_msg_enum_h +#define included_ioam_export_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_ioam_export_msg_enum_h */ diff --git a/src/plugins/ioam/export/ioam_export_test.c b/src/plugins/ioam/export/ioam_export_test.c new file mode 100644 index 00000000..f991fc0c --- /dev/null +++ b/src/plugins/ioam/export/ioam_export_test.c @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * ioam_export_test.c - test harness plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include + + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} export_test_main_t; + +export_test_main_t export_test_main; + +#define foreach_standard_reply_retval_handler \ +_(ioam_export_ip6_enable_disable_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = export_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(IOAM_EXPORT_IP6_ENABLE_DISABLE_REPLY, ioam_export_ip6_enable_disable_reply) + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int +api_ioam_export_ip6_enable_disable (vat_main_t * vam) +{ + export_test_main_t *sm = &export_test_main; + unformat_input_t *i = vam->input; + f64 timeout; + int is_disable = 0; + vl_api_ioam_export_ip6_enable_disable_t *mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "disable")) + is_disable = 1; + else + break; + } + + /* Construct the API message */ + M (IOAM_EXPORT_IP6_ENABLE_DISABLE, ioam_export_ip6_enable_disable); + mp->is_disable = is_disable; + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(ioam_export_ip6_enable_disable, " [disable]") + +void +vat_api_hookup (vat_main_t * vam) +{ + export_test_main_t *sm = &export_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * +vat_plugin_register (vat_main_t * vam) +{ + export_test_main_t *sm = &export_test_main; + u8 *name; + + sm->vat_main = vam; + + name = format (0, "ioam_export_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~ 0) + vat_api_hookup (vam); + + vec_free (name); + + return 0; +} diff --git a/src/plugins/ioam/export/ioam_export_thread.c b/src/plugins/ioam/export/ioam_export_thread.c new file mode 100644 index 00000000..d2eb2009 --- /dev/null +++ b/src/plugins/ioam/export/ioam_export_thread.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * ioam_export_thread.c + */ +#include +#include +#include + +static vlib_node_registration_t ioam_export_process_node; + +static uword +ioam_export_process (vlib_main_t * vm, + vlib_node_runtime_t * rt, vlib_frame_t * f) +{ + return (ioam_export_process_common(&ioam_export_main, + vm, rt, f, + ioam_export_process_node.index)); +} + +VLIB_REGISTER_NODE (ioam_export_process_node, static) = +{ + .function = ioam_export_process, + .type = VLIB_NODE_TYPE_PROCESS, + .name = "ioam-export-process", +}; diff --git a/src/plugins/ioam/export/node.c b/src/plugins/ioam/export/node.c new file mode 100644 index 00000000..19f143df --- /dev/null +++ b/src/plugins/ioam/export/node.c @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +typedef struct +{ + u32 next_index; + u32 flow_label; +} export_trace_t; + +/* packet trace format function */ +static u8 * +format_export_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + export_trace_t *t = va_arg (*args, export_trace_t *); + + s = format (s, "EXPORT: flow_label %d, next index %d", + t->flow_label, t->next_index); + return s; +} + +vlib_node_registration_t export_node; + +#define foreach_export_error \ +_(RECORDED, "Packets recorded for export") + +typedef enum +{ +#define _(sym,str) EXPORT_ERROR_##sym, + foreach_export_error +#undef _ + EXPORT_N_ERROR, +} export_error_t; + +static char *export_error_strings[] = { +#define _(sym,string) string, + foreach_export_error +#undef _ +}; + +typedef enum +{ + EXPORT_NEXT_POP_HBYH, + EXPORT_N_NEXT, +} export_next_t; + +always_inline void +copy3cachelines (void *dst, const void *src, size_t n) +{ +#if 0 + if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) + { + /* Copy only the first 1/2 cache lines whatever is available */ + if (n >= 64) + clib_mov64 ((u8 *) dst, (const u8 *) src); + if (n >= 128) + clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); + return; + } + clib_mov64 ((u8 *) dst, (const u8 *) src); + clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64); + clib_mov64 ((u8 *) dst + 128, (const u8 *) src + 128); +#endif +#if 1 + + u64 *copy_dst, *copy_src; + int i; + copy_dst = (u64 *) dst; + copy_src = (u64 *) src; + if (PREDICT_FALSE (n < DEFAULT_EXPORT_SIZE)) + { + for (i = 0; i < n / 64; i++) + { + copy_dst[0] = copy_src[0]; + copy_dst[1] = copy_src[1]; + copy_dst[2] = copy_src[2]; + copy_dst[3] = copy_src[3]; + copy_dst[4] = copy_src[4]; + copy_dst[5] = copy_src[5]; + copy_dst[6] = copy_src[6]; + copy_dst[7] = copy_src[7]; + copy_dst += 8; + copy_src += 8; + } + return; + } + for (i = 0; i < 3; i++) + { + copy_dst[0] = copy_src[0]; + copy_dst[1] = copy_src[1]; + copy_dst[2] = copy_src[2]; + copy_dst[3] = copy_src[3]; + copy_dst[4] = copy_src[4]; + copy_dst[5] = copy_src[5]; + copy_dst[6] = copy_src[6]; + copy_dst[7] = copy_src[7]; + copy_dst += 8; + copy_src += 8; + } +#endif +} + +static uword +ip6_export_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + ioam_export_main_t *em = &ioam_export_main; + ioam_export_node_common(em, vm, node, frame, ip6_header_t, payload_length, + ip_version_traffic_class_and_flow_label, + EXPORT_NEXT_POP_HBYH); + return frame->n_vectors; +} + +/* + * Node for IP6 export + */ +VLIB_REGISTER_NODE (export_node) = +{ + .function = ip6_export_node_fn, + .name = "ip6-export", + .vector_size = sizeof (u32), + .format_trace = format_export_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN (export_error_strings), + .error_strings = export_error_strings, + .n_next_nodes = EXPORT_N_NEXT, + /* edit / add dispositions here */ + .next_nodes = + { + [EXPORT_NEXT_POP_HBYH] = "ip6-pop-hop-by-hop" + }, +}; diff --git a/src/plugins/ioam/ioam_plugin_doc.md b/src/plugins/ioam/ioam_plugin_doc.md new file mode 100644 index 00000000..343abcf7 --- /dev/null +++ b/src/plugins/ioam/ioam_plugin_doc.md @@ -0,0 +1,464 @@ +## VPP Inband OAM (iOAM) {#ioam_plugin_doc} + +In-band OAM (iOAM) is an implementation study to record operational +information in the packet while the packet traverses a path between +two points in the network. + +Overview of iOAM can be found in [iOAM-Devnet] page. +The following IETF drafts detail the motivation and mechanism for +recording operational information: + - [iOAM-ietf-requirements] - Describes motivation and usecases for iOAM + - [iOAM-ietf-data] - Describes data records that can be collected using iOAM + - [iOAM-ietf-transport] - Lists out the transport protocols + and mechanism to carry iOAM data records + - [iOAM-ietf-proof-of-transit] - Describes the idea of Proof of Transit (POT) + and mechanisms to operationalize the idea + +## Terminology +In-band OAM is expected to be deployed in a specific domain rather +than on the overall Internet. The part of the network which employs in-band OAM +is referred to as **"in-band OAM-domain"**. + +In-band OAM data is added to a packet on entering the in-band OAM-domain +and is removed from the packet when exiting the domain. +Within the in-band OAM-domain, network nodes that the packet traverses +may update the in-band OAM data records. + +- The node which adds in-band OAM data to the packet is called the +**"in-band OAM encapsulating node"**. + +- The node which removes the in-band OAM data is referred to as the +**"in-band OAM decapsulating node"**. + +- Nodes within the domain which are aware of in-band OAM data and read +and/or write or process the in-band OAM data are called +**"in-band OAM transit nodes"**. + +## Features supported in the current release +VPP can function as in-band OAM encapsulating, transit and decapsulating node. +In this version of VPP in-band OAM data is transported as options in an +IPv6 hop-by-hop extension header. Hence in-band OAM can be enabled +for IPv6 traffic. + +The following iOAM features are supported: + +- **In-band OAM Tracing** : In-band OAM supports multiple data records to be +recorded in the packet as the packet traverses the network. +These data records offer insights into the operational behavior of the network. +The following information can be collected in the tracing +data from the nodes a packet traverses: + - Node ID + - Ingress interface ID + - Egress interface ID + - Timestamp + - Pre-configured application data + +- **In-band OAM Proof of Transit (POT)**: Proof of transit iOAM data is +added to every packet for verifying that a packet traverses a specific +set of nodes. +In-band OAM data is updated at every node that is enabled with iOAM +proof of transit and is used to verify whether a packet traversed +all the specified nodes. When the verifier receives each packet, +it can validate whether the packet traversed the specified nodes. + + +## Configuration +Configuring iOAM involves: +- Selecting the packets for which iOAM data must be inserted, updated or removed + - Selection of packets for iOAM data insertion on iOAM encapsulating node. + Selection of packets is done by 5-tuple based classification + - Selection of packets for updating iOAM data is implicitly done on the + presence of iOAM options in the packet + - Selection of packets for removing the iOAM data is done on 5-tuple + based classification +- The kind of data to be collected + - Tracing data + - Proof of transit +- Additional details for processing iOAM data to be collected + - For trace data - trace type, number of nodes to be recorded in the trace, + time stamp precision, etc. + - For POT data - configuration of POT profile required to process the POT data + +The CLI for configuring iOAM is explained here followed by detailed steps +and examples to deploy iOAM on VPP as an encapsulating, transit or +decapsulating iOAM node in the subsequent sub-sections. + +VPP iOAM configuration for enabling trace and POT is as follows: + + set ioam rewrite trace-type <0x1f|0x7|0x9|0x11|0x19> + trace-elts trace-tsp <0|1|2|3> + node-id app-data [pot] + +A description of each of the options of the CLI follows: +- trace-type : An entry in the "Node data List" array of the trace option +can have different formats, following the needs of the a deployment. +For example: Some deployments might only be interested +in recording the node identifiers, whereas others might be interested +in recording node identifier and timestamp. +The following types are currently supported: + - 0x1f : Node data to include hop limit (8 bits), node ID (24 bits), + ingress and egress interface IDs (16 bits each), timestamp (32 bits), + application data (32 bits) + - 0x7 : Node data to include hop limit (8 bits), node ID (24 bits), + ingress and egress interface IDs (16 bits each) + - 0x9 : Node data to include hop limit (8 bits), node ID (24 bits), + timestamp (32 bits) + - 0x11: Node data to include hop limit (8 bits), node ID (24 bits), + application data (32 bits) + - 0x19: Node data to include hop limit (8 bits), node ID (24 bits), + timestamp (32 bits), application data (32 bits) +- trace-elts : Defines the length of the node data array in the trace option. +- trace-tsp : Defines the timestamp precision to use with the enumerated value + for precision as follows: + - 0 : 32bits timestamp in seconds + - 1 : 32bits timestamp in milliseconds + - 2 : 32bits timestamp in microseconds + - 3 : 32bits timestamp in nanoseconds +- node-id : Unique identifier for the node, included in the node ID + field of the node data in trace option. +- app-data : The value configured here is included as is in +application data field of node data in trace option. +- pot : Enables POT option to be included in the iOAM options. + +### Trace configuration + +#### On in-band OAM encapsulating node + - **Configure classifier and apply ACL** to select packets for + iOAM data insertion + - Example to enable iOAM data insertion for all the packets + towards IPv6 address db06::06: + + vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst + + vpp# classify session acl-hit-next node ip6-add-hop-by-hop + table-index 0 match l3 ip6 dst db06::06 + + vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 + + - **Enable tracing** : Specify node ID, maximum number of nodes for which + trace data should be recorded, type of data to be included for recording, + optionally application data to be included + - Example to enable tracing with a maximum of 4 nodes recorded + and the data to be recorded to include - hop limit, node id, + ingress and egress interface IDs, timestamp (millisecond precision), + application data (0x1234): + + + vpp# set ioam rewrite trace-type 0x1f trace-elts 4 trace-tsp 1 + node-id 0x1 app-data 0x1234 + + + +#### On in-band OAM transit node +- The transit node requires trace type, timestamp precision, node ID and +optionally application data to be configured, +to update its node data in the trace option. + +Example: + + vpp# set ioam rewrite trace-type 0x1f trace-elts 4 trace-tsp 1 + node-id 0x2 app-data 0x1234 + +#### On the In-band OAM decapsulating node +- The decapsulating node similar to encapsulating node requires +**classification** of the packets to remove iOAM data from. + - Example to decapsulate iOAM data for packets towards + db06::06, configure classifier and enable it as an ACL as follows: + + + vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst + + vpp# classify session acl-hit-next node ip6-lookup table-index 0 + match l3 ip6 dst db06::06 opaque-index 100 + + vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 + + +- Decapsulating node requires trace type, timestamp precision, +node ID and optionally application data to be configured, +to update its node data in the trace option before it is decapsulated. + +Example: + + vpp# set ioam rewrite trace-type 0x1f trace-elts 4 + trace-tsp 1 node-id 0x3 app-data 0x1234 + + +### Proof of Transit configuration + +For details on proof-of-transit, +see the IETF draft [iOAM-ietf-proof-of-transit]. +To enable Proof of Transit all the nodes that participate +and hence are verified for transit need a proof of transit profile. +A script to generate a proof of transit profile as per the mechanism +described in [iOAM-ietf-proof-of-transit] will be available at [iOAM-Devnet]. + +The Proof of transit mechanism implemented here is based on +Shamir's Secret Sharing algorithm. +The overall algorithm uses two polynomials +POLY-1 and POLY-2. The degree of polynomials depends on number of nodes +to be verified for transit. +POLY-1 is secret and constant. Each node gets a point on POLY-1 +at setup-time and keeps it secret. +POLY-2 is public, random and per packet. +Each node is assigned a point on POLY-1 and POLY-2 with the same x index. +Each node derives its point on POLY-2 each time a packet arrives at it. +A node then contributes its points on POLY-1 and POLY-2 to construct +POLY-3 (POLY-3 = POLY-1 + POLY-2) using lagrange extrapolation and +forwards it towards the verifier by updating POT data in the packet. +The verifier constructs POLY-3 from the accumulated value from all the nodes +and its own points on POLY-1 and POLY-2 and verifies whether +POLY-3 = POLY-1 + POLY-2. Only the verifier knows POLY-1. +The solution leverages finite field arithmetic in a field of size "prime number" +for reasons explained in description of Shamir's secret sharing algorithm. + +Here is an explanation of POT profile list and profile configuration CLI to +realize the above mechanism. +It is best to use the script provided at [iOAM-Devnet] to generate +this configuration. +- **Create POT profile** : set pot profile name id [0-1] +[validator-key 0xu64] prime-number 0xu64 secret_share 0xu64 +lpc 0xu64 polynomial2 0xu64 bits-in-random [0-64] + - name : Profile list name. + - id : Profile id, it can be 0 or 1. + A maximum of two profiles can be configured per profile list. + - validator-key : Secret key configured only on the + verifier/decapsulating node used to compare and verify proof of transit. + - prime-number : Prime number for finite field arithmetic as required by the + proof of transit mechanism. + - secret_share : Unique point for each node on the secret polynomial POLY-1. + - lpc : Lagrange Polynomial Constant(LPC) calculated per node based on + its point (x value used for evaluating the points on the polynomial) + on the polynomial used in lagrange extrapolation + for reconstructing polynomial (POLY-3). + - polynomial2 : Is the pre-evaluated value of the point on + 2nd polynomial(POLY-2). This is unique for each node. + It is pre-evaluated for all the coefficients of POLY-2 except + for the constant part of the polynomial that changes per packet + and is received as part of the POT data in the packet. + - bits-in-random : To control the size of the random number to be + generated. This number has to match the other numbers generated and used + in the profile as per the algorithm. + +- **Set a configured profile as active/in-use** : +set pot profile-active name ID [0-1] + - name : Name of the profile list to be used for computing + POT data per packet. + - ID : Identifier of the profile within the list to be used. + +#### On In-band OAM encapsulating node + - Configure the classifier and apply ACL to select packets for iOAM data insertion. + - Example to enable iOAM data insertion for all the packet towards + IPv6 address db06::06 - + + + vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst + + vpp# classify session acl-hit-next node + ip6-add-hop-by-hop table-index 0 match l3 ip6 dst db06::06 + + vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 + + + - Configure the proof of transit profile list with profiles. +Each profile list referred to by a name can contain 2 profiles, +only one is in use for updating proof of transit data at any time. + - Example profile list example with a profile generated from the + script to verify transit through 3 nodes is: + + + vpp# set pot profile name example id 0 prime-number 0x7fff0000fa884685 + secret_share 0x6c22eff0f45ec56d lpc 0x7fff0000fa884682 + polynomial2 0xffb543d4a9c bits-in-random 63 + + - Enable one of the profiles from the configured profile list as active + so that is will be used for calculating proof of transit + +Example enable profile ID 0 from profile list example configured above: + + + vpp# set pot profile-active name example ID 0 + + + - Enable POT option to be inserted + + + vpp# set ioam rewrite pot + + +#### On in-band OAM transit node + - Configure the proof of transit profile list with profiles for transit node. +Example: + + + vpp# set pot profile name example id 0 prime-number 0x7fff0000fa884685 + secret_share 0x564cdbdec4eb625d lpc 0x1 + polynomial2 0x23f3a227186a bits-in-random 63 + +#### On in-band OAM decapsulating node / verifier +- The decapsulating node, similar to the encapsulating node requires +classification of the packets to remove iOAM data from. + - Example to decapsulate iOAM data for packets towards db06::06 + configure classifier and enable it as an ACL as follows: + + + vpp# classify table miss-next node ip6-lookup mask l3 ip6 dst + + vpp# classify session acl-hit-next node ip6-lookup table-index 0 + match l3 ip6 dst db06::06 opaque-index 100 + + vpp# set int input acl intfc GigabitEthernet0/0/0 ip6-table 0 + +- To update and verify the proof of transit, POT profile list should be configured. + - Example POT profile list configured as follows: + + vpp# set pot profile name example id 0 validate-key 0x7fff0000fa88465d + prime-number 0x7fff0000fa884685 secret_share 0x7a08fbfc5b93116d lpc 0x3 + polynomial2 0x3ff738597ce bits-in-random 63 + +## Operational data + +Following CLIs are available to check iOAM operation: +- To check iOAM configuration that are effective use "show ioam summary" + +Example: + + vpp# show ioam summary + REWRITE FLOW CONFIGS - Not configured + HOP BY HOP OPTIONS - TRACE CONFIG - + Trace Type : 0x1f (31) + Trace timestamp precision : 1 (Milliseconds) + Num of trace nodes : 4 + Node-id : 0x2 (2) + App Data : 0x1234 (4660) + POT OPTION - 1 (Enabled) + Try 'show ioam pot and show pot profile' for more information + +- To find statistics about packets for which iOAM options were +added (encapsulating node) and removed (decapsulating node) execute +*show errors* + +Example on encapsulating node: + + + vpp# show error + Count Node Reason + 1208804706 ip6-inacl input ACL hits + 1208804706 ip6-add-hop-by-hop Pkts w/ added ip6 hop-by-hop options + +Example on decapsulating node: + + vpp# show error + Count Node Reason + 69508569 ip6-inacl input ACL hits + 69508569 ip6-pop-hop-by-hop Pkts w/ removed ip6 hop-by-hop options + +- To check the POT profiles use "show pot profile" + +Example: + + vpp# show pot profile + Profile list in use : example + POT Profile at index: 0 + ID : 0 + Validator : False (0) + Secret share : 0x564cdbdec4eb625d (6218586935324795485) + Prime number : 0x7fff0000fa884685 (9223090566081300101) + 2nd polynomial(eval) : 0x23f3a227186a (39529304496234) + LPC : 0x1 (1) + Bit mask : 0x7fffffffffffffff (9223372036854775807) + Profile index in use: 0 + Pkts passed : 0x36 (54) + +- To get statistics of POT for packets use "show ioam pot" + +Example at encapsulating or transit node: + + vpp# show ioam pot + Pkts with ip6 hop-by-hop POT options - 54 + Pkts with ip6 hop-by-hop POT options but no profile set - 0 + Pkts with POT in Policy - 0 + Pkts with POT out of Policy - 0 + + +Example at decapsulating/verification node: + + + vpp# show ioam pot + Pkts with ip6 hop-by-hop POT options - 54 + Pkts with ip6 hop-by-hop POT options but no profile set - 0 + Pkts with POT in Policy - 54 + Pkts with POT out of Policy - 0 + +- Tracing - enable trace of IPv6 packets to view the data inserted and +collected. + +Example when the nodes are receiving data over a DPDK interface: +Enable tracing using "trace add dpdk-input 20" and +execute "show trace" to view the iOAM data collected: + + + vpp# trace add dpdk-input 20 + + vpp# show trace + + ------------------- Start of thread 0 vpp_main ------------------- + + Packet 1 + + 00:00:19:294697: dpdk-input + GigabitEthernetb/0/0 rx queue 0 + buffer 0x10e6b: current data 0, length 214, free-list 0, totlen-nifb 0, trace 0x0 + PKT MBUF: port 0, nb_segs 1, pkt_len 214 + buf_len 2176, data_len 214, ol_flags 0x0, data_off 128, phys_addr 0xe9a35a00 + packet_type 0x0 + IP6: 00:50:56:9c:df:72 -> 00:50:56:9c:be:55 + IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 + tos 0x00, flow label 0x0, hop limit 63, payload length 160 + 00:00:19:294737: ethernet-input + IP6: 00:50:56:9c:df:72 -> 00:50:56:9c:be:55 + 00:00:19:294753: ip6-input + IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 + tos 0x00, flow label 0x0, hop limit 63, payload length 160 + 00:00:19:294757: ip6-lookup + fib 0 adj-idx 15 : indirect via db05::2 flow hash: 0x00000000 + IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 + tos 0x00, flow label 0x0, hop limit 63, payload length 160 + 00:00:19:294802: ip6-hop-by-hop + IP6_HOP_BY_HOP: next index 5 len 96 traced 96 Trace Type 0x1f , 1 elts left + [0] ttl 0x0 node ID 0x0 ingress 0x0 egress 0x0 ts 0x0 + app 0x0 + [1] ttl 0x3e node ID 0x3 ingress 0x1 egress 0x2 ts 0xb68c2213 + app 0x1234 + [2] ttl 0x3f node ID 0x2 ingress 0x1 egress 0x2 ts 0xb68c2204 + app 0x1234 + [3] ttl 0x40 node ID 0x1 ingress 0x5 egress 0x6 ts 0xb68c2200 + app 0x1234 + POT opt present + random = 0x577a916946071950, Cumulative = 0x10b46e78a35a392d, Index = 0x0 + 00:00:19:294810: ip6-rewrite + tx_sw_if_index 1 adj-idx 14 : GigabitEthernetb/0/0 + IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 flow hash: 0x00000000 + IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 + IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 + tos 0x00, flow label 0x0, hop limit 62, payload length 160 + 00:00:19:294814: GigabitEthernetb/0/0-output + GigabitEthernetb/0/0 + IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 + IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 + tos 0x00, flow label 0x0, hop limit 62, payload length 160 + 00:00:19:294820: GigabitEthernetb/0/0-tx + GigabitEthernetb/0/0 tx queue 0 + buffer 0x10e6b: current data 0, length 214, free-list 0, totlen-nifb 0, trace 0x0 + IP6: 00:50:56:9c:be:55 -> 00:50:56:9c:df:72 + + IP6_HOP_BY_HOP_OPTIONS: db05::2 -> db06::6 + + tos 0x00, flow label 0x0, hop limit 62, payload length 160 + + +[iOAM-Devnet]: +[iOAM-ietf-requirements]: +[iOAM-ietf-transport]: +[iOAM-ietf-data]: +[iOAM-ietf-proof-of-transit]: diff --git a/src/plugins/ioam/lib-pot/math64.h b/src/plugins/ioam/lib-pot/math64.h new file mode 100644 index 00000000..4c608a37 --- /dev/null +++ b/src/plugins/ioam/lib-pot/math64.h @@ -0,0 +1,159 @@ +/* + * math64.h provides the 64 bit unsigned integer add, multiply followed by modulo operation + * The linux/math64.h provides divide and multiply 64 bit integers but: + * 1. multiply: mul_u64_u64_shr - only returns 64 bits of the result and has to be called + * twice to get the complete 128 bits of the result. + * 2. Modulo operation of the result of addition and multiplication of u64 that may result + * in integers > 64 bits is not supported + * Hence this header to combine add/multiply followed by modulo of u64 integrers + * always resulting in u64. + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef include_vnet_math64_h +#define include_vnet_math64_h +#include + +/* + * multiplies and returns result in hi and lo + */ +static inline void mul64by64(u64 a, u64 b, u64 * hi, u64 * lo) +{ + u64 a_lo = (u64) (uint32_t) a; + u64 a_hi = a >> 32; + u64 b_lo = (u64) (u32) b; + u64 b_hi = b >> 32; + + u64 p0 = a_lo * b_lo; + u64 p1 = a_lo * b_hi; + u64 p2 = a_hi * b_lo; + u64 p3 = a_hi * b_hi; + + u32 cy = (u32) (((p0 >> 32) + (u32) p1 + (u32) p2) >> 32); + + *lo = p0 + (p1 << 32) + (p2 << 32); + *hi = p3 + (p1 >> 32) + (p2 >> 32) + cy; + return; +} + +#define TWO64 18446744073709551616.0 + +static inline u64 mod128by64(u64 x, u64 y, u64 m, double di) +{ + u64 q1, q2, q; + u64 p1, p0; + double dq; + + /* calculate quotient first pass 53 bits */ + dq = (TWO64 * (double)x + (double)y) * di; + + if (dq >= TWO64) + q1 = 0xfffffffffffff800L; + else + q1 = dq; + + /* q1 * m to compare the product to the dividend. */ + mul64by64(q1, m, &p1, &p0); + + /* Adjust quotient. is it > actual result: */ + if (x < p1 || (x == p1 && y < p0)) + { + /* q1 > quotient. calculate abs remainder */ + x = p1 - (x + (p0 < y)); + y = p0 - y; + + /* use the remainder as new dividend to adjust quotient */ + q2 = (u64) ((TWO64 * (double)x + (double)y) * di); + mul64by64(q2, m, &p1, &p0); + + q = q1 - q2; + if (x < p1 || (x == p1 && y <= p0)) + { + y = p0 - y; + } + else + { + y = p0 - y; + y += m; + q--; + } + } + else + { + x = x - (p1 + (y < p0)); + y = y - p0; + + q2 = (u64) ((TWO64 * (double)x + (double)y) * di); + mul64by64(q2, m, &p1, &p0); + + q = q1 + q2; + if (x < p1 || (x == p1 && y < p0)) + { + y = y - p0; + y += m; + q--; + } + else + { + y = y - p0; + if (y >= m) + { + y -= m; + q++; + } + } + } + + return y; +} + +/* + * returns a % p + */ +static inline u64 mod64by64(u64 a, u64 p, u64 primeinv) +{ + return (mod128by64(0, a, p, primeinv)); +} + +static inline void add64(u64 a, u64 b, u64 * whi, u64 * wlo) +{ + *wlo = a + b; + if (*wlo < a) + *whi = 1; + +} + +/* + * returns (a + b)%p + */ +static inline u64 add64_mod(u64 a, u64 b, u64 p, double pi) +{ + u64 shi = 0, slo = 0; + + add64(a, b, &shi, &slo); + return (mod128by64(shi, slo, p, pi)); +} + +/* + * returns (ab) % p + */ +static inline u64 mul64_mod(u64 a, u64 b, u64 p, double pi) +{ + u64 phi = 0, plo = 0; + + mul64by64(a, b, &phi, &plo); + return (mod128by64(phi, plo, p, pi)); +} + +#endif diff --git a/src/plugins/ioam/lib-pot/pot.api b/src/plugins/ioam/lib-pot/pot.api new file mode 100644 index 00000000..fa2fc126 --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot.api @@ -0,0 +1,133 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** \brief Proof of Transit(POT): Set POT profile + @param id - id of the profile + @param validator - True/False to indicate if this is verifier + @param secret_key - Verification key + @param secret_share - Share of the 1st polynomial + @param prime - Prime number used for modulo operation + @param max_bits - Max bits to be used for Random number generation + @param lpc - Lagrange basis polynomial + @param polynomial_public - pre-evaluated public polynomial + @param list_name_len - length of the name of this profile list + @param list_name - name of this profile list +*/ +define pot_profile_add { + u32 client_index; + u32 context; + u8 id; + u8 validator; + u64 secret_key; + u64 secret_share; + u64 prime; + u8 max_bits; + u64 lpc; + u64 polynomial_public; + u8 list_name_len; + u8 list_name[0]; +}; + +/** \brief Proof of Transit profile add / del response + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define pot_profile_add_reply { + u32 context; + i32 retval; +}; + + +/** \brief Proof of Transit(POT): Activate POT profile in the list + @param id - id of the profile + @param list_name_len - length of the name of this profile list + @param list_name - name of this profile list +*/ +define pot_profile_activate { + u32 client_index; + u32 context; + u8 id; + u8 list_name_len; + u8 list_name[0]; +}; + +/** \brief Proof of Transit profile activate response + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define pot_profile_activate_reply { + u32 context; + i32 retval; +}; + +/** \brief Delete POT Profile + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param list_name_len - length of the name of the profile list + @param list_name - name of profile list to delete +*/ +define pot_profile_del { + u32 client_index; + u32 context; + u8 list_name_len; + u8 list_name[0]; +}; + +/** \brief Proof of Transit profile add / del response + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define pot_profile_del_reply { + u32 context; + i32 retval; +}; + +/** \brief Show POT Profiles + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param id - id of the profile +*/ +define pot_profile_show_config_dump { + u32 client_index; + u32 context; + u8 id; +}; + +/** \brief Show POT profile reply + @param id - id of the profile + @param validator - True/False to indicate if this is verifier + @param secret_key - Verification key + @param secret_share - Share of the 1st polynomial + @param prime - Prime number used for modulo operation + @param max_bits - Max bits to be used for Random number generation + @param lpc - Lagrange basis polynomial + @param polynomial_public - pre-evaluated public polynomial + @param list_name_len - length of the name of this profile list + @param list_name - name of this profile list +*/ +define pot_profile_show_config_details { + u32 context; + i32 retval; + u8 id; + u8 validator; + u64 secret_key; + u64 secret_share; + u64 prime; + u64 bit_mask; + u64 lpc; + u64 polynomial_public; +}; diff --git a/src/plugins/ioam/lib-pot/pot_all_api_h.h b/src/plugins/ioam/lib-pot/pot_all_api_h.h new file mode 100644 index 00000000..63967c45 --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot_all_api_h.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/plugins/ioam/lib-pot/pot_api.c b/src/plugins/ioam/lib-pot/pot_api.c new file mode 100644 index 00000000..d3af7b40 --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot_api.c @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * pot_api.c - Proof of Transit related APIs to create + * and maintain profiles + *------------------------------------------------------------------ + */ + +#include +#include +#include + +#include +#include +#include + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +#define REPLY_MACRO2(t, body) \ +do { \ + unix_shared_memory_queue_t * q; \ + rv = vl_msg_api_pd_handler (mp, rv); \ + q = vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + do {body;} while (0); \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +/* List of message types that this plugin understands */ + +#define foreach_pot_plugin_api_msg \ +_(POT_PROFILE_ADD, pot_profile_add) \ +_(POT_PROFILE_ACTIVATE, pot_profile_activate) \ +_(POT_PROFILE_DEL, pot_profile_del) \ +_(POT_PROFILE_SHOW_CONFIG_DUMP, pot_profile_show_config_dump) \ + +static void vl_api_pot_profile_add_t_handler +(vl_api_pot_profile_add_t *mp) +{ + pot_main_t * sm = &pot_main; + int rv = 0; + vl_api_pot_profile_add_reply_t * rmp; + u8 id; + pot_profile *profile = NULL; + u8 *name = 0; + + if (mp->list_name_len) + name = format(0, "%s", mp->list_name); + + pot_profile_list_init(name); + id = mp->id; + profile = pot_profile_find(id); + if (profile) { + rv = pot_profile_create(profile, + clib_net_to_host_u64(mp->prime), + clib_net_to_host_u64(mp->polynomial_public), + clib_net_to_host_u64(mp->lpc), + clib_net_to_host_u64(mp->secret_share)); + if (rv != 0) + goto ERROROUT; + if (1 == mp->validator) + (void)pot_set_validator(profile, clib_net_to_host_u64(mp->secret_key)); + (void)pot_profile_set_bit_mask(profile, mp->max_bits); + } else { + rv = -3; + } + ERROROUT: + vec_free(name); + REPLY_MACRO(VL_API_POT_PROFILE_ADD_REPLY); +} + +static void send_pot_profile_details(vl_api_pot_profile_show_config_dump_t *mp, u8 id) +{ + vl_api_pot_profile_show_config_details_t * rmp; + pot_main_t * sm = &pot_main; + pot_profile *profile = pot_profile_find(id); + int rv = 0; + if(profile){ + REPLY_MACRO2(VL_API_POT_PROFILE_SHOW_CONFIG_DETAILS, + rmp->id=id; + rmp->validator=profile->validator; + rmp->secret_key=clib_host_to_net_u64(profile->secret_key); + rmp->secret_share=clib_host_to_net_u64(profile->secret_share); + rmp->prime=clib_host_to_net_u64(profile->prime); + rmp->bit_mask=clib_host_to_net_u64(profile->bit_mask); + rmp->lpc=clib_host_to_net_u64(profile->lpc); + rmp->polynomial_public=clib_host_to_net_u64(profile->poly_pre_eval); + ); + } + else{ + REPLY_MACRO2(VL_API_POT_PROFILE_SHOW_CONFIG_DETAILS, + rmp->id=id; + rmp->validator=0; + rmp->secret_key=0; + rmp->secret_share=0; + rmp->prime=0; + rmp->bit_mask=0; + rmp->lpc=0; + rmp->polynomial_public=0; + ); + } +} + +static void vl_api_pot_profile_show_config_dump_t_handler +(vl_api_pot_profile_show_config_dump_t *mp) +{ + u8 id = mp->id; + u8 dump_call_id = ~0; + if(dump_call_id==id){ + for(id=0;idlist_name_len) + name = format(0, "%s", mp->list_name); + if (!pot_profile_list_is_enabled(name)) { + rv = -1; + } else { + id = mp->id; + rv = pot_profile_set_active(id); + } + + vec_free(name); + REPLY_MACRO(VL_API_POT_PROFILE_ACTIVATE_REPLY); +} + + +static void vl_api_pot_profile_del_t_handler +(vl_api_pot_profile_del_t *mp) +{ + pot_main_t * sm = &pot_main; + int rv = 0; + vl_api_pot_profile_del_reply_t * rmp; + + clear_pot_profiles(); + + REPLY_MACRO(VL_API_POT_PROFILE_DEL_REPLY); +} + + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + pot_main_t * sm = &pot_main; + clib_error_t * error = 0; + + sm->vlib_main = vm; + sm->vnet_main = h->vnet_main; + return error; +} + +/* Set up the API message handling tables */ +static clib_error_t * +pot_plugin_api_hookup (vlib_main_t *vm) +{ + pot_main_t * sm = &pot_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_pot_plugin_api_msg; +#undef _ + + return 0; +} + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (pot_main_t * sm, api_main_t * am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); + foreach_vl_msg_name_crc_pot; +#undef _ +} + +static clib_error_t * pot_init (vlib_main_t * vm) +{ + pot_main_t * sm = &pot_main; + clib_error_t * error = 0; + u8 * name; + + bzero(sm, sizeof(pot_main)); + (void)pot_util_init(); + name = format (0, "ioam_pot_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + sm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + + error = pot_plugin_api_hookup (vm); + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (sm, &api_main); + + vec_free(name); + + return error; +} + +VLIB_INIT_FUNCTION (pot_init); diff --git a/src/plugins/ioam/lib-pot/pot_msg_enum.h b/src/plugins/ioam/lib-pot/pot_msg_enum.h new file mode 100644 index 00000000..a4a88bed --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_pot_msg_enum_h +#define included_pot_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_pot_msg_enum_h */ diff --git a/src/plugins/ioam/lib-pot/pot_test.c b/src/plugins/ioam/lib-pot/pot_test.c new file mode 100644 index 00000000..2e870238 --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot_test.c @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * pot_test.c - test harness for pot plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} pot_test_main_t; + +pot_test_main_t pot_test_main; + +#define foreach_standard_reply_retval_handler \ +_(pot_profile_add_reply) \ +_(pot_profile_activate_reply) \ +_(pot_profile_del_reply) + +#define foreach_custom_reply_retval_handler \ +_(pot_profile_show_config_details, \ + errmsg(" ID:%d\n",mp->id); \ + errmsg(" Validator:%d\n",mp->validator); \ + errmsg(" secret_key:%Lx\n",clib_net_to_host_u64(mp->secret_key)); \ + errmsg(" secret_share:%Lx\n",clib_net_to_host_u64(mp->secret_share)); \ + errmsg(" prime:%Lx\n",clib_net_to_host_u64(mp->prime)); \ + errmsg(" bitmask:%Lx\n",clib_net_to_host_u64(mp->bit_mask)); \ + errmsg(" lpc:%Lx\n",clib_net_to_host_u64(mp->lpc)); \ + errmsg(" public poly:%Lx\n",clib_net_to_host_u64(mp->polynomial_public)); \ + ) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = pot_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +#define _(n,body) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = pot_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + do{body;}while(0); \ + } +foreach_custom_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(POT_PROFILE_ADD_REPLY, pot_profile_add_reply) \ +_(POT_PROFILE_ACTIVATE_REPLY, pot_profile_activate_reply) \ +_(POT_PROFILE_DEL_REPLY, pot_profile_del_reply) \ +_(POT_PROFILE_SHOW_CONFIG_DETAILS, pot_profile_show_config_details) + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + + +static int api_pot_profile_add (vat_main_t *vam) +{ +#define MAX_BITS 64 + pot_test_main_t * sm = &pot_test_main; + unformat_input_t *input = vam->input; + vl_api_pot_profile_add_t *mp; + u8 *name = NULL; + u64 prime = 0; + u64 secret_share = 0; + u64 secret_key = 0; + u32 bits = MAX_BITS; + u64 lpc = 0, poly2 = 0; + f64 timeout; + u8 id = 0; + int rv = 0; + + while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(input, "name %s", &name)) + ; + else if(unformat(input, "id %d", &id)) + ; + else if (unformat(input, "validator-key 0x%Lx", &secret_key)) + ; + else if (unformat(input, "prime-number 0x%Lx", &prime)) + ; + else if (unformat(input, "secret-share 0x%Lx", &secret_share)) + ; + else if (unformat(input, "polynomial-public 0x%Lx", &poly2)) + ; + else if (unformat(input, "lpc 0x%Lx", &lpc)) + ; + else if (unformat(input, "bits-in-random %u", &bits)) + { + if (bits > MAX_BITS) + bits = MAX_BITS; + } + else + break; + } + + if (!name) + { + errmsg ("name required\n"); + rv = -99; + goto OUT; + } + + M2(POT_PROFILE_ADD, pot_profile_add, vec_len(name)); + + mp->list_name_len = vec_len(name); + clib_memcpy(mp->list_name, name, mp->list_name_len); + mp->secret_share = clib_host_to_net_u64(secret_share); + mp->polynomial_public = clib_host_to_net_u64(poly2); + mp->lpc = clib_host_to_net_u64(lpc); + mp->prime = clib_host_to_net_u64(prime); + if (secret_key != 0) + { + mp->secret_key = clib_host_to_net_u64(secret_key); + mp->validator = 1; + } + else + { + mp->validator = 0; + } + mp->id = id; + mp->max_bits = bits; + + S; W; + +OUT: + vec_free(name); + return(rv); +} + +static int api_pot_profile_activate (vat_main_t *vam) +{ +#define MAX_BITS 64 + pot_test_main_t * sm = &pot_test_main; + unformat_input_t *input = vam->input; + vl_api_pot_profile_activate_t *mp; + u8 *name = NULL; + u8 id = 0; + int rv = 0; + f64 timeout; + + while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(input, "name %s", &name)) + ; + else if(unformat(input, "id %d", &id)) + ; + else + break; + } + + if (!name) + { + errmsg ("name required\n"); + rv = -99; + goto OUT; + } + + M2(POT_PROFILE_ACTIVATE, pot_profile_activate, vec_len(name)); + + mp->list_name_len = vec_len(name); + clib_memcpy(mp->list_name, name, mp->list_name_len); + mp->id = id; + + S; W; + +OUT: + vec_free(name); + return(rv); +} + + +static int api_pot_profile_del (vat_main_t *vam) +{ + pot_test_main_t * sm = &pot_test_main; + vl_api_pot_profile_del_t *mp; + f64 timeout; + + M(POT_PROFILE_DEL, pot_profile_del); + mp->list_name_len = 0; + S; W; + return 0; +} + +static int api_pot_profile_show_config_dump (vat_main_t *vam) +{ + pot_test_main_t * sm = &pot_test_main; + unformat_input_t *input = vam->input; + vl_api_pot_profile_show_config_dump_t *mp; + f64 timeout; + u8 id = 0; + + while(unformat_check_input(input) != UNFORMAT_END_OF_INPUT) + { + if(unformat(input,"id %d",&id)); + else + break; + } + M(POT_PROFILE_SHOW_CONFIG_DUMP, pot_profile_show_config_dump); + + mp->id = id; + + S; W; + return 0; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(pot_profile_add, "name id [0-1] " \ + "prime-number <0xu64> bits-in-random [0-64] " \ + "secret-share <0xu64> lpc <0xu64> polynomial-public <0xu64> " \ + "[validator-key <0xu64>] [validity <0xu64>]") \ +_(pot_profile_activate, "name id [0-1] ") \ +_(pot_profile_del, "[id ]") \ +_(pot_profile_show_config_dump, "id [0-1]") + +void vat_api_hookup (vat_main_t *vam) +{ + pot_test_main_t * sm = &pot_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * vat_plugin_register (vat_main_t *vam) +{ + pot_test_main_t * sm = &pot_test_main; + u8 * name; + + sm->vat_main = vam; + + name = format (0, "ioam_pot_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~0) + vat_api_hookup (vam); + + vec_free(name); + + return 0; +} diff --git a/src/plugins/ioam/lib-pot/pot_util.c b/src/plugins/ioam/lib-pot/pot_util.c new file mode 100644 index 00000000..a253ad41 --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot_util.c @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include "math64.h" +#include "pot_util.h" + +pot_main_t pot_main; + +static void pot_profile_cleanup(pot_profile *profile); + +static void pot_main_profiles_reset (void) +{ + pot_main_t *sm = &pot_main; + int i = 0; + + for (i = 0; i < MAX_POT_PROFILES; i++) + { + pot_profile_cleanup(&(sm->profile_list[i])); + } + sm->active_profile_id = 0; + if (sm->profile_list_name) + vec_free(sm->profile_list_name); + sm->profile_list_name = NULL; +} + +int pot_util_init (void) +{ + pot_main_profiles_reset(); + + return(0); +} + +static void pot_profile_init(pot_profile * new, u8 id) +{ + if (new) + { + memset(new, 0, sizeof(pot_profile)); + new->id = id; + } +} + +pot_profile *pot_profile_find(u8 id) +{ + pot_main_t *sm = &pot_main; + + if (id < MAX_POT_PROFILES) + { + return (&(sm->profile_list[id])); + } + return (NULL); +} +static int pot_profile_name_equal (u8 *name0, u8 *name1) +{ + int len0, len1; + + len0 = vec_len (name0); + len1 = vec_len (name1); + if (len0 != len1) + return(0); + return (0==strncmp ((char *) name0, (char *)name1, len0)); +} + +int pot_profile_list_is_enabled (u8 *name) +{ + pot_main_t *sm = &pot_main; + return (pot_profile_name_equal(sm->profile_list_name, name)); +} + +void pot_profile_list_init(u8 * profile_list_name) +{ + pot_main_t *sm = &pot_main; + int i = 0; + + /* If it is the same profile list skip reset */ + if (pot_profile_name_equal(sm->profile_list_name, profile_list_name)) + { + return; + } + + pot_main_profiles_reset(); + if (vec_len(profile_list_name)) + sm->profile_list_name = (u8 *)vec_dup(profile_list_name); + else + sm->profile_list_name = 0; + sm->active_profile_id = 0; + + for (i = 0; i < MAX_POT_PROFILES; i++) + { + pot_profile_init(&(sm->profile_list[i]), i); + } +} + +static void pot_profile_cleanup(pot_profile * profile) +{ + u16 id = profile->id; + + memset(profile, 0, sizeof(pot_profile)); + profile->id = id; /* Restore id alone */ +} + +int pot_profile_create(pot_profile * profile, u64 prime, + u64 poly2, u64 lpc, u64 secret_share) +{ + if (profile && !profile->in_use) + { + pot_profile_cleanup(profile); + profile->prime = prime; + profile->primeinv = 1.0 / prime; + profile->lpc = lpc; + profile->poly_pre_eval = poly2; + profile->secret_share = secret_share; + profile->total_pkts_using_this_profile = 0; + profile->valid = 1; + return(0); + } + + return(-1); +} + +int pot_set_validator(pot_profile * profile, u64 key) +{ + if (profile && !profile->in_use) + { + profile->validator = 1; + profile->secret_key = key; + return(0); + } + return(-1); +} + +always_inline u64 pot_update_cumulative_inline(u64 cumulative, u64 random, + u64 secret_share, u64 prime, u64 lpc, u64 pre_split, double prime_inv) +{ + u64 share_random = 0; + u64 cumulative_new = 0; + + /* + * calculate split share for random + */ + share_random = add64_mod(pre_split, random, prime, prime_inv); + + /* + * lpc * (share_secret + share_random) + */ + share_random = add64_mod(share_random, secret_share, prime, prime_inv); + share_random = mul64_mod(share_random, lpc, prime, prime_inv); + + cumulative_new = add64_mod(cumulative, share_random, prime, prime_inv); + + return (cumulative_new); +} + +u64 pot_update_cumulative(pot_profile * profile, u64 cumulative, u64 random) +{ + if (profile && profile->valid != 0) + { + return (pot_update_cumulative_inline(cumulative, random, profile->secret_share, + profile->prime, profile->lpc, profile->poly_pre_eval, + profile->primeinv)); + } + return (0); +} + +always_inline u8 pot_validate_inline(u64 secret, u64 prime, double prime_inv, + u64 cumulative, u64 random) +{ + if (cumulative == (random + secret)) + { + return (1); + } + else if (cumulative == add64_mod(random, secret, prime, prime_inv)) + { + return (1); + } + return (0); +} + +/* + * return True if the cumulative matches secret from a profile + */ +u8 pot_validate(pot_profile * profile, u64 cumulative, u64 random) +{ + if (profile && profile->validator) + { + return (pot_validate_inline(profile->secret_key, profile->prime, + profile->primeinv, cumulative, random)); + } + return (0); +} + +/* + * Utility function to get random number per pack + */ +u64 pot_generate_random(pot_profile * profile) +{ + u64 random = 0; + int32_t second_half; + static u32 seed = 0; + + if (PREDICT_FALSE(!seed)) + seed = random_default_seed(); + + /* + * Upper 4 bytes seconds + */ + random = (u64) time(NULL); + + random &= 0xffffffff; + random = random << 32; + /* + * Lower 4 bytes random number + */ + second_half = random_u32(&seed); + + random |= second_half; + + if (PREDICT_TRUE(profile != NULL)) + { + random &= profile->bit_mask; + } + return (random); +} + +int pot_profile_set_bit_mask(pot_profile * profile, u16 bits) +{ + int sizeInBits; + + if (profile && !profile->in_use) + { + sizeInBits = sizeof(profile->bit_mask) * 8; + profile->bit_mask = + (bits >= + sizeInBits ? (u64) - 1 : (u64) ((u64) 1 << (u64) bits) - 1); + return(0); + } + return(-1); +} + +clib_error_t *clear_pot_profile_command_fn(vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + + pot_main_profiles_reset(); + + return 0; +} + +void clear_pot_profiles() +{ + clear_pot_profile_command_fn(0, 0, 0); +} + +VLIB_CLI_COMMAND(clear_pot_profile_command) = +{ +.path = "clear pot profile", +.short_help = "clear pot profile [|all]", +.function = clear_pot_profile_command_fn, +}; + +static clib_error_t *set_pot_profile_command_fn(vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + u64 prime; + u64 secret_share; + u64 secret_key; + u8 validator = 0; + u32 profile_id = ~0; + u32 bits; + u64 lpc = 0, poly2 = 0; + pot_profile *profile = NULL; + u8 *profile_list_name = NULL; + + bits = MAX_BITS; + + while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(input, "name %s", + &profile_list_name)); + else if (unformat(input, "id %d", &profile_id)) + ; + else if (unformat(input, "validate-key 0x%Lx", &secret_key)) + validator = 1; + else if (unformat(input, "prime-number 0x%Lx", &prime)) + ; + else if (unformat(input, "secret_share 0x%Lx", &secret_share)) + ; + else if (unformat(input, "polynomial2 0x%Lx", &poly2)) + ; + else if (unformat(input, "lpc 0x%Lx", &lpc)) + ; + else if (unformat(input, "bits-in-random %d", &bits)) + { + if (bits > MAX_BITS) + bits = MAX_BITS; + } + else + break; + } + if (profile_list_name == 0) + { + return clib_error_return(0, "Name cannot be null"); + } + pot_profile_list_init(profile_list_name); + profile = pot_profile_find(profile_id); + + if (profile) + { + pot_profile_create(profile, prime, poly2, lpc, secret_share); + if (validator) + pot_set_validator(profile, secret_key); + pot_profile_set_bit_mask(profile, bits); + } + vec_free(profile_list_name); + return 0; +} + +VLIB_CLI_COMMAND(set_pot_profile_command) = +{ +.path = "set pot profile", +.short_help = "set pot profile name id [0-1] [validator-key 0xu64] \ + prime-number 0xu64 secret_share 0xu64 lpc 0xu64 \ + polynomial2 0xu64 bits-in-random [0-64] ", +.function = set_pot_profile_command_fn, +}; + +static clib_error_t *set_pot_profile_activate_command_fn(vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + pot_main_t *sm = &pot_main; + u8 *profile_list_name = NULL; + u32 id = 0; + clib_error_t *result = NULL; + + while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(input, "name %s", + &profile_list_name)); + else if (unformat(input, "id %d", &id)) + ; + else + return clib_error_return(0, "unknown input `%U'", + format_unformat_error, input); + } + if (profile_list_name == 0) + { + return clib_error_return(0, "Name cannot be null"); + } + + if (!pot_profile_list_is_enabled(profile_list_name)) { + result = clib_error_return(0, "%s list is not enabled, profile in use %s", + profile_list_name, sm->profile_list_name); + } else if (0 != pot_profile_set_active((u8)id)) { + result = clib_error_return(0, "Profile %d not defined in %s", + id, sm->profile_list_name); + } + vec_free(profile_list_name); + return result; +} + +VLIB_CLI_COMMAND(set_pot_profile_activate_command) = +{ +.path = "set pot profile-active", +.short_help = "set pot profile-active name id [0-1]", +.function = set_pot_profile_activate_command_fn, +}; + +static clib_error_t *show_pot_profile_command_fn(vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + pot_main_t *sm = &pot_main; + pot_profile *p = NULL; + u16 i; + u8 *s = 0; + + if (vec_len(sm->profile_list_name) == 0) + { + s = format(s, "POT Profiles not configured\n"); + vlib_cli_output(vm, "%v", s); + return 0; + } + s = format(s, "Profile list in use : %s\n",sm->profile_list_name); + for (i = 0; i < MAX_POT_PROFILES; i++) + { + p = pot_profile_find(i); + if (p->valid == 0) + continue; + s = format(s, "POT Profile at index: %d\n", i); + s = format(s, " Id : %d\n", p->id); + s = format(s, " Validator : %s (%d)\n", + (p->validator) ? "True" : "False", p->validator); + if (p->validator == 1) + s = format(s, " Secret key : 0x%Lx (%Ld)\n", + p->secret_key, p->secret_key); + s = format(s, " Secret share : 0x%Lx (%Ld)\n", + p->secret_share, p->secret_share); + s = format(s, " Prime number : 0x%Lx (%Ld)\n", + p->prime, p->prime); + s = format(s, "2nd polynomial(eval) : 0x%Lx (%Ld)\n", + p->poly_pre_eval, p->poly_pre_eval); + s = format(s, " LPC : 0x%Lx (%Ld)\n", p->lpc, p->lpc); + + s = format(s, " Bit mask : 0x%Lx (%Ld)\n", + p->bit_mask, p->bit_mask); + } + + p = pot_profile_find(sm->active_profile_id); + + if (p && p->valid && p->in_use) { + s = format(s, "\nProfile index in use: %d\n", sm->active_profile_id); + s = format(s, "Pkts passed : 0x%Lx (%Ld)\n", + p->total_pkts_using_this_profile, + p->total_pkts_using_this_profile); + if (pot_is_decap(p)) + s = format(s, " This is Decap node. \n"); + } else { + s = format(s, "\nProfile index in use: None\n"); + } + vlib_cli_output(vm, "%v", s); + vec_free(s); + + return 0; +} + +VLIB_CLI_COMMAND(show_pot_profile_command) = +{ +.path = "show pot profile", +.short_help = "show pot profile", +.function = show_pot_profile_command_fn, +}; diff --git a/src/plugins/ioam/lib-pot/pot_util.h b/src/plugins/ioam/lib-pot/pot_util.h new file mode 100644 index 00000000..9df31fae --- /dev/null +++ b/src/plugins/ioam/lib-pot/pot_util.h @@ -0,0 +1,195 @@ +/* + * pot_util.h -- Proof Of Transit Utility Header + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef include_vnet_pot_util_h +#define include_vnet_pot_util_h + +#include +#define debug_ioam debug_ioam_fn +/* Dont change this size 256. This is there across multiple components */ +#define PATH_NAME_SIZE 256 + +/* Ring size. this should be same as the one in ODL. Do not change this + without change in ODL. */ +#define MAX_POT_PROFILES 2 + +/** + * Usage: + * + * On any node that participates in Proof of Transit: + * + * Step 1: Initialize this library by calling pot_init() + * Step 2: Setup a proof of transit profile that contains all the parameters needed to compute cumulative: + * Call these functions: + * pot_profile_find + * pot_profile_create + * pot_profile_set_bit_mask - To setup how large we want the numbers used in the computation and random number <= 64 bits + * Step 2a: For validator do this: + * pot_set_validator + * Step 2b: On initial node enable the profile to be used: + * pot_profile_set_active / pot_profile_get_active will return the profile + * Step 3a: At the initial node to generate Random number that will be read by all other nodes: + * pot_generate_random + * Step 3b: At all nodes including initial and verifier call this to compute cumulative: + * pot_update_cumulative + * Step 4: At the verifier: + * pot_validate + * + */ + +typedef struct pot_profile_ +{ + u8 id : 1; + u8 valid : 1; + u8 in_use : 1; + u64 random; + u8 validator; + u64 secret_key; + u64 secret_share; + u64 prime; + u64 lpc; + u64 poly_pre_eval; + u64 bit_mask; + u64 limit; + double primeinv; + u64 total_pkts_using_this_profile; +} pot_profile; + +typedef struct { + /* Name of the default profile list in use*/ + u8 *profile_list_name; + pot_profile profile_list[MAX_POT_PROFILES]; + /* number of profiles in the list */ + u8 active_profile_id : 1; + + /* API message ID base */ + u16 msg_id_base; + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; +} pot_main_t; + +extern pot_main_t pot_main; + +/* + * Initialize proof of transit + */ +int pot_util_init(void); +void pot_profile_list_init(u8 * name); + + +/* + * Find a pot profile by ID + */ +pot_profile *pot_profile_find(u8 id); + +static inline u16 pot_profile_get_id(pot_profile * profile) +{ + if (profile) + { + return (profile->id); + } + return (0); +} + +/* setup and clean up profile */ +int pot_profile_create(pot_profile * profile, u64 prime, + u64 poly2, u64 lpc, u64 secret_share); +/* + * Setup profile as a validator + */ +int pot_set_validator(pot_profile * profile, u64 key); + +/* + * Setup max bits to be used for random number generation + */ +#define MAX_BITS 64 +int pot_profile_set_bit_mask(pot_profile * profile, u16 bits); + +/* + * Given a random and cumulative compute the new cumulative for a given profile + */ +u64 pot_update_cumulative(pot_profile * profile, u64 cumulative, u64 random); + +/* + * return True if the cumulative matches secret from a profile + */ +u8 pot_validate(pot_profile * profile, u64 cumulative, u64 random); + +/* + * Utility function to get random number per pack + */ +u64 pot_generate_random(pot_profile * profile); + + +extern void clear_pot_profiles(); +extern int pot_profile_list_is_enabled(u8 *name); + +static inline u8 pot_is_decap(pot_profile * p) +{ + return (p->validator == 1); +} + +static inline int pot_profile_set_active (u8 id) +{ + pot_main_t *sm = &pot_main; + pot_profile *profile = NULL; + pot_profile *current_active_prof = NULL; + + current_active_prof = pot_profile_find(sm->active_profile_id); + profile = pot_profile_find(id); + if (profile && profile->valid) { + sm->active_profile_id = id; + current_active_prof->in_use = 0; + profile->in_use = 1; + return(0); + } + return(-1); +} +static inline u8 pot_profile_get_active_id (void) +{ + pot_main_t *sm = &pot_main; + return (sm->active_profile_id); +} + +static inline pot_profile * pot_profile_get_active (void) +{ + pot_main_t *sm = &pot_main; + pot_profile *profile = NULL; + profile = pot_profile_find(sm->active_profile_id); + if (profile && profile->in_use) + return(profile); + return (NULL); +} + +static inline void pot_profile_reset_usage_stats (pot_profile *pow) +{ + if (pow) { + pow->total_pkts_using_this_profile = 0; + } +} + +static inline void pot_profile_incr_usage_stats (pot_profile *pow) +{ + if (pow) { + pow->total_pkts_using_this_profile++; + } +} + + +#endif diff --git a/src/plugins/ioam/lib-trace/trace.api b/src/plugins/ioam/lib-trace/trace.api new file mode 100644 index 00000000..cb958325 --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace.api @@ -0,0 +1,92 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** \brief iOAM6 Trace - Set the iOAM6 trace profile + @param trace_type - Type of trace requested + @param num_elts - Number of trace elements to be inserted + @param node_id - Trace Node ID + @param trace_tsp- Timestamp resolution + @param app_data - Application specific opaque +*/ +define trace_profile_add { + u32 client_index; + u32 context; + u8 trace_type; + u8 num_elts; + u8 trace_tsp; + u32 node_id; + u32 app_data; +}; + +/** \brief Trace profile add / del response + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define trace_profile_add_reply { + u32 context; + i32 retval; +}; + + + +/** \brief Delete trace Profile + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define trace_profile_del { + u32 client_index; + u32 context; +}; + +/** \brief Trace profile add / del response + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define trace_profile_del_reply { + u32 context; + i32 retval; +}; + + + +/** \brief Show trace Profile + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define trace_profile_show_config { + u32 client_index; + u32 context; +}; + +/** \brief Show trace config response + @param context - sender context, to match reply w/ request + @param retval - return value for request + @param trace_type - Type of trace requested + @param num_elts - Number of trace elements to be inserted + @param node_id - Trace Node ID + @param trace_tsp- Timestamp resolution + @param app_data - Application specific opaque +*/ +define trace_profile_show_config_reply { + u32 context; + i32 retval; + u8 trace_type; + u8 num_elts; + u8 trace_tsp; + u32 node_id; + u32 app_data; +}; diff --git a/src/plugins/ioam/lib-trace/trace_all_api_h.h b/src/plugins/ioam/lib-trace/trace_all_api_h.h new file mode 100644 index 00000000..223f9545 --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace_all_api_h.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/plugins/ioam/lib-trace/trace_api.c b/src/plugins/ioam/lib-trace/trace_api.c new file mode 100644 index 00000000..7e0d708e --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace_api.c @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * trace_api.c - iOAM Trace related APIs to create + * and maintain profiles + *------------------------------------------------------------------ + */ + +#include +#include +#include + +#include +#include +#include + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define TRACE_REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +/* *INDENT-OFF* */ +#define TRACE_REPLY_MACRO2(t, body) \ +do { \ + unix_shared_memory_queue_t * q; \ + rv = vl_msg_api_pd_handler (mp, rv); \ + q = vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + do {body;} while (0); \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); +/* *INDENT-ON* */ + +/* List of message types that this plugin understands */ + +#define foreach_trace_plugin_api_msg \ +_(TRACE_PROFILE_ADD, trace_profile_add) \ +_(TRACE_PROFILE_DEL, trace_profile_del) \ +_(TRACE_PROFILE_SHOW_CONFIG, trace_profile_show_config) + +static void vl_api_trace_profile_add_t_handler + (vl_api_trace_profile_add_t * mp) +{ + trace_main_t *sm = &trace_main; + int rv = 0; + vl_api_trace_profile_add_reply_t *rmp; + trace_profile *profile = NULL; + + profile = trace_profile_find (); + if (profile) + { + rv = + trace_profile_create (profile, mp->trace_type, mp->num_elts, + mp->trace_tsp, ntohl (mp->node_id), + ntohl (mp->app_data)); + if (rv != 0) + goto ERROROUT; + } + else + { + rv = -3; + } +ERROROUT: + TRACE_REPLY_MACRO (VL_API_TRACE_PROFILE_ADD_REPLY); +} + + +static void vl_api_trace_profile_del_t_handler + (vl_api_trace_profile_del_t * mp) +{ + trace_main_t *sm = &trace_main; + int rv = 0; + vl_api_trace_profile_del_reply_t *rmp; + + clear_trace_profiles (); + + TRACE_REPLY_MACRO (VL_API_TRACE_PROFILE_DEL_REPLY); +} + +static void vl_api_trace_profile_show_config_t_handler + (vl_api_trace_profile_show_config_t * mp) +{ + trace_main_t *sm = &trace_main; + vl_api_trace_profile_show_config_reply_t *rmp; + int rv = 0; + trace_profile *profile = trace_profile_find (); + if (profile->valid) + { + TRACE_REPLY_MACRO2 (VL_API_TRACE_PROFILE_SHOW_CONFIG_REPLY, + rmp->trace_type = profile->trace_type; + rmp->num_elts = profile->num_elts; + rmp->trace_tsp = profile->trace_tsp; + rmp->node_id = htonl (profile->node_id); + rmp->app_data = htonl (profile->app_data); + ); + } + else + { + TRACE_REPLY_MACRO2 (VL_API_TRACE_PROFILE_SHOW_CONFIG_REPLY, + rmp->trace_type = 0; + rmp->num_elts = 0; rmp->trace_tsp = 0; + rmp->node_id = 0; rmp->app_data = 0; + ); + } +} + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + trace_main_t *sm = &trace_main; + clib_error_t *error = 0; + + sm->vlib_main = vm; + sm->vnet_main = h->vnet_main; + return error; +} + +/* Set up the API message handling tables */ +static clib_error_t * +trace_plugin_api_hookup (vlib_main_t * vm) +{ + trace_main_t *sm = &trace_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_trace_plugin_api_msg; +#undef _ + + return 0; +} + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (trace_main_t * sm, api_main_t * am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); + foreach_vl_msg_name_crc_trace; +#undef _ +} + +static clib_error_t * +trace_init (vlib_main_t * vm) +{ + trace_main_t *sm = &trace_main; + clib_error_t *error = 0; + u8 *name; + + bzero (sm, sizeof (trace_main)); + (void) trace_util_init (); + name = format (0, "ioam_trace_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + sm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + + error = trace_plugin_api_hookup (vm); + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (sm, &api_main); + + vec_free (name); + + return error; +} + +VLIB_INIT_FUNCTION (trace_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-trace/trace_msg_enum.h b/src/plugins/ioam/lib-trace/trace_msg_enum.h new file mode 100644 index 00000000..78c35665 --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_trace_msg_enum_h +#define included_trace_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_trace_msg_enum_h */ diff --git a/src/plugins/ioam/lib-trace/trace_test.c b/src/plugins/ioam/lib-trace/trace_test.c new file mode 100644 index 00000000..111dd461 --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace_test.c @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * trace_test.c - test harness for trace plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} trace_test_main_t; + +trace_test_main_t trace_test_main; + +#define foreach_standard_reply_retval_handler \ +_(trace_profile_add_reply) \ +_(trace_profile_del_reply) + +#define foreach_custom_reply_handler \ +_(trace_profile_show_config_reply, \ + if(mp->trace_type) \ + { \ + errmsg(" Trace Type : 0x%x (%d)\n",mp->trace_type, mp->trace_type); \ + errmsg(" Trace timestamp precision : %d \n",mp->trace_tsp); \ + errmsg(" Node Id : 0x%x (%d)\n",htonl(mp->node_id), htonl(mp->node_id)); \ + errmsg(" App Data : 0x%x (%d)\n",htonl(mp->app_data), htonl(mp->app_data)); \ + } \ + else errmsg("No valid trace profile configuration found\n");) +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = trace_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +#define _(n,body) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = trace_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + if(retval>=0)do{body;} while(0); \ + else errmsg("Error, retval: %d",retval); \ + } +foreach_custom_reply_handler; +#undef _ +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(TRACE_PROFILE_ADD_REPLY, trace_profile_add_reply) \ +_(TRACE_PROFILE_DEL_REPLY, trace_profile_del_reply) \ +_(TRACE_PROFILE_SHOW_CONFIG_REPLY, trace_profile_show_config_reply) + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + + +static int +api_trace_profile_add (vat_main_t * vam) +{ + trace_test_main_t *sm = &trace_test_main; + unformat_input_t *input = vam->input; + vl_api_trace_profile_add_t *mp; + u8 trace_type = 0; + u8 num_elts = 0; + int rv = 0; + u32 node_id = 0; + u32 app_data = 0; + u8 trace_tsp = 0; + f64 timeout; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "trace-type 0x%x", &trace_type)) + ; + else if (unformat (input, "trace-elts %d", &num_elts)) + ; + else if (unformat (input, "trace-tsp %d", &trace_tsp)) + ; + else if (unformat (input, "node-id 0x%x", &node_id)) + ; + else if (unformat (input, "app-data 0x%x", &app_data)) + ; + + else + break; + } + + + M (TRACE_PROFILE_ADD, trace_profile_add); + + mp->trace_type = trace_type; + mp->trace_tsp = trace_tsp; + mp->node_id = htonl (node_id); + mp->app_data = htonl (app_data); + mp->num_elts = num_elts; + + S; + W; + + return (rv); +} + + + +static int +api_trace_profile_del (vat_main_t * vam) +{ + trace_test_main_t *sm = &trace_test_main; + vl_api_trace_profile_del_t *mp; + f64 timeout; + + M (TRACE_PROFILE_DEL, trace_profile_del); + S; + W; + return 0; +} + +static int +api_trace_profile_show_config (vat_main_t * vam) +{ + trace_test_main_t *sm = &trace_test_main; + vl_api_trace_profile_show_config_t *mp; + f64 timeout; + M (TRACE_PROFILE_SHOW_CONFIG, trace_profile_show_config); + S; + W; + return 0; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(trace_profile_add, ""\ + "trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts trace-tsp <0|1|2|3> node-id app-data ") \ +_(trace_profile_del, "[id ]") \ +_(trace_profile_show_config, "[id ]") + + +void +vat_api_hookup (vat_main_t * vam) +{ + trace_test_main_t *sm = &trace_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * +vat_plugin_register (vat_main_t * vam) +{ + trace_test_main_t *sm = &trace_test_main; + u8 *name; + + sm->vat_main = vam; + + name = format (0, "ioam_trace_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~ 0) + vat_api_hookup (vam); + + vec_free (name); + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-trace/trace_util.c b/src/plugins/ioam/lib-trace/trace_util.c new file mode 100644 index 00000000..5c7f1eef --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace_util.c @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include "trace_util.h" + +trace_main_t trace_main; + +static int +trace_profile_cleanup (trace_profile * profile) +{ + + memset (profile, 0, sizeof (trace_profile)); + profile->trace_tsp = TSP_MICROSECONDS; /* Micro seconds */ + ip6_trace_profile_cleanup (); /* lib-trace_TODO: Remove this once IOAM-IPv6 transport is a plugin */ + return 0; + +} + +static int +trace_main_profiles_reset (void) +{ + int rv; + + trace_main_t *sm = &trace_main; + rv = trace_profile_cleanup (&(sm->profile)); + return (rv); +} + +int +trace_util_init (void) +{ + int rv; + + rv = trace_main_profiles_reset (); + return (rv); +} + + +int +trace_profile_create (trace_profile * profile, u8 trace_type, u8 num_elts, + u32 trace_tsp, u32 node_id, u32 app_data) +{ + + if (!trace_type || !num_elts || !(node_id)) + { + return (-1); + } + if (profile && !profile->valid) + { + //rv = trace_profile_cleanup (profile); + profile->trace_type = trace_type; + profile->num_elts = num_elts; + profile->trace_tsp = trace_tsp; + profile->node_id = node_id; + profile->app_data = app_data; + profile->valid = 1; + + /* lib-trace_TODO: Remove this once IOAM-IPv6 transport is a plugin */ + ip6_trace_profile_setup (); + return (0); + } + + return (-1); +} + + + +clib_error_t * +clear_trace_profile_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + + trace_main_profiles_reset (); + return 0; +} + +void +clear_trace_profiles (void) +{ + clear_trace_profile_command_fn (0, 0, 0); +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND(clear_trace_profile_command) = +{ +.path = "clear ioam-trace profile", +.short_help = "clear ioam-trace profile [|all]", +.function = clear_trace_profile_command_fn, +}; +/* *INDENT-ON* */ + +static clib_error_t * +set_trace_profile_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + u8 trace_type = 0; + u8 num_elts = 0; + u32 node_id = 0; + u32 app_data = 0; + u32 trace_tsp = 0; + trace_profile *profile = NULL; + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "trace-type 0x%x", &trace_type)); + else if (unformat (input, "trace-elts %d", &num_elts)); + else if (unformat (input, "trace-tsp %d", &trace_tsp)); + else if (unformat (input, "node-id 0x%x", &node_id)); + else if (unformat (input, "app-data 0x%x", &app_data)); + else + break; + } + profile = trace_profile_find (); + if (profile) + { + trace_profile_create (profile, trace_type, num_elts, trace_tsp, + node_id, app_data); + } + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (set_trace_profile_command, static) = +{ +.path = "set ioam-trace profile", +.short_help = "set ioam-trace \ + trace-type <0x1f|0x3|0x9|0x11|0x19> trace-elts trace-tsp <0|1|2|3> \ + node-id app-data ", +.function = set_trace_profile_command_fn, +}; +/* *INDENT-ON* */ + +static clib_error_t * +show_trace_profile_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + trace_profile *p = NULL; + u8 *s = 0; + p = trace_profile_find (); + if (!(p && p->valid)) + { + s = format (s, "\nTrace configuration not valid\n"); + vlib_cli_output (vm, "%v", s); + vec_free (s); + return 0; + } + s = format (s, " HOP BY HOP OPTIONS - TRACE CONFIG - \n"); + s = format (s, " Trace Type : 0x%x (%d)\n", + p->trace_type, p->trace_type); + s = + format (s, " Trace timestamp precision : %d (%s)\n", + p->trace_tsp, + (p->trace_tsp == + TSP_SECONDS) ? "Seconds" : ((p->trace_tsp == + TSP_MILLISECONDS) ? + "Milliseconds" + : (((p->trace_tsp == + TSP_MICROSECONDS) ? + "Microseconds" : + "Nanoseconds")))); + s = format (s, " Num of trace nodes : %d\n", p->num_elts); + s = + format (s, " Node-id : 0x%x (%d)\n", + p->node_id, p->node_id); + s = + format (s, " App Data : 0x%x (%d)\n", + p->app_data, p->app_data); + vlib_cli_output (vm, "%v", s); + vec_free (s); + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (show_trace_profile_command, static) = +{ +.path = "show ioam-trace profile", +.short_help = "show ioam-trace profile", +.function = show_trace_profile_command_fn, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-trace/trace_util.h b/src/plugins/ioam/lib-trace/trace_util.h new file mode 100644 index 00000000..556f07ee --- /dev/null +++ b/src/plugins/ioam/lib-trace/trace_util.h @@ -0,0 +1,247 @@ +/* + * trace_util.h -- Trace Profile Utility header + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef include_vnet_trace_util_h +#define include_vnet_trace_util_h + +#define debug_ioam debug_ioam_fn + + +/** + * Usage: + * + * On any node that participates in iOAM Trace. + * + * Step 1: Initialize this library by calling trace_init() + * Step 2: Setup a trace profile that contains all the parameters needed to compute cumulative: + * Call these functions: + * trace_profile_find + * trace_profile_create + * Step 2a: On initial node enable the profile to be used: + * trace_profile_set_active / trace_profile_get_active will return the profile + * Step 4: TBD + * trace_validate + * + */ + +typedef struct trace_profile_ +{ + u8 valid:1; + u8 trace_type; + u8 num_elts; + /* Configured node-id */ + u32 node_id; + u32 app_data; + u32 trace_tsp; +} trace_profile; + +typedef struct +{ + /* Name of the default profile list in use */ + trace_profile profile; + + /* API message ID base */ + u16 msg_id_base; + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; +} trace_main_t; + +extern trace_main_t trace_main; + +/* + * Initialize Trace profile + */ +int trace_util_init (void); + + +/* + * Find a trace profile + */ + +always_inline trace_profile * +trace_profile_find (void) +{ + trace_main_t *sm = &trace_main; + + return (&(sm->profile)); +} + + +/* setup and clean up profile */ +int trace_profile_create (trace_profile * profile, u8 trace_type, u8 num_elts, + u32 trace_tsp, u32 node_id, u32 app_data); + +void clear_trace_profiles (void); + + + +#define BIT_TTL_NODEID (1<<0) +#define BIT_ING_INTERFACE (1<<1) +#define BIT_EGR_INTERFACE (1<<2) +#define BIT_TIMESTAMP (1<<3) +#define BIT_APPDATA (1<<4) +#define TRACE_TYPE_MASK 0x1F /* Mask of all above bits */ + +/* + 0x00011111 iOAM-trace-type is 0x00011111 then the format of node + data is: + + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Hop_Lim | node_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | ingress_if_id | egress_if_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + timestamp + + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | app_data | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +*/ +#define TRACE_TYPE_IF_TS_APP 0x1f +typedef struct +{ + u32 ttl_node_id; + u16 ingress_if; + u16 egress_if; + u32 timestamp; + u32 app_data; +} ioam_trace_if_ts_app_t; + +/* + 0x00000111 iOAM-trace-type is 0x00000111 then the format is: + + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Hop_Lim | node_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | ingress_if_id | egress_if_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +*/ + +#define TRACE_TYPE_IF 0x03 +typedef struct +{ + u32 ttl_node_id; + u16 ingress_if; + u16 egress_if; +} ioam_trace_if_t; + +/* + 0x00001001 iOAM-trace-type is 0x00001001 then the format is: + + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Hop_Lim | node_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + timestamp + + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +*/ + +#define TRACE_TYPE_TS 0x09 +typedef struct +{ + u32 ttl_node_id; + u32 timestamp; +} ioam_trace_ts_t; + +/* + 0x00010001 iOAM-trace-type is 0x00010001 then the format is: + + + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Hop_Lim | node_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | app_data | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +*/ + + +#define TRACE_TYPE_APP 0x11 +typedef struct +{ + u32 ttl_node_id; + u32 app_data; +} ioam_trace_app_t; + +/* + + 0x00011001 iOAM-trace-type is 0x00011001 then the format is: + + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Hop_Lim | node_id | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + timestamp + + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | app_data | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +*/ + +#define TRACE_TYPE_TS_APP 0x19 +typedef struct +{ + u32 ttl_node_id; + u32 timestamp; + u32 app_data; +} ioam_trace_ts_app_t; + + + +static inline u8 +fetch_trace_data_size (u8 trace_type) +{ + u8 trace_data_size = 0; + + if (trace_type == TRACE_TYPE_IF_TS_APP) + trace_data_size = sizeof (ioam_trace_if_ts_app_t); + else if (trace_type == TRACE_TYPE_IF) + trace_data_size = sizeof (ioam_trace_if_t); + else if (trace_type == TRACE_TYPE_TS) + trace_data_size = sizeof (ioam_trace_ts_t); + else if (trace_type == TRACE_TYPE_APP) + trace_data_size = sizeof (ioam_trace_app_t); + else if (trace_type == TRACE_TYPE_TS_APP) + trace_data_size = sizeof (ioam_trace_ts_app_t); + + return trace_data_size; +} + +int ioam_trace_get_sizeof_handler (u32 * result); +int ip6_trace_profile_setup (void); +int ip6_trace_profile_cleanup (void); + +#define TSP_SECONDS 0 +#define TSP_MILLISECONDS 1 +#define TSP_MICROSECONDS 2 +#define TSP_NANOSECONDS 3 + +#endif + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/ioam_decap.c b/src/plugins/ioam/lib-vxlan-gpe/ioam_decap.c new file mode 100644 index 00000000..fd308657 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/ioam_decap.c @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Statistics (not really errors) */ +#define foreach_vxlan_gpe_decap_ioam_v4_error \ +_(DECAPSULATED, "good packets decapsulated") + +static char *vxlan_gpe_decap_ioam_v4_error_strings[] = { +#define _(sym,string) string, + foreach_vxlan_gpe_decap_ioam_v4_error +#undef _ +}; + +typedef enum +{ +#define _(sym,str) VXLAN_GPE_DECAP_IOAM_V4_ERROR_##sym, + foreach_vxlan_gpe_decap_ioam_v4_error +#undef _ + VXLAN_GPE_DECAP_IOAM_V4_N_ERROR, +} vxlan_gpe_decap_ioam_v4_error_t; + + +always_inline void +vxlan_gpe_decap_ioam_v4_two_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vxlan_gpe_main_t * ngm, + vlib_buffer_t * b0, vlib_buffer_t * b1, + u32 * next0, u32 * next1) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + next0[0] = next1[0] = hm->decap_v4_next_override; + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, &next0[0], + VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, + 0 /* use_adj */ ); + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b1, &next0[1], + VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, + 0 /* use_adj */ ); +} + + + +static uword +vxlan_gpe_decap_ioam (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, u8 is_ipv6) +{ + u32 n_left_from, next_index, *from, *to_next; + vxlan_gpe_main_t *ngm = &vxlan_gpe_main; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + u32 next0, next1; + + next0 = next1 = hm->decap_v4_next_override; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p2, *p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); + } + + bi0 = from[0]; + bi1 = from[1]; + to_next[0] = bi0; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_to_next -= 2; + n_left_from -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + + vlib_buffer_advance (b0, + -(word) (sizeof (udp_header_t) + + sizeof (ip4_header_t) + + sizeof (vxlan_gpe_header_t))); + vlib_buffer_advance (b1, + -(word) (sizeof (udp_header_t) + + sizeof (ip4_header_t) + + sizeof (vxlan_gpe_header_t))); + + vxlan_gpe_decap_ioam_v4_two_inline (vm, node, ngm, b0, b1, + &next0, &next1); + + + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, next0, + next1); + + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + vxlan_gpe_ioam_v4_trace_t *tr = vlib_add_trace (vm, node, b0, + sizeof (*tr)); + } + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = hm->decap_v4_next_override; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + + vlib_buffer_advance (b0, + -(word) (sizeof (udp_header_t) + + sizeof (ip4_header_t) + + sizeof (vxlan_gpe_header_t))); + + next0 = hm->decap_v4_next_override; + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, + &next0, + VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, + 0 /* use_adj */ ); + + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + vxlan_gpe_ioam_v4_trace_t *tr = vlib_add_trace (vm, node, b0, + sizeof (*tr)); + } + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return from_frame->n_vectors; +} + + +static uword +vxlan_gpe_decap_ioam_v4 (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return vxlan_gpe_decap_ioam (vm, node, from_frame, 0); +} + + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (vxlan_gpe_decap_ioam_v4_node) = { + .function = vxlan_gpe_decap_ioam_v4, + .name = "vxlan-gpe-decap-ioam-v4", + .vector_size = sizeof (u32), + .format_trace = format_vxlan_gpe_ioam_v4_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(vxlan_gpe_decap_ioam_v4_error_strings), + .error_strings = vxlan_gpe_decap_ioam_v4_error_strings, + + .n_next_nodes = VXLAN_GPE_DECAP_IOAM_V4_N_NEXT, + + .next_nodes = { + [VXLAN_GPE_DECAP_IOAM_V4_NEXT_POP] = "vxlan-gpe-pop-ioam-v4", + [VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP] = "error-drop", + }, +}; +/* *INDENT-ON* */ + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/ioam_encap.c b/src/plugins/ioam/lib-vxlan-gpe/ioam_encap.c new file mode 100644 index 00000000..4b18bfea --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/ioam_encap.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Statistics (not really errors) */ +#define foreach_vxlan_gpe_encap_ioam_v4_error \ +_(ENCAPSULATED, "good packets encapsulated") + +static char *vxlan_gpe_encap_ioam_v4_error_strings[] = { +#define _(sym,string) string, + foreach_vxlan_gpe_encap_ioam_v4_error +#undef _ +}; + +typedef enum +{ +#define _(sym,str) VXLAN_GPE_ENCAP_IOAM_V4_ERROR_##sym, + foreach_vxlan_gpe_encap_ioam_v4_error +#undef _ + VXLAN_GPE_ENCAP_IOAM_V4_N_ERROR, +} vxlan_gpe_encap_ioam_v4_error_t; + +typedef enum +{ + VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP, + VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, + VXLAN_GPE_ENCAP_IOAM_V4_N_NEXT +} vxlan_gpe_encap_ioam_v4_next_t; + + +always_inline void +vxlan_gpe_encap_ioam_v4_two_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vxlan_gpe_main_t * ngm, + vlib_buffer_t * b0, vlib_buffer_t * b1, + u32 * next0, u32 * next1) +{ + next0[0] = next1[0] = VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP; + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, next0, + VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, + 0 /* use_adj */ ); + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b1, next1, + VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, + 0 /* use_adj */ ); +} + + +static uword +vxlan_gpe_encap_ioam_v4 (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + u32 n_left_from, next_index, *from, *to_next; + vxlan_gpe_main_t *ngm = &vxlan_gpe_main; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + u32 next0, next1; + + next0 = next1 = VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p2, *p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); + } + + bi0 = from[0]; + bi1 = from[1]; + to_next[0] = bi0; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_to_next -= 2; + n_left_from -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + vxlan_gpe_encap_ioam_v4_two_inline (vm, node, ngm, b0, b1, + &next0, &next1); + + + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, next0, + next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, b0, + &next0, + VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP, + 0 /* use_adj */ ); + + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + vxlan_gpe_ioam_v4_trace_t *tr = vlib_add_trace (vm, node, b0, + sizeof (*tr)); + } + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return from_frame->n_vectors; +} + + + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (vxlan_gpe_encap_ioam_v4_node) = { + .function = vxlan_gpe_encap_ioam_v4, + .name = "vxlan-gpe-encap-ioam-v4", + .vector_size = sizeof (u32), + .format_trace = format_vxlan_gpe_ioam_v4_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(vxlan_gpe_encap_ioam_v4_error_strings), + .error_strings = vxlan_gpe_encap_ioam_v4_error_strings, + + .n_next_nodes = VXLAN_GPE_ENCAP_IOAM_V4_N_NEXT, + + .next_nodes = { + [VXLAN_GPE_ENCAP_IOAM_V4_NEXT_IP4_LOOKUP] = "ip4-lookup", + [VXLAN_GPE_ENCAP_IOAM_V4_NEXT_DROP] = "error-drop", + }, +}; +/* *INDENT-ON* */ + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c b/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c new file mode 100644 index 00000000..55c33b14 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include + +/* Statistics (not really errors) */ +#define foreach_vxlan_gpe_pop_ioam_v4_error \ +_(POPPED, "good packets popped") + +static char *vxlan_gpe_pop_ioam_v4_error_strings[] = { +#define _(sym,string) string, + foreach_vxlan_gpe_pop_ioam_v4_error +#undef _ +}; + +typedef enum +{ +#define _(sym,str) VXLAN_GPE_POP_IOAM_V4_ERROR_##sym, + foreach_vxlan_gpe_pop_ioam_v4_error +#undef _ + VXLAN_GPE_POP_IOAM_V4_N_ERROR, +} vxlan_gpe_pop_ioam_v4_error_t; + +typedef struct +{ + ioam_trace_t fmt_trace; +} vxlan_gpe_pop_ioam_v4_trace_t; + + +u8 * +format_vxlan_gpe_pop_ioam_v4_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + vxlan_gpe_pop_ioam_v4_trace_t *t1 + = va_arg (*args, vxlan_gpe_pop_ioam_v4_trace_t *); + ioam_trace_t *t = &(t1->fmt_trace); + vxlan_gpe_ioam_option_t *fmt_trace0; + vxlan_gpe_ioam_option_t *opt0, *limit0; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + u8 type0; + + fmt_trace0 = (vxlan_gpe_ioam_option_t *) t->option_data; + + s = format (s, "VXLAN_GPE_IOAM_POP: next_index %d len %d traced %d", + t->next_index, fmt_trace0->length, t->trace_len); + + opt0 = (vxlan_gpe_ioam_option_t *) (fmt_trace0 + 1); + limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) fmt_trace0) + t->trace_len; + + while (opt0 < limit0) + { + type0 = opt0->type; + switch (type0) + { + case 0: /* Pad, just stop */ + opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; + break; + + default: + if (hm->trace[type0]) + { + s = (*hm->trace[type0]) (s, opt0); + } + else + { + s = + format (s, "\n unrecognized option %d length %d", type0, + opt0->length); + } + opt0 = + (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + + sizeof (vxlan_gpe_ioam_option_t)); + break; + } + } + + return s; +} + +always_inline void +vxlan_gpe_ioam_pop_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_buffer_t * b0) +{ + ip4_header_t *ip0; + udp_header_t *udp_hdr0; + vxlan_gpe_header_t *gpe_hdr0; + vxlan_gpe_ioam_hdr_t *gpe_ioam0; + + ip0 = vlib_buffer_get_current (b0); + + udp_hdr0 = (udp_header_t *) (ip0 + 1); + gpe_hdr0 = (vxlan_gpe_header_t *) (udp_hdr0 + 1); + gpe_ioam0 = (vxlan_gpe_ioam_hdr_t *) (gpe_hdr0 + 1); + + /* Pop the iOAM data */ + vlib_buffer_advance (b0, + (word) (sizeof (udp_header_t) + + sizeof (ip4_header_t) + + sizeof (vxlan_gpe_header_t) + + gpe_ioam0->length)); + + return; +} + + + +always_inline void +vxlan_gpe_pop_ioam_v4_one_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vxlan_gpe_main_t * ngm, + vlib_buffer_t * b0, u32 * next0) +{ + CLIB_UNUSED (ip4_header_t * ip0); + CLIB_UNUSED (udp_header_t * udp_hdr0); + CLIB_UNUSED (vxlan_gpe_header_t * gpe_hdr0); + CLIB_UNUSED (vxlan_gpe_ioam_hdr_t * gpe_ioam0); + CLIB_UNUSED (vxlan_gpe_ioam_option_t * opt0); + CLIB_UNUSED (vxlan_gpe_ioam_option_t * limit0); + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + + /* Pop the iOAM header */ + ip0 = vlib_buffer_get_current (b0); + udp_hdr0 = (udp_header_t *) (ip0 + 1); + gpe_hdr0 = (vxlan_gpe_header_t *) (udp_hdr0 + 1); + gpe_ioam0 = (vxlan_gpe_ioam_hdr_t *) (gpe_hdr0 + 1); + opt0 = (vxlan_gpe_ioam_option_t *) (gpe_ioam0 + 1); + limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) gpe_ioam0 + gpe_ioam0->length); + + /* + * Basic validity checks + */ + if (gpe_ioam0->length > clib_net_to_host_u16 (ip0->length)) + { + next0[0] = VXLAN_GPE_INPUT_NEXT_DROP; + goto trace00; + } + + /* Scan the set of h-b-h options, process ones that we understand */ + while (opt0 < limit0) + { + u8 type0; + type0 = opt0->type; + switch (type0) + { + case 0: /* Pad1 */ + opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; + continue; + case 1: /* PadN */ + break; + default: + if (hm->pop_options[type0]) + { + if ((*hm->pop_options[type0]) (ip0, opt0) < 0) + { + next0[0] = VXLAN_GPE_INPUT_NEXT_DROP; + goto trace00; + } + } + break; + } + opt0 = + (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + + sizeof (vxlan_gpe_ioam_hdr_t)); + } + + + next0[0] = + (gpe_ioam0->protocol < VXLAN_GPE_PROTOCOL_MAX) ? + ngm-> + decap_next_node_list[gpe_ioam0->protocol] : VXLAN_GPE_INPUT_NEXT_DROP; + +trace00: + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + vxlan_gpe_pop_ioam_v4_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + u32 trace_len = gpe_ioam0->length; + t->fmt_trace.next_index = next0[0]; + /* Capture the h-b-h option verbatim */ + trace_len = + trace_len < + ARRAY_LEN (t->fmt_trace. + option_data) ? trace_len : ARRAY_LEN (t->fmt_trace. + option_data); + t->fmt_trace.trace_len = trace_len; + clib_memcpy (&(t->fmt_trace.option_data), gpe_ioam0, trace_len); + } + + /* Remove the iOAM header inside the VxLAN-GPE header */ + vxlan_gpe_ioam_pop_v4 (vm, node, b0); + return; +} + +always_inline void +vxlan_gpe_pop_ioam_v4_two_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vxlan_gpe_main_t * ngm, + vlib_buffer_t * b0, vlib_buffer_t * b1, + u32 * next0, u32 * next1) +{ + + vxlan_gpe_pop_ioam_v4_one_inline (vm, node, ngm, b0, next0); + vxlan_gpe_pop_ioam_v4_one_inline (vm, node, ngm, b1, next1); +} + + + +static uword +vxlan_gpe_pop_ioam (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, u8 is_ipv6) +{ + u32 n_left_from, next_index, *from, *to_next; + vxlan_gpe_main_t *ngm = &vxlan_gpe_main; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + u32 next0, next1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p2, *p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); + } + + bi0 = from[0]; + bi1 = from[1]; + to_next[0] = bi0; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_to_next -= 2; + n_left_from -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + vxlan_gpe_pop_ioam_v4_two_inline (vm, node, ngm, b0, b1, &next0, + &next1); + + + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, next0, + next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + vxlan_gpe_pop_ioam_v4_one_inline (vm, node, ngm, b0, &next0); + + + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return from_frame->n_vectors; +} + + +static uword +vxlan_gpe_pop_ioam_v4 (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * from_frame) +{ + return vxlan_gpe_pop_ioam (vm, node, from_frame, 0); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (vxlan_gpe_pop_ioam_v4_node) = { + .function = vxlan_gpe_pop_ioam_v4, + .name = "vxlan-gpe-pop-ioam-v4", + .vector_size = sizeof (u32), + .format_trace = format_vxlan_gpe_pop_ioam_v4_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(vxlan_gpe_pop_ioam_v4_error_strings), + .error_strings = vxlan_gpe_pop_ioam_v4_error_strings, + + .n_next_nodes = VXLAN_GPE_INPUT_N_NEXT, + + .next_nodes = { +#define _(s,n) [VXLAN_GPE_INPUT_NEXT_##s] = n, + foreach_vxlan_gpe_input_next +#undef _ + }, +}; +/* *INDENT-ON* */ + + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/ioam_transit.c b/src/plugins/ioam/lib-vxlan-gpe/ioam_transit.c new file mode 100644 index 00000000..b42c357c --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/ioam_transit.c @@ -0,0 +1,188 @@ + /* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Statistics (not really errors) */ +#define foreach_vxlan_gpe_transit_ioam_error \ +_(ENCAPSULATED, "good packets encapsulated") + +static char *vxlan_gpe_transit_ioam_error_strings[] = { +#define _(sym,string) string, + foreach_vxlan_gpe_transit_ioam_error +#undef _ +}; + +typedef enum +{ +#define _(sym,str) VXLAN_GPE_TRANSIT_IOAM_ERROR_##sym, + foreach_vxlan_gpe_transit_ioam_error +#undef _ + VXLAN_GPE_TRANSIT_IOAM_N_ERROR, +} vxlan_gpe_transit_ioam_error_t; + +typedef enum +{ + VXLAN_GPE_TRANSIT_IOAM_NEXT_OUTPUT, + VXLAN_GPE_TRANSIT_IOAM_NEXT_DROP, + VXLAN_GPE_TRANSIT_IOAM_N_NEXT +} vxlan_gpe_transit_ioam_next_t; + + +/* *INDENT-OFF* */ +VNET_FEATURE_INIT (vxlan_gpe_transit_ioam, static) = +{ + .arc_name = "ip4-output", + .node_name = "vxlan-gpe-transit-ioam", + .runs_before = VNET_FEATURES ("interface-output"), +}; +/* *INDENT-ON* */ + +static uword +vxlan_gpe_transit_ioam (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * from_frame) +{ + u32 n_left_from, next_index, *from, *to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = VXLAN_GPE_TRANSIT_IOAM_NEXT_OUTPUT; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + ip4_header_t *ip0; + u32 iph_offset = 0; + + b0 = vlib_get_buffer (vm, bi0); + iph_offset = vnet_buffer (b0)->ip.save_rewrite_length; + ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + + iph_offset); + + /* just forward non ipv4 packets */ + if (PREDICT_FALSE + ((ip0->ip_version_and_header_length & 0xF0) == 0x40)) + { + /* ipv4 packets */ + udp_header_t *udp_hdr0 = (udp_header_t *) (ip0 + 1); + if (PREDICT_FALSE + ((ip0->protocol == IP_PROTOCOL_UDP) && + (clib_net_to_host_u16 (udp_hdr0->dst_port) == + UDP_DST_PORT_vxlan_gpe))) + { + + /* Check the iOAM header */ + vxlan_gpe_header_t *gpe_hdr0 = + (vxlan_gpe_header_t *) (udp_hdr0 + 1); + + if (PREDICT_FALSE + (gpe_hdr0->protocol == VXLAN_GPE_PROTOCOL_IOAM)) + { + uword *t = NULL; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + fib_prefix_t key4; + memset (&key4, 0, sizeof (key4)); + key4.fp_proto = FIB_PROTOCOL_IP4; + key4.fp_addr.ip4.as_u32 = ip0->dst_address.as_u32; + t = hash_get_mem (hm->dst_by_ip4, &key4); + if (t) + { + + + vlib_buffer_advance (b0, + (word) (sizeof + (ethernet_header_t))); + vxlan_gpe_encap_decap_ioam_v4_one_inline (vm, node, + b0, + &next0, + VXLAN_GPE_TRANSIT_IOAM_NEXT_DROP, + 1 + /* use_adj */ + ); + vlib_buffer_advance (b0, + -(word) (sizeof + (ethernet_header_t))); + } + } + } + } + + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return from_frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (vxlan_gpe_transit_ioam_node) = { + .function = vxlan_gpe_transit_ioam, + .name = "vxlan-gpe-transit-ioam", + .vector_size = sizeof (u32), + .format_trace = format_vxlan_gpe_ioam_v4_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(vxlan_gpe_transit_ioam_error_strings), + .error_strings = vxlan_gpe_transit_ioam_error_strings, + + .n_next_nodes = VXLAN_GPE_TRANSIT_IOAM_N_NEXT, + + .next_nodes = { + [VXLAN_GPE_TRANSIT_IOAM_NEXT_OUTPUT] = "interface-output", + [VXLAN_GPE_TRANSIT_IOAM_NEXT_DROP] = "error-drop", + }, + +}; +/* *INDENT-ON* */ + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api b/src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api new file mode 100644 index 00000000..056529a4 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api @@ -0,0 +1,181 @@ +/* Hey Emacs use -*- mode: C -*- */ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** \brief iOAM Over VxLAN-GPE - Set iOAM transport for VxLAN-GPE + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param id - profile id + @param trace_ppc - Trace PPC (none/encap/decap) + @param pow_enable - Proof of Work enabled or not flag + @param trace_enable - iOAM Trace enabled or not flag + +*/ +define vxlan_gpe_ioam_enable { + u32 client_index; + u32 context; + u16 id; + u8 trace_ppc; + u8 pow_enable; + u8 trace_enable; +}; + +/** \brief iOAM Over VxLAN-GPE - Set iOAM transport for VXLAN-GPE reply + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define vxlan_gpe_ioam_enable_reply { + u32 context; + i32 retval; +}; + + +/** \brief iOAM for VxLAN-GPE disable + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param id - profile id +*/ +define vxlan_gpe_ioam_disable +{ + u32 client_index; + u32 context; + u16 id; +}; + +/** \brief vxlan_gpe_ioam disable response + @param context - sender context, to match reply w/ request + @param retval - return value for request +*/ +define vxlan_gpe_ioam_disable_reply +{ + u32 context; + i32 retval; +}; + +/** \brief Enable iOAM for a VNI (VXLAN-GPE) + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param vni - VXLAN-GPE VNI + @param local - IPv4/6 Address of the local VTEP + @param remote - IPv4/6 Address of the remote VTEP + +*/ +define vxlan_gpe_ioam_vni_enable { + u32 client_index; + u32 context; + u32 vni; + u8 local[16]; + u8 remote[16]; + u8 is_ipv6; +}; + +/** \brief Reply to enable iOAM for a VNI (VXLAN-GPE) + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return value for request + +*/ +define vxlan_gpe_ioam_vni_enable_reply { + u32 client_index; + u32 context; + i32 retval; +}; + +/** \brief Disable iOAM for a VNI (VXLAN-GPE) + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param vni - VXLAN-GPE VNI + @param local - IPv4/6 Address of the local VTEP + @param remote - IPv4/6 Address of the remote VTEP + +*/ +define vxlan_gpe_ioam_vni_disable { + u32 client_index; + u32 context; + u32 vni; + u8 local[16]; + u8 remote[16]; + u8 is_ipv6; +}; + +/** \brief Reply to disable iOAM for a VNI (VXLAN-GPE) + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return value for request + +*/ +define vxlan_gpe_ioam_vni_disable_reply { + u32 client_index; + u32 context; + i32 retval; +}; + + +/** \brief Enable iOAM for a VXLAN-GPE transit + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param dst_addr - IPv4/6 Address of the local VTEP + @param outer_fib_index- FIB index + +*/ +define vxlan_gpe_ioam_transit_enable { + u32 client_index; + u32 context; + u32 outer_fib_index; + u8 dst_addr[16]; + u8 is_ipv6; +}; + +/** \brief Reply to enable iOAM for VXLAN-GPE transit + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return value for request + +*/ +define vxlan_gpe_ioam_transit_enable_reply { + u32 client_index; + u32 context; + i32 retval; +}; + +/** \brief Disable iOAM for VXLAN-GPE transit + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param dst_addr - IPv4/6 Address of the local VTEP + @param outer_fib_index- FIB index + +*/ +define vxlan_gpe_ioam_transit_disable { + u32 client_index; + u32 context; + u32 outer_fib_index; + u8 dst_addr[16]; + u8 is_ipv6; +}; + +/** \brief Reply to disable iOAM for VXLAN-GPE transit + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return value for request + +*/ +define vxlan_gpe_ioam_transit_disable_reply { + u32 client_index; + u32 context; + i32 retval; +}; + + diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h new file mode 100644 index 00000000..06fa0d2c --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_all_api_h.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_api.c b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_api.c new file mode 100644 index 00000000..68752365 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_api.c @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * vxlan_gpe_api.c - iOAM VxLAN-GPE related APIs to create + * and maintain profiles + *------------------------------------------------------------------ + */ + +#include +#include +#include + +#include +#include +#include + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define VXLAN_GPE_REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +/* *INDENT-OFF* */ +#define VXLAN_GPE_REPLY_MACRO2(t, body) \ +do { \ + unix_shared_memory_queue_t * q; \ + rv = vl_msg_api_pd_handler (mp, rv); \ + q = vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + do {body;} while (0); \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); +/* *INDENT-ON* */ + +/* List of message types that this plugin understands */ + +#define foreach_vxlan_gpe_plugin_api_msg \ +_(VXLAN_GPE_IOAM_ENABLE, vxlan_gpe_ioam_enable) \ +_(VXLAN_GPE_IOAM_DISABLE, vxlan_gpe_ioam_disable) \ +_(VXLAN_GPE_IOAM_VNI_ENABLE, vxlan_gpe_ioam_vni_enable) \ +_(VXLAN_GPE_IOAM_VNI_DISABLE, vxlan_gpe_ioam_vni_disable) \ +_(VXLAN_GPE_IOAM_TRANSIT_ENABLE, vxlan_gpe_ioam_transit_enable) \ +_(VXLAN_GPE_IOAM_TRANSIT_DISABLE, vxlan_gpe_ioam_transit_disable) \ + + +static void vl_api_vxlan_gpe_ioam_enable_t_handler + (vl_api_vxlan_gpe_ioam_enable_t * mp) +{ + int rv = 0; + vl_api_vxlan_gpe_ioam_enable_reply_t *rmp; + clib_error_t *error; + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + + /* Ignoring the profile id as currently a single profile + * is supported */ + error = + vxlan_gpe_ioam_enable (mp->trace_enable, mp->pow_enable, mp->trace_ppc); + if (error) + { + clib_error_report (error); + rv = clib_error_get_code (error); + } + + VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_ENABLE_REPLY); +} + +static void vl_api_vxlan_gpe_ioam_disable_t_handler + (vl_api_vxlan_gpe_ioam_disable_t * mp) +{ + int rv = 0; + vl_api_vxlan_gpe_ioam_disable_reply_t *rmp; + clib_error_t *error; + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + + /* Ignoring the profile id as currently a single profile + * is supported */ + error = vxlan_gpe_ioam_disable (0, 0, 0); + if (error) + { + clib_error_report (error); + rv = clib_error_get_code (error); + } + + VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_DISABLE_REPLY); +} + +static void vl_api_vxlan_gpe_ioam_vni_enable_t_handler + (vl_api_vxlan_gpe_ioam_vni_enable_t * mp) +{ + int rv = 0; + vl_api_vxlan_gpe_ioam_vni_enable_reply_t *rmp; + clib_error_t *error; + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + vxlan4_gpe_tunnel_key_t key4; + uword *p = NULL; + vxlan_gpe_main_t *gm = &vxlan_gpe_main; + vxlan_gpe_tunnel_t *t = 0; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + u32 vni; + + + if (!mp->is_ipv6) + { + clib_memcpy (&key4.local, &mp->local, sizeof (key4.local)); + clib_memcpy (&key4.remote, &mp->remote, sizeof (key4.remote)); + vni = clib_net_to_host_u32 (mp->vni); + key4.vni = clib_host_to_net_u32 (vni << 8); + key4.pad = 0; + + p = hash_get_mem (gm->vxlan4_gpe_tunnel_by_key, &key4); + } + else + { + return; + } + + if (!p) + return; + + t = pool_elt_at_index (gm->tunnels, p[0]); + + error = vxlan_gpe_ioam_set (t, hm->has_trace_option, + hm->has_pot_option, + hm->has_ppc_option, mp->is_ipv6); + + + if (error) + { + clib_error_report (error); + rv = clib_error_get_code (error); + } + + VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_VNI_ENABLE_REPLY); +} + + +static void vl_api_vxlan_gpe_ioam_vni_disable_t_handler + (vl_api_vxlan_gpe_ioam_vni_disable_t * mp) +{ + int rv = 0; + vl_api_vxlan_gpe_ioam_vni_enable_reply_t *rmp; + clib_error_t *error; + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + vxlan4_gpe_tunnel_key_t key4; + uword *p = NULL; + vxlan_gpe_main_t *gm = &vxlan_gpe_main; + vxlan_gpe_tunnel_t *t = 0; + u32 vni; + + + if (!mp->is_ipv6) + { + clib_memcpy (&key4.local, &mp->local, sizeof (key4.local)); + clib_memcpy (&key4.remote, &mp->remote, sizeof (key4.remote)); + vni = clib_net_to_host_u32 (mp->vni); + key4.vni = clib_host_to_net_u32 (vni << 8); + key4.pad = 0; + + p = hash_get_mem (gm->vxlan4_gpe_tunnel_by_key, &key4); + } + else + { + return; + } + + if (!p) + return; + + t = pool_elt_at_index (gm->tunnels, p[0]); + + error = vxlan_gpe_ioam_clear (t, 0, 0, 0, 0); + + + if (error) + { + clib_error_report (error); + rv = clib_error_get_code (error); + } + + + VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_VNI_DISABLE_REPLY); +} + +static void vl_api_vxlan_gpe_ioam_transit_enable_t_handler + (vl_api_vxlan_gpe_ioam_transit_enable_t * mp) +{ + int rv = 0; + vl_api_vxlan_gpe_ioam_transit_enable_reply_t *rmp; + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + ip46_address_t dst_addr; + + memset (&dst_addr.ip4, 0, sizeof (dst_addr.ip4)); + if (!mp->is_ipv6) + { + clib_memcpy (&dst_addr.ip4, &mp->dst_addr, sizeof (dst_addr.ip4)); + } + rv = vxlan_gpe_enable_disable_ioam_for_dest (sm->vlib_main, + dst_addr, + ntohl (mp->outer_fib_index), + mp->is_ipv6 ? 0 : 1, + 1 /* is_add */ ); + + VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_TRANSIT_ENABLE_REPLY); +} + +static void vl_api_vxlan_gpe_ioam_transit_disable_t_handler + (vl_api_vxlan_gpe_ioam_transit_disable_t * mp) +{ + int rv = 0; + vl_api_vxlan_gpe_ioam_transit_disable_reply_t *rmp; + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + ip46_address_t dst_addr; + + memset (&dst_addr.ip4, 0, sizeof (dst_addr.ip4)); + if (!mp->is_ipv6) + { + clib_memcpy (&dst_addr.ip4, &mp->dst_addr, sizeof (dst_addr.ip4)); + } + + rv = vxlan_gpe_ioam_disable_for_dest (sm->vlib_main, + dst_addr, + ntohl (mp->outer_fib_index), + mp->is_ipv6 ? 0 : 1); + VXLAN_GPE_REPLY_MACRO (VL_API_VXLAN_GPE_IOAM_TRANSIT_DISABLE_REPLY); +} + + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + clib_error_t *error = 0; + + sm->vlib_main = vm; + sm->vnet_main = h->vnet_main; + sm->unix_time_0 = (u32) time (0); /* Store starting time */ + sm->vlib_time_0 = vlib_time_now (vm); + return error; +} + +/* Set up the API message handling tables */ +static clib_error_t * +vxlan_gpe_plugin_api_hookup (vlib_main_t * vm) +{ + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vxlan_gpe_plugin_api_msg; +#undef _ + + return 0; +} + +static clib_error_t * +vxlan_gpe_init (vlib_main_t * vm) +{ + vxlan_gpe_ioam_main_t *sm = &vxlan_gpe_ioam_main; + clib_error_t *error = 0; + u8 *name; + u32 encap_node_index = vxlan_gpe_encap_ioam_v4_node.index; + u32 decap_node_index = vxlan_gpe_decap_ioam_v4_node.index; + vlib_node_t *vxlan_gpe_encap_node = NULL; + vlib_node_t *vxlan_gpe_decap_node = NULL; + uword next_node = 0; + + name = format (0, "ioam_vxlan_gpe_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + sm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + + error = vxlan_gpe_plugin_api_hookup (vm); + + /* Hook the ioam-encap node to vxlan-gpe-encap */ + vxlan_gpe_encap_node = vlib_get_node_by_name (vm, (u8 *) "vxlan-gpe-encap"); + sm->encap_v4_next_node = + vlib_node_add_next (vm, vxlan_gpe_encap_node->index, encap_node_index); + + vxlan_gpe_decap_node = + vlib_get_node_by_name (vm, (u8 *) "vxlan4-gpe-input"); + next_node = + vlib_node_add_next (vm, vxlan_gpe_decap_node->index, decap_node_index); + vxlan_gpe_register_decap_protocol (VXLAN_GPE_PROTOCOL_IOAM, next_node); + + vec_new (vxlan_gpe_ioam_sw_interface_t, pool_elts (sm->sw_interfaces)); + sm->dst_by_ip4 = hash_create_mem (0, sizeof (fib_prefix_t), sizeof (uword)); + + sm->dst_by_ip6 = hash_create_mem (0, sizeof (fib_prefix_t), sizeof (uword)); + + vxlan_gpe_ioam_interface_init (); + vec_free (name); + + return error; +} + +VLIB_INIT_FUNCTION (vxlan_gpe_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c new file mode 100644 index 00000000..6c04d9af --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c @@ -0,0 +1,773 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include + +vxlan_gpe_ioam_main_t vxlan_gpe_ioam_main; + +int +vxlan_gpe_ioam_set_rewrite (vxlan_gpe_tunnel_t * t, int has_trace_option, + int has_pot_option, int has_ppc_option, + u8 ipv6_set) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + u32 size; + vxlan_gpe_ioam_hdr_t *vxlan_gpe_ioam_hdr; + u8 *current; + u8 trace_data_size = 0; + u8 pot_data_size = 0; + + if (has_trace_option == 0 && has_pot_option == 0) + return -1; + + /* Work out how much space we need */ + size = sizeof (vxlan_gpe_ioam_hdr_t); + + if (has_trace_option + && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] != 0) + { + size += sizeof (vxlan_gpe_ioam_option_t); + size += hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE]; + } + if (has_pot_option + && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0) + { + size += sizeof (vxlan_gpe_ioam_option_t); + size += hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]; + } + + t->rewrite_size = size; + + if (!ipv6_set) + { + vxlan4_gpe_rewrite (t, size, VXLAN_GPE_PROTOCOL_IOAM, + hm->encap_v4_next_node); + vxlan_gpe_ioam_hdr = + (vxlan_gpe_ioam_hdr_t *) (t->rewrite + + sizeof (ip4_vxlan_gpe_header_t)); + } + else + { + vxlan6_gpe_rewrite (t, size, VXLAN_GPE_PROTOCOL_IOAM, + VXLAN_GPE_ENCAP_NEXT_IP6_LOOKUP); + vxlan_gpe_ioam_hdr = + (vxlan_gpe_ioam_hdr_t *) (t->rewrite + + sizeof (ip6_vxlan_gpe_header_t)); + } + + + vxlan_gpe_ioam_hdr->type = VXLAN_GPE_PROTOCOL_IOAM; + /* Length of the header in octets */ + vxlan_gpe_ioam_hdr->length = size; + vxlan_gpe_ioam_hdr->protocol = t->protocol; + current = (u8 *) vxlan_gpe_ioam_hdr + sizeof (vxlan_gpe_ioam_hdr_t); + + if (has_trace_option + && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] != 0) + { + if (0 != hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] (current, + &trace_data_size)) + return -1; + current += trace_data_size; + } + if (has_pot_option + && hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] != 0) + { + pot_data_size = + hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT]; + if (0 == + hm->add_options[VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT] + (current, &pot_data_size)) + current += pot_data_size; + } + + return 0; +} + +int +vxlan_gpe_ioam_clear_rewrite (vxlan_gpe_tunnel_t * t, int has_trace_option, + int has_pot_option, int has_ppc_option, + u8 ipv6_set) +{ + + t->rewrite_size = 0; + + if (!ipv6_set) + { + vxlan4_gpe_rewrite (t, 0, 0, VXLAN_GPE_ENCAP_NEXT_IP4_LOOKUP); + } + else + { + vxlan6_gpe_rewrite (t, 0, 0, VXLAN_GPE_ENCAP_NEXT_IP6_LOOKUP); + } + + + return 0; +} + +clib_error_t * +vxlan_gpe_ioam_clear (vxlan_gpe_tunnel_t * t, + int has_trace_option, int has_pot_option, + int has_ppc_option, u8 ipv6_set) +{ + int rv; + rv = vxlan_gpe_ioam_clear_rewrite (t, 0, 0, 0, 0); + + if (rv == 0) + { + return (0); + } + else + { + return clib_error_return_code (0, rv, 0, + "vxlan_gpe_ioam_clear_rewrite returned %d", + rv); + } + +} + + +clib_error_t * +vxlan_gpe_ioam_set (vxlan_gpe_tunnel_t * t, + int has_trace_option, int has_pot_option, + int has_ppc_option, u8 ipv6_set) +{ + int rv; + rv = vxlan_gpe_ioam_set_rewrite (t, has_trace_option, + has_pot_option, has_ppc_option, ipv6_set); + + if (rv == 0) + { + return (0); + } + else + { + return clib_error_return_code (0, rv, 0, + "vxlan_gpe_ioam_set_rewrite returned %d", + rv); + } + +} + +static void +vxlan_gpe_set_clear_output_feature_on_intf (vlib_main_t * vm, + u32 sw_if_index0, u8 is_add) +{ + + + + vnet_feature_enable_disable ("ip4-output", "vxlan-gpe-transit-ioam", + sw_if_index0, is_add, + 0 /* void *feature_config */ , + 0 /* u32 n_feature_config_bytes */ ); + return; +} + +void +vxlan_gpe_clear_output_feature_on_all_intfs (vlib_main_t * vm) +{ + vnet_sw_interface_t *si = 0; + vnet_main_t *vnm = vnet_get_main (); + vnet_interface_main_t *im = &vnm->interface_main; + + pool_foreach (si, im->sw_interfaces, ( + { + vxlan_gpe_set_clear_output_feature_on_intf + (vm, si->sw_if_index, 0); + })); + return; +} + + +extern fib_forward_chain_type_t +fib_entry_get_default_chain_type (const fib_entry_t * fib_entry); + +int +vxlan_gpe_enable_disable_ioam_for_dest (vlib_main_t * vm, + ip46_address_t dst_addr, + u32 outer_fib_index, + u8 is_ipv4, u8 is_add) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + u32 fib_index0 = 0; + u32 sw_if_index0 = ~0; + + fib_node_index_t fei = ~0; + fib_entry_t *fib_entry; + u32 adj_index0; + ip_adjacency_t *adj0; + fib_prefix_t fib_prefix; + //fib_forward_chain_type_t fct; + load_balance_t *lb_m, *lb_b; + const dpo_id_t *dpo0, *dpo1; + u32 i, j; + //vnet_hw_interface_t *hw; + + if (is_ipv4) + { + memset (&fib_prefix, 0, sizeof (fib_prefix_t)); + fib_prefix.fp_len = 32; + fib_prefix.fp_proto = FIB_PROTOCOL_IP4; + fib_prefix.fp_addr = dst_addr; + } + else + { + return 0; + } + + fei = fib_table_lookup (fib_index0, &fib_prefix); + fib_entry = fib_entry_get (fei); + + //fct = fib_entry_get_default_chain_type (fib_entry); + + if (!dpo_id_is_valid (&fib_entry->fe_lb /*[fct] */ )) + { + return (-1); + } + + lb_m = load_balance_get (fib_entry->fe_lb /*[fct] */ .dpoi_index); + + for (i = 0; i < lb_m->lb_n_buckets; i++) + { + dpo0 = load_balance_get_bucket_i (lb_m, i); + + if (dpo0->dpoi_type == DPO_LOAD_BALANCE) + { + lb_b = load_balance_get (dpo0->dpoi_index); + + for (j = 0; j < lb_b->lb_n_buckets; j++) + { + dpo1 = load_balance_get_bucket_i (lb_b, j); + + if (dpo1->dpoi_type == DPO_ADJACENCY) + { + adj_index0 = dpo1->dpoi_index; + + if (ADJ_INDEX_INVALID == adj_index0) + { + continue; + } + + adj0 = + ip_get_adjacency (&(ip4_main.lookup_main), adj_index0); + sw_if_index0 = adj0->rewrite_header.sw_if_index; + + if (~0 == sw_if_index0) + { + continue; + } + + + if (is_add) + { + vnet_feature_enable_disable ("ip4-output", + "vxlan-gpe-transit-ioam", + sw_if_index0, is_add, 0 + /* void *feature_config */ + , 0 /* u32 n_feature_config_bytes */ + ); + + vec_validate_init_empty (hm->bool_ref_by_sw_if_index, + sw_if_index0, ~0); + hm->bool_ref_by_sw_if_index[sw_if_index0] = 1; + } + else + { + hm->bool_ref_by_sw_if_index[sw_if_index0] = ~0; + } + } + } + } + } + + if (is_ipv4) + { + + uword *t = NULL; + vxlan_gpe_ioam_dest_tunnels_t *t1; + fib_prefix_t key4, *key4_copy; + hash_pair_t *hp; + memset (&key4, 0, sizeof (key4)); + key4.fp_proto = FIB_PROTOCOL_IP4; + key4.fp_addr.ip4.as_u32 = fib_prefix.fp_addr.ip4.as_u32; + t = hash_get_mem (hm->dst_by_ip4, &key4); + if (is_add) + { + if (t) + { + return 0; + } + pool_get_aligned (hm->dst_tunnels, t1, CLIB_CACHE_LINE_BYTES); + memset (t1, 0, sizeof (*t1)); + t1->fp_proto = FIB_PROTOCOL_IP4; + t1->dst_addr.ip4.as_u32 = fib_prefix.fp_addr.ip4.as_u32; + key4_copy = clib_mem_alloc (sizeof (*key4_copy)); + clib_memcpy (key4_copy, &key4, sizeof (*key4_copy)); + hash_set_mem (hm->dst_by_ip4, key4_copy, t1 - hm->dst_tunnels); + /* + * Attach to the FIB entry for the VxLAN-GPE destination + * and become its child. The dest route will invoke a callback + * when the fib entry changes, it can be used to + * re-program the output feature on the egress interface. + */ + + const fib_prefix_t tun_dst_pfx = { + .fp_len = 32, + .fp_proto = FIB_PROTOCOL_IP4, + .fp_addr = {.ip4 = t1->dst_addr.ip4,} + }; + + t1->fib_entry_index = + fib_table_entry_special_add (outer_fib_index, + &tun_dst_pfx, + FIB_SOURCE_RR, + FIB_ENTRY_FLAG_NONE, + ADJ_INDEX_INVALID); + t1->sibling_index = + fib_entry_child_add (t1->fib_entry_index, + hm->fib_entry_type, t1 - hm->dst_tunnels); + t1->outer_fib_index = outer_fib_index; + + } + else + { + if (!t) + { + return 0; + } + t1 = pool_elt_at_index (hm->dst_tunnels, t[0]); + hp = hash_get_pair (hm->dst_by_ip4, &key4); + key4_copy = (void *) (hp->key); + hash_unset_mem (hm->dst_by_ip4, &key4); + clib_mem_free (key4_copy); + pool_put (hm->dst_tunnels, t1); + } + } + else + { + // TBD for IPv6 + } + + return 0; +} + +void +vxlan_gpe_refresh_output_feature_on_all_dest (void) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + vxlan_gpe_ioam_dest_tunnels_t *t; + u32 i; + if (pool_elts (hm->dst_tunnels) == 0) + return; + vxlan_gpe_clear_output_feature_on_all_intfs (hm->vlib_main); + i = vec_len (hm->bool_ref_by_sw_if_index); + vec_free (hm->bool_ref_by_sw_if_index); + vec_validate_init_empty (hm->bool_ref_by_sw_if_index, i, ~0); + pool_foreach (t, hm->dst_tunnels, ( + { + vxlan_gpe_enable_disable_ioam_for_dest + (hm->vlib_main, + t->dst_addr, + t->outer_fib_index, + (t->fp_proto == FIB_PROTOCOL_IP4), 1 + /* is_add */ + ); + } + )); + return; +} + +void +vxlan_gpe_clear_output_feature_on_select_intfs (void) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + u32 sw_if_index0 = 0; + for (sw_if_index0 = 0; + sw_if_index0 < vec_len (hm->bool_ref_by_sw_if_index); sw_if_index0++) + { + if (hm->bool_ref_by_sw_if_index[sw_if_index0] == 0xFF) + { + vxlan_gpe_set_clear_output_feature_on_intf + (hm->vlib_main, sw_if_index0, 0); + } + } + + return; +} + +static clib_error_t * +vxlan_gpe_set_ioam_rewrite_command_fn (vlib_main_t * + vm, + unformat_input_t + * input, vlib_cli_command_t * cmd) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + ip46_address_t local, remote; + u8 local_set = 0; + u8 remote_set = 0; + u8 ipv4_set = 0; + u8 ipv6_set = 0; + u32 vni; + u8 vni_set = 0; + u8 disable = 0; + clib_error_t *rv = 0; + vxlan4_gpe_tunnel_key_t key4; + vxlan6_gpe_tunnel_key_t key6; + uword *p; + vxlan_gpe_main_t *gm = &vxlan_gpe_main; + vxlan_gpe_tunnel_t *t = 0; + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "local %U", unformat_ip4_address, &local.ip4)) + { + local_set = 1; + ipv4_set = 1; + } + else + if (unformat (input, "remote %U", unformat_ip4_address, &remote.ip4)) + { + remote_set = 1; + ipv4_set = 1; + } + else if (unformat (input, "local %U", unformat_ip6_address, &local.ip6)) + { + local_set = 1; + ipv6_set = 1; + } + else + if (unformat (input, "remote %U", unformat_ip6_address, &remote.ip6)) + { + remote_set = 1; + ipv6_set = 1; + } + else if (unformat (input, "vni %d", &vni)) + vni_set = 1; + else if (unformat (input, "disable")) + disable = 1; + else + break; + } + + if (local_set == 0) + return clib_error_return (0, "tunnel local address not specified"); + if (remote_set == 0) + return clib_error_return (0, "tunnel remote address not specified"); + if (ipv4_set && ipv6_set) + return clib_error_return (0, "both IPv4 and IPv6 addresses specified"); + if ((ipv4_set + && memcmp (&local.ip4, &remote.ip4, + sizeof (local.ip4)) == 0) || (ipv6_set + && + memcmp + (&local.ip6, + &remote.ip6, + sizeof (local.ip6)) == 0)) + return clib_error_return (0, "src and dst addresses are identical"); + if (vni_set == 0) + return clib_error_return (0, "vni not specified"); + if (!ipv6_set) + { + key4.local = local.ip4.as_u32; + key4.remote = remote.ip4.as_u32; + key4.vni = clib_host_to_net_u32 (vni << 8); + key4.pad = 0; + p = hash_get_mem (gm->vxlan4_gpe_tunnel_by_key, &key4); + } + else + { + key6.local.as_u64[0] = local.ip6.as_u64[0]; + key6.local.as_u64[1] = local.ip6.as_u64[1]; + key6.remote.as_u64[0] = remote.ip6.as_u64[0]; + key6.remote.as_u64[1] = remote.ip6.as_u64[1]; + key6.vni = clib_host_to_net_u32 (vni << 8); + p = hash_get_mem (gm->vxlan6_gpe_tunnel_by_key, &key6); + } + + if (!p) + return clib_error_return (0, "VxLAN Tunnel not found"); + t = pool_elt_at_index (gm->tunnels, p[0]); + if (!disable) + { + rv = + vxlan_gpe_ioam_set (t, hm->has_trace_option, + hm->has_pot_option, hm->has_ppc_option, ipv6_set); + } + else + { + rv = vxlan_gpe_ioam_clear (t, 0, 0, 0, 0); + } + return rv; +} + + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (vxlan_gpe_set_ioam_rewrite_cmd, static) = { + .path = "set vxlan-gpe-ioam", + .short_help = "set vxlan-gpe-ioam vxlan [disable]", + .function = vxlan_gpe_set_ioam_rewrite_command_fn, +}; +/* *INDENT-ON* */ + + + +clib_error_t * +vxlan_gpe_ioam_enable (int has_trace_option, + int has_pot_option, int has_ppc_option) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + hm->has_trace_option = has_trace_option; + hm->has_pot_option = has_pot_option; + hm->has_ppc_option = has_ppc_option; + if (hm->has_trace_option) + { + vxlan_gpe_trace_profile_setup (); + } + + return 0; +} + +clib_error_t * +vxlan_gpe_ioam_disable (int + has_trace_option, + int has_pot_option, int has_ppc_option) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + hm->has_trace_option = has_trace_option; + hm->has_pot_option = has_pot_option; + hm->has_ppc_option = has_ppc_option; + if (!hm->has_trace_option) + { + vxlan_gpe_trace_profile_cleanup (); + } + + return 0; +} + +void +vxlan_gpe_set_next_override (uword next) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + hm->decap_v4_next_override = next; + return; +} + +static clib_error_t * +vxlan_gpe_set_ioam_flags_command_fn (vlib_main_t * vm, + unformat_input_t + * input, vlib_cli_command_t * cmd) +{ + int has_trace_option = 0; + int has_pot_option = 0; + int has_ppc_option = 0; + clib_error_t *rv = 0; + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "trace")) + has_trace_option = 1; + else if (unformat (input, "pot")) + has_pot_option = 1; + else if (unformat (input, "ppc encap")) + has_ppc_option = PPC_ENCAP; + else if (unformat (input, "ppc decap")) + has_ppc_option = PPC_DECAP; + else if (unformat (input, "ppc none")) + has_ppc_option = PPC_NONE; + else + break; + } + + + rv = + vxlan_gpe_ioam_enable (has_trace_option, has_pot_option, has_ppc_option); + return rv; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (vxlan_gpe_set_ioam_flags_cmd, static) = +{ +.path = "set vxlan-gpe-ioam rewrite", +.short_help = "set vxlan-gpe-ioam [trace] [pot] [ppc ]", +.function = vxlan_gpe_set_ioam_flags_command_fn,}; +/* *INDENT-ON* */ + + +int vxlan_gpe_ioam_disable_for_dest + (vlib_main_t * vm, ip46_address_t dst_addr, u32 outer_fib_index, + u8 ipv4_set) +{ + vxlan_gpe_ioam_dest_tunnels_t *t; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + vxlan_gpe_enable_disable_ioam_for_dest (hm->vlib_main, + dst_addr, outer_fib_index, ipv4_set, + 0); + if (pool_elts (hm->dst_tunnels) == 0) + { + vxlan_gpe_clear_output_feature_on_select_intfs (); + return 0; + } + + pool_foreach (t, hm->dst_tunnels, ( + { + vxlan_gpe_enable_disable_ioam_for_dest + (hm->vlib_main, + t->dst_addr, + t->outer_fib_index, + (t->fp_proto == + FIB_PROTOCOL_IP4), 1 /* is_add */ ); + } + )); + vxlan_gpe_clear_output_feature_on_select_intfs (); + return (0); + +} + +static clib_error_t *vxlan_gpe_set_ioam_transit_rewrite_command_fn + (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + ip46_address_t dst_addr; + u8 dst_addr_set = 0; + u8 ipv4_set = 0; + u8 ipv6_set = 0; + u8 disable = 0; + clib_error_t *rv = 0; + u32 outer_fib_index = 0; + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "dst-ip %U", unformat_ip4_address, &dst_addr.ip4)) + { + dst_addr_set = 1; + ipv4_set = 1; + } + else + if (unformat + (input, "dst-ip %U", unformat_ip6_address, &dst_addr.ip6)) + { + dst_addr_set = 1; + ipv6_set = 1; + } + else if (unformat (input, "outer-fib-index %d", &outer_fib_index)) + { + } + + else if (unformat (input, "disable")) + disable = 1; + else + break; + } + + if (dst_addr_set == 0) + return clib_error_return (0, "tunnel destination address not specified"); + if (ipv4_set && ipv6_set) + return clib_error_return (0, "both IPv4 and IPv6 addresses specified"); + if (!disable) + { + vxlan_gpe_enable_disable_ioam_for_dest (hm->vlib_main, + dst_addr, outer_fib_index, + ipv4_set, 1); + } + else + { + vxlan_gpe_ioam_disable_for_dest + (vm, dst_addr, outer_fib_index, ipv4_set); + } + return rv; +} + + /* *INDENT-OFF* */ +VLIB_CLI_COMMAND (vxlan_gpe_set_ioam_transit_rewrite_cmd, static) = { + .path = "set vxlan-gpe-ioam-transit", + .short_help = "set vxlan-gpe-ioam-transit dst-ip [outer-fib-index ] [disable]", + .function = vxlan_gpe_set_ioam_transit_rewrite_command_fn, +}; +/* *INDENT-ON* */ + +clib_error_t *clear_vxlan_gpe_ioam_rewrite_command_fn + (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) +{ + return (vxlan_gpe_ioam_disable (0, 0, 0)); +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (vxlan_gpe_clear_ioam_flags_cmd, static) = +{ +.path = "clear vxlan-gpe-ioam rewrite", +.short_help = "clear vxlan-gpe-ioam rewrite", +.function = clear_vxlan_gpe_ioam_rewrite_command_fn, +}; +/* *INDENT-ON* */ + + +/** + * Function definition to backwalk a FIB node + */ +static fib_node_back_walk_rc_t +vxlan_gpe_ioam_back_walk (fib_node_t * node, fib_node_back_walk_ctx_t * ctx) +{ + vxlan_gpe_refresh_output_feature_on_all_dest (); + return (FIB_NODE_BACK_WALK_CONTINUE); +} + +/** + * Function definition to get a FIB node from its index + */ +static fib_node_t * +vxlan_gpe_ioam_fib_node_get (fib_node_index_t index) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + return (&hm->node); +} + +/** + * Function definition to inform the FIB node that its last lock has gone. + */ +static void +vxlan_gpe_ioam_last_lock_gone (fib_node_t * node) +{ + ASSERT (0); +} + + +/* + * Virtual function table registered by MPLS GRE tunnels + * for participation in the FIB object graph. + */ +const static fib_node_vft_t vxlan_gpe_ioam_vft = { + .fnv_get = vxlan_gpe_ioam_fib_node_get, + .fnv_last_lock = vxlan_gpe_ioam_last_lock_gone, + .fnv_back_walk = vxlan_gpe_ioam_back_walk, +}; + +void +vxlan_gpe_ioam_interface_init (void) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + hm->fib_entry_type = fib_node_register_new_type (&vxlan_gpe_ioam_vft); + return; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h new file mode 100644 index 00000000..0711b87a --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_vxlan_gpe_ioam_h__ +#define __included_vxlan_gpe_ioam_h__ + +#include +#include +#include +#include + + +typedef struct vxlan_gpe_sw_interface_ +{ + u32 sw_if_index; +} vxlan_gpe_ioam_sw_interface_t; + +typedef struct vxlan_gpe_dest_tunnels_ +{ + ip46_address_t dst_addr; + u32 fp_proto; + u32 sibling_index; + fib_node_index_t fib_entry_index; + u32 outer_fib_index; +} vxlan_gpe_ioam_dest_tunnels_t; + +typedef struct vxlan_gpe_ioam_main_ +{ + /** + * Linkage into the FIB object graph + */ + fib_node_t node; + + /* time scale transform. Joy. */ + u32 unix_time_0; + f64 vlib_time_0; + + + /* Trace option */ + u8 has_trace_option; + + /* Pot option */ + u8 has_pot_option; + +#define PPC_NONE 0 +#define PPC_ENCAP 1 +#define PPC_DECAP 2 + u8 has_ppc_option; + +#define TSP_SECONDS 0 +#define TSP_MILLISECONDS 1 +#define TSP_MICROSECONDS 2 +#define TSP_NANOSECONDS 3 + + /* Array of function pointers to ADD and POP VxLAN-GPE iOAM option handling routines */ + u8 options_size[256]; + int (*add_options[256]) (u8 * rewrite_string, u8 * rewrite_size); + int (*pop_options[256]) (ip4_header_t * ip, vxlan_gpe_ioam_option_t * opt); + + /* Array of function pointers to iOAM option handling routines */ + int (*options[256]) (vlib_buffer_t * b, vxlan_gpe_ioam_option_t * opt, + u8 is_ipv4, u8 use_adj); + u8 *(*trace[256]) (u8 * s, vxlan_gpe_ioam_option_t * opt); + + /* API message ID base */ + u16 msg_id_base; + + /* Override to export for iOAM */ + uword decap_v4_next_override; + uword decap_v6_next_override; + + /* sequence of node graph for encap */ + uword encap_v4_next_node; + uword encap_v6_next_node; + + /* Software interfaces. */ + vxlan_gpe_ioam_sw_interface_t *sw_interfaces; + + /* hash ip4/ip6 -> list of destinations for doing transit iOAM operation */ + vxlan_gpe_ioam_dest_tunnels_t *dst_tunnels; + uword *dst_by_ip4; + uword *dst_by_ip6; + + /** per sw_if_index, to maintain bitmap */ + u8 *bool_ref_by_sw_if_index; + fib_node_type_t fib_entry_type; + + /** State convenience vlib_main_t */ + vlib_main_t *vlib_main; + /** State convenience vnet_main_t */ + vnet_main_t *vnet_main; + + +} vxlan_gpe_ioam_main_t; +extern vxlan_gpe_ioam_main_t vxlan_gpe_ioam_main; + +/* + * Primary h-b-h handler trace support + */ +typedef struct +{ + u32 next_index; + u32 trace_len; + u8 option_data[256]; +} ioam_trace_t; + + +extern vlib_node_registration_t vxlan_gpe_encap_ioam_v4_node; +extern vlib_node_registration_t vxlan_gpe_decap_ioam_v4_node; +extern vlib_node_registration_t vxlan_gpe_transit_ioam_v4_node; + +clib_error_t *vxlan_gpe_ioam_enable (int has_trace_option, int has_pot_option, + int has_ppc_option); + +clib_error_t *vxlan_gpe_ioam_disable (int has_trace_option, + int has_pot_option, int has_ppc_option); + +clib_error_t *vxlan_gpe_ioam_set (vxlan_gpe_tunnel_t * t, + int has_trace_option, + int has_pot_option, + int has_ppc_option, u8 ipv6_set); +clib_error_t *vxlan_gpe_ioam_clear (vxlan_gpe_tunnel_t * t, + int has_trace_option, int has_pot_option, + int has_ppc_option, u8 ipv6_set); + +int vxlan_gpe_ioam_add_register_option (u8 option, + u8 size, + int rewrite_options (u8 * + rewrite_string, + u8 * + rewrite_size)); + +int vxlan_gpe_add_unregister_option (u8 option); + +int vxlan_gpe_ioam_register_option (u8 option, + int options (vlib_buffer_t * b, + vxlan_gpe_ioam_option_t * + opt, u8 is_ipv4, u8 use_adj), + u8 * trace (u8 * s, + vxlan_gpe_ioam_option_t * + opt)); +int vxlan_gpe_ioam_unregister_option (u8 option); + +int vxlan_gpe_trace_profile_setup (void); + +int vxlan_gpe_trace_profile_cleanup (void); +extern void vxlan_gpe_ioam_interface_init (void); +int +vxlan_gpe_enable_disable_ioam_for_dest (vlib_main_t * vm, + ip46_address_t dst_addr, + u32 outer_fib_index, + u8 is_ipv4, u8 is_add); +int vxlan_gpe_ioam_disable_for_dest + (vlib_main_t * vm, ip46_address_t dst_addr, u32 outer_fib_index, + u8 ipv4_set); + +typedef enum +{ + VXLAN_GPE_DECAP_IOAM_V4_NEXT_POP, + VXLAN_GPE_DECAP_IOAM_V4_NEXT_DROP, + VXLAN_GPE_DECAP_IOAM_V4_N_NEXT +} vxlan_gpe_decap_ioam_v4_next_t; + +#endif + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h new file mode 100644 index 00000000..a7ef859e --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_packet.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_vxlan_gpe_ioam_packet_h__ +#define __included_vxlan_gpe_ioam_packet_h__ + +#include +#include +#include + + + +#define VXLAN_GPE_OPTION_TYPE_IOAM_TRACE 59 +#define VXLAN_GPE_OPTION_TYPE_IOAM_PROOF_OF_TRANSIT 60 + +/** + * @brief VXLAN GPE Extension (iOAM) Header definition + */ +typedef struct +{ + u8 type; + u8 length; + /** Reserved */ + u8 reserved; + /** see vxlan_gpe_protocol_t */ + u8 protocol; +} vxlan_gpe_ioam_hdr_t; + +/* + * @brief VxLAN GPE iOAM Option definition + */ +typedef struct +{ + /* Option Type */ + u8 type; + /* Length in octets of the option data field */ + u8 length; +} vxlan_gpe_ioam_option_t; + + +#endif + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c new file mode 100644 index 00000000..e37b1642 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c @@ -0,0 +1,552 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +/* Timestamp precision multipliers for seconds, milliseconds, microseconds + * and nanoseconds respectively. + */ +static f64 trace_tsp_mul[4] = { 1, 1e3, 1e6, 1e9 }; + +typedef union +{ + u64 as_u64; + u32 as_u32[2]; +} time_u64_t; + + +/* *INDENT-OFF* */ +typedef CLIB_PACKED(struct { + vxlan_gpe_ioam_option_t hdr; + u8 ioam_trace_type; + u8 data_list_elts_left; + u32 elts[0]; /* Variable type. So keep it generic */ +}) vxlan_gpe_ioam_trace_option_t; +/* *INDENT-ON* */ + + +#define foreach_vxlan_gpe_ioam_trace_stats \ + _(SUCCESS, "Pkts updated with TRACE records") \ + _(FAILED, "Errors in TRACE due to lack of TRACE records") + +static char *vxlan_gpe_ioam_trace_stats_strings[] = { +#define _(sym,string) string, + foreach_vxlan_gpe_ioam_trace_stats +#undef _ +}; + +typedef enum +{ +#define _(sym,str) VXLAN_GPE_IOAM_TRACE_##sym, + foreach_vxlan_gpe_ioam_trace_stats +#undef _ + VXLAN_GPE_IOAM_TRACE_N_STATS, +} vxlan_gpe_ioam_trace_stats_t; + + +typedef struct +{ + /* stats */ + u64 counters[ARRAY_LEN (vxlan_gpe_ioam_trace_stats_strings)]; + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; +} vxlan_gpe_ioam_trace_main_t; + +vxlan_gpe_ioam_trace_main_t vxlan_gpe_ioam_trace_main; + +int +vxlan_gpe_ioam_add_register_option (u8 option, + u8 size, + int rewrite_options (u8 * rewrite_string, + u8 * rewrite_size)) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + ASSERT (option < ARRAY_LEN (hm->add_options)); + + /* Already registered */ + if (hm->add_options[option]) + return (-1); + + hm->add_options[option] = rewrite_options; + hm->options_size[option] = size; + + return (0); +} + +int +vxlan_gpe_add_unregister_option (u8 option) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + ASSERT (option < ARRAY_LEN (hm->add_options)); + + /* Not registered */ + if (!hm->add_options[option]) + return (-1); + + hm->add_options[option] = NULL; + hm->options_size[option] = 0; + return (0); +} + + +int +vxlan_gpe_ioam_register_option (u8 option, + int options (vlib_buffer_t * b, + vxlan_gpe_ioam_option_t * opt, + u8 is_ipv4, u8 use_adj), + u8 * trace (u8 * s, + vxlan_gpe_ioam_option_t * opt)) +{ + vxlan_gpe_ioam_main_t *im = &vxlan_gpe_ioam_main; + + ASSERT (option < ARRAY_LEN (im->options)); + + /* Already registered */ + if (im->options[option]) + return (-1); + + im->options[option] = options; + im->trace[option] = trace; + + return (0); +} + +int +vxlan_gpe_ioam_unregister_option (u8 option) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + ASSERT (option < ARRAY_LEN (hm->options)); + + /* Not registered */ + if (!hm->options[option]) + return (-1); + + hm->options[option] = NULL; + hm->trace[option] = NULL; + + return (0); +} + + +always_inline void +vxlan_gpe_ioam_trace_stats_increment_counter (u32 counter_index, + u64 increment) +{ + vxlan_gpe_ioam_trace_main_t *hm = &vxlan_gpe_ioam_trace_main; + + hm->counters[counter_index] += increment; +} + + +static u8 * +format_ioam_data_list_element (u8 * s, va_list * args) +{ + u32 *elt = va_arg (*args, u32 *); + u8 *trace_type_p = va_arg (*args, u8 *); + u8 trace_type = *trace_type_p; + + + if (trace_type & BIT_TTL_NODEID) + { + u32 ttl_node_id_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "ttl 0x%x node id 0x%x ", + ttl_node_id_host_byte_order >> 24, + ttl_node_id_host_byte_order & 0x00FFFFFF); + + elt++; + } + + if (trace_type & BIT_ING_INTERFACE && trace_type & BIT_ING_INTERFACE) + { + u32 ingress_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "ingress 0x%x egress 0x%x ", + ingress_host_byte_order >> 16, + ingress_host_byte_order & 0xFFFF); + elt++; + } + + if (trace_type & BIT_TIMESTAMP) + { + u32 ts_in_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "ts 0x%x \n", ts_in_host_byte_order); + elt++; + } + + if (trace_type & BIT_APPDATA) + { + u32 appdata_in_host_byte_order = clib_net_to_host_u32 (*elt); + s = format (s, "app 0x%x ", appdata_in_host_byte_order); + elt++; + } + + return s; +} + + + +int +vxlan_gpe_ioam_trace_rewrite_handler (u8 * rewrite_string, u8 * rewrite_size) +{ + vxlan_gpe_ioam_trace_option_t *trace_option = NULL; + u8 trace_data_size = 0; + u8 trace_option_elts = 0; + trace_profile *profile = NULL; + + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + return (-1); + } + + if (PREDICT_FALSE (!rewrite_string)) + return -1; + + trace_option_elts = profile->num_elts; + trace_data_size = fetch_trace_data_size (profile->trace_type); + trace_option = (vxlan_gpe_ioam_trace_option_t *) rewrite_string; + trace_option->hdr.type = VXLAN_GPE_OPTION_TYPE_IOAM_TRACE; + trace_option->hdr.length = 2 /*ioam_trace_type,data_list_elts_left */ + + trace_option_elts * trace_data_size; + trace_option->ioam_trace_type = profile->trace_type & TRACE_TYPE_MASK; + trace_option->data_list_elts_left = trace_option_elts; + *rewrite_size = + sizeof (vxlan_gpe_ioam_trace_option_t) + + (trace_option_elts * trace_data_size); + + return 0; +} + + +int +vxlan_gpe_ioam_trace_data_list_handler (vlib_buffer_t * b, + vxlan_gpe_ioam_option_t * opt, + u8 is_ipv4, u8 use_adj) +{ + u8 elt_index = 0; + vxlan_gpe_ioam_trace_option_t *trace = + (vxlan_gpe_ioam_trace_option_t *) opt; + time_u64_t time_u64; + u32 *elt; + int rv = 0; + trace_profile *profile = NULL; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + return (-1); + } + + + time_u64.as_u64 = 0; + + if (PREDICT_TRUE (trace->data_list_elts_left)) + { + trace->data_list_elts_left--; + /* fetch_trace_data_size returns in bytes. Convert it to 4-bytes + * to skip to this node's location. + */ + elt_index = + trace->data_list_elts_left * + fetch_trace_data_size (trace->ioam_trace_type) / 4; + elt = &trace->elts[elt_index]; + if (is_ipv4) + { + if (trace->ioam_trace_type & BIT_TTL_NODEID) + { + ip4_header_t *ip0 = vlib_buffer_get_current (b); + /* The transit case is the only case where the TTL decrement happens + * before iOAM processing. For now, use the use_adj flag as an overload. + * We can probably use a separate flag instead of overloading the use_adj flag. + */ + *elt = clib_host_to_net_u32 (((ip0->ttl - 1 + use_adj) << 24) | + profile->node_id); + elt++; + } + + if (trace->ioam_trace_type & BIT_ING_INTERFACE) + { + u16 tx_if = 0; + u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; + ip4_main_t *im4 = &ip4_main; + ip_lookup_main_t *lm = &im4->lookup_main; + if (use_adj) + { + ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index); + tx_if = adj->rewrite_header.sw_if_index & 0xFFFF; + } + + *elt = + (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 | + tx_if; + *elt = clib_host_to_net_u32 (*elt); + elt++; + } + } + else + { + if (trace->ioam_trace_type & BIT_TTL_NODEID) + { + ip6_header_t *ip0 = vlib_buffer_get_current (b); + *elt = clib_host_to_net_u32 ((ip0->hop_limit << 24) | + profile->node_id); + elt++; + } + if (trace->ioam_trace_type & BIT_ING_INTERFACE) + { + u16 tx_if = 0; + u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; + ip6_main_t *im6 = &ip6_main; + ip_lookup_main_t *lm = &im6->lookup_main; + if (use_adj) + { + ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index); + tx_if = adj->rewrite_header.sw_if_index & 0xFFFF; + } + + *elt = + (vnet_buffer (b)->sw_if_index[VLIB_RX] & 0xFFFF) << 16 | + tx_if; + *elt = clib_host_to_net_u32 (*elt); + elt++; + } + } + + if (trace->ioam_trace_type & BIT_TIMESTAMP) + { + /* Send least significant 32 bits */ + f64 time_f64 = + (f64) (((f64) hm->unix_time_0) + + (vlib_time_now (hm->vlib_main) - hm->vlib_time_0)); + + time_u64.as_u64 = time_f64 * trace_tsp_mul[profile->trace_tsp]; + *elt = clib_host_to_net_u32 (time_u64.as_u32[0]); + elt++; + } + + if (trace->ioam_trace_type & BIT_APPDATA) + { + /* $$$ set elt0->app_data */ + *elt = clib_host_to_net_u32 (profile->app_data); + elt++; + } + vxlan_gpe_ioam_trace_stats_increment_counter + (VXLAN_GPE_IOAM_TRACE_SUCCESS, 1); + } + else + { + vxlan_gpe_ioam_trace_stats_increment_counter + (VXLAN_GPE_IOAM_TRACE_FAILED, 1); + } + return (rv); +} + +u8 * +vxlan_gpe_ioam_trace_data_list_trace_handler (u8 * s, + vxlan_gpe_ioam_option_t * opt) +{ + vxlan_gpe_ioam_trace_option_t *trace; + u8 trace_data_size_in_words = 0; + u32 *elt; + int elt_index = 0; + + trace = (vxlan_gpe_ioam_trace_option_t *) opt; + s = + format (s, " Trace Type 0x%x , %d elts left\n", trace->ioam_trace_type, + trace->data_list_elts_left); + trace_data_size_in_words = + fetch_trace_data_size (trace->ioam_trace_type) / 4; + elt = &trace->elts[0]; + while ((u8 *) elt < ((u8 *) (&trace->elts[0]) + trace->hdr.length - 2 + /* -2 accounts for ioam_trace_type,elts_left */ )) + { + s = format (s, " [%d] %U\n", elt_index, + format_ioam_data_list_element, + elt, &trace->ioam_trace_type); + elt_index++; + elt += trace_data_size_in_words; + } + return (s); +} + + +static clib_error_t * +vxlan_gpe_show_ioam_trace_cmd_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + vxlan_gpe_ioam_trace_main_t *hm = &vxlan_gpe_ioam_trace_main; + u8 *s = 0; + int i = 0; + + for (i = 0; i < VXLAN_GPE_IOAM_TRACE_N_STATS; i++) + { + s = format (s, " %s - %lu\n", vxlan_gpe_ioam_trace_stats_strings[i], + hm->counters[i]); + } + + vlib_cli_output (vm, "%v", s); + vec_free (s); + return 0; +} + + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (vxlan_gpe_show_ioam_trace_cmd, static) = { + .path = "show ioam vxlan-gpe trace", + .short_help = "iOAM trace statistics", + .function = vxlan_gpe_show_ioam_trace_cmd_fn, +}; +/* *INDENT-ON* */ + + +static clib_error_t * +vxlan_gpe_ioam_trace_init (vlib_main_t * vm) +{ + vxlan_gpe_ioam_trace_main_t *hm = &vxlan_gpe_ioam_trace_main; + clib_error_t *error; + + if ((error = vlib_call_init_function (vm, ip_main_init))) + return (error); + + if ((error = vlib_call_init_function (vm, ip6_lookup_init))) + return error; + + if ((error = vlib_call_init_function (vm, vxlan_gpe_init))) + return (error); + + hm->vlib_main = vm; + hm->vnet_main = vnet_get_main (); + memset (hm->counters, 0, sizeof (hm->counters)); + + if (vxlan_gpe_ioam_register_option + (VXLAN_GPE_OPTION_TYPE_IOAM_TRACE, + vxlan_gpe_ioam_trace_data_list_handler, + vxlan_gpe_ioam_trace_data_list_trace_handler) < 0) + return (clib_error_create + ("registration of VXLAN_GPE_OPTION_TYPE_IOAM_TRACE failed")); + + + if (vxlan_gpe_ioam_add_register_option + (VXLAN_GPE_OPTION_TYPE_IOAM_TRACE, + sizeof (vxlan_gpe_ioam_trace_option_t), + vxlan_gpe_ioam_trace_rewrite_handler) < 0) + return (clib_error_create + ("registration of VXLAN_GPE_OPTION_TYPE_IOAM_TRACE for rewrite failed")); + + + return (0); +} + +VLIB_INIT_FUNCTION (vxlan_gpe_ioam_trace_init); + +int +vxlan_gpe_trace_profile_cleanup (void) +{ + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] = 0; + + return 0; + +} + +static int +vxlan_gpe_ioam_trace_get_sizeof_handler (u32 * result) +{ + u16 size = 0; + u8 trace_data_size = 0; + trace_profile *profile = NULL; + + *result = 0; + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + return (-1); + } + + trace_data_size = fetch_trace_data_size (profile->trace_type); + if (PREDICT_FALSE (trace_data_size == 0)) + return VNET_API_ERROR_INVALID_VALUE; + + if (PREDICT_FALSE (profile->num_elts * trace_data_size > 254)) + return VNET_API_ERROR_INVALID_VALUE; + + size += + sizeof (vxlan_gpe_ioam_trace_option_t) + + profile->num_elts * trace_data_size; + *result = size; + + return 0; +} + + +int +vxlan_gpe_trace_profile_setup (void) +{ + u32 trace_size = 0; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + trace_profile *profile = NULL; + + + profile = trace_profile_find (); + + if (PREDICT_FALSE (!profile)) + { + return (-1); + } + + + if (vxlan_gpe_ioam_trace_get_sizeof_handler (&trace_size) < 0) + return (-1); + + hm->options_size[VXLAN_GPE_OPTION_TYPE_IOAM_TRACE] = trace_size; + + return (0); +} + + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h new file mode 100644 index 00000000..c0ad8d9d --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_util.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_vxlan_gpe_ioam_util_h__ +#define __included_vxlan_gpe_ioam_util_h__ + +#include +#include +#include + + +typedef struct +{ + u32 tunnel_index; + ioam_trace_t fmt_trace; +} vxlan_gpe_ioam_v4_trace_t; + + +static u8 * +format_vxlan_gpe_ioam_v4_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + vxlan_gpe_ioam_v4_trace_t *t1 = va_arg (*args, vxlan_gpe_ioam_v4_trace_t *); + ioam_trace_t *t = &(t1->fmt_trace); + vxlan_gpe_ioam_option_t *fmt_trace0; + vxlan_gpe_ioam_option_t *opt0, *limit0; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + u8 type0; + + fmt_trace0 = (vxlan_gpe_ioam_option_t *) t->option_data; + + s = format (s, "VXLAN-GPE-IOAM: next_index %d len %d traced %d", + t->next_index, fmt_trace0->length, t->trace_len); + + opt0 = (vxlan_gpe_ioam_option_t *) (fmt_trace0 + 1); + limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) fmt_trace0) + t->trace_len; + + while (opt0 < limit0) + { + type0 = opt0->type; + switch (type0) + { + case 0: /* Pad, just stop */ + opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; + break; + + default: + if (hm->trace[type0]) + { + s = (*hm->trace[type0]) (s, opt0); + } + else + { + s = + format (s, "\n unrecognized option %d length %d", type0, + opt0->length); + } + opt0 = + (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + + sizeof (vxlan_gpe_ioam_option_t)); + break; + } + } + + s = format (s, "VXLAN-GPE-IOAM: tunnel %d", t1->tunnel_index); + return s; +} + + +always_inline void +vxlan_gpe_encap_decap_ioam_v4_one_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_buffer_t * b0, + u32 * next0, u32 drop_node_val, + u8 use_adj) +{ + ip4_header_t *ip0; + udp_header_t *udp_hdr0; + vxlan_gpe_header_t *gpe_hdr0; + vxlan_gpe_ioam_hdr_t *gpe_ioam0; + vxlan_gpe_ioam_option_t *opt0; + vxlan_gpe_ioam_option_t *limit0; + vxlan_gpe_ioam_main_t *hm = &vxlan_gpe_ioam_main; + + /* Populate the iOAM header */ + ip0 = vlib_buffer_get_current (b0); + udp_hdr0 = (udp_header_t *) (ip0 + 1); + gpe_hdr0 = (vxlan_gpe_header_t *) (udp_hdr0 + 1); + gpe_ioam0 = (vxlan_gpe_ioam_hdr_t *) (gpe_hdr0 + 1); + opt0 = (vxlan_gpe_ioam_option_t *) (gpe_ioam0 + 1); + limit0 = (vxlan_gpe_ioam_option_t *) ((u8 *) gpe_ioam0 + gpe_ioam0->length); + + /* + * Basic validity checks + */ + if (gpe_ioam0->length > clib_net_to_host_u16 (ip0->length)) + { + *next0 = drop_node_val; + return; + } + + /* Scan the set of h-b-h options, process ones that we understand */ + while (opt0 < limit0) + { + u8 type0; + type0 = opt0->type; + switch (type0) + { + case 0: /* Pad1 */ + opt0 = (vxlan_gpe_ioam_option_t *) ((u8 *) opt0) + 1; + continue; + case 1: /* PadN */ + break; + default: + if (hm->options[type0]) + { + if ((*hm->options[type0]) (b0, opt0, 1 /* is_ipv4 */ , + use_adj) < 0) + { + *next0 = drop_node_val; + return; + } + } + break; + } + opt0 = + (vxlan_gpe_ioam_option_t *) (((u8 *) opt0) + opt0->length + + sizeof (vxlan_gpe_ioam_hdr_t)); + } + + + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + vxlan_gpe_ioam_v4_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + u32 trace_len = gpe_ioam0->length; + t->fmt_trace.next_index = *next0; + /* Capture the ioam option verbatim */ + trace_len = + trace_len < + ARRAY_LEN (t->fmt_trace. + option_data) ? trace_len : ARRAY_LEN (t->fmt_trace. + option_data); + t->fmt_trace.trace_len = trace_len; + clib_memcpy (&(t->fmt_trace.option_data), gpe_ioam0, trace_len); + } + return; +} + + +#endif + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h new file mode 100644 index 00000000..cc0a10a3 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_msg_enum.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_vxlan_gpe_msg_enum_h +#define included_vxlan_gpe_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_vxlan_gpe_msg_enum_h */ diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_test.c b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_test.c new file mode 100644 index 00000000..47253eb6 --- /dev/null +++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_test.c @@ -0,0 +1,600 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + *------------------------------------------------------------------ + * vxlan_gpe_test.c - test harness for vxlan_gpe plugin + *------------------------------------------------------------------ + */ + +#include +#include +#include +#include +#include + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version +#include +#include + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} vxlan_gpe_test_main_t; + +vxlan_gpe_test_main_t vxlan_gpe_test_main; + +#define foreach_standard_reply_retval_handler \ +_(vxlan_gpe_ioam_enable_reply) \ +_(vxlan_gpe_ioam_disable_reply) \ +_(vxlan_gpe_ioam_vni_enable_reply) \ +_(vxlan_gpe_ioam_vni_disable_reply) \ +_(vxlan_gpe_ioam_transit_enable_reply) \ +_(vxlan_gpe_ioam_transit_disable_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = vxlan_gpe_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(VXLAN_GPE_IOAM_ENABLE_REPLY, vxlan_gpe_ioam_enable_reply) \ +_(VXLAN_GPE_IOAM_DISABLE_REPLY, vxlan_gpe_ioam_disable_reply) \ +_(VXLAN_GPE_IOAM_VNI_ENABLE_REPLY, vxlan_gpe_ioam_vni_enable_reply) \ +_(VXLAN_GPE_IOAM_VNI_DISABLE_REPLY, vxlan_gpe_ioam_vni_disable_reply) \ +_(VXLAN_GPE_IOAM_TRANSIT_ENABLE_REPLY, vxlan_gpe_ioam_transit_enable_reply) \ +_(VXLAN_GPE_IOAM_TRANSIT_DISABLE_REPLY, vxlan_gpe_ioam_transit_disable_reply) \ + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + + +static int +api_vxlan_gpe_ioam_enable (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + + unformat_input_t *input = vam->input; + vl_api_vxlan_gpe_ioam_enable_t *mp; + f64 timeout; + u32 id = 0; + int has_trace_option = 0; + int has_pow_option = 0; + int has_ppc_option = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "trace")) + has_trace_option = 1; + else if (unformat (input, "pow")) + has_pow_option = 1; + else if (unformat (input, "ppc encap")) + has_ppc_option = PPC_ENCAP; + else if (unformat (input, "ppc decap")) + has_ppc_option = PPC_DECAP; + else if (unformat (input, "ppc none")) + has_ppc_option = PPC_NONE; + else + break; + } + M (VXLAN_GPE_IOAM_ENABLE, vxlan_gpe_ioam_enable); + mp->id = htons (id); + mp->trace_ppc = has_ppc_option; + mp->pow_enable = has_pow_option; + mp->trace_enable = has_trace_option; + + + S; + W; + + return (0); +} + + +static int +api_vxlan_gpe_ioam_disable (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + vl_api_vxlan_gpe_ioam_disable_t *mp; + f64 timeout; + + M (VXLAN_GPE_IOAM_DISABLE, vxlan_gpe_ioam_disable); + S; + W; + return 0; +} + +static int +api_vxlan_gpe_ioam_vni_enable (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + + unformat_input_t *line_input = vam->input; + vl_api_vxlan_gpe_ioam_vni_enable_t *mp; + ip4_address_t local4, remote4; + ip6_address_t local6, remote6; + u8 ipv4_set = 0, ipv6_set = 0; + u8 local_set = 0; + u8 remote_set = 0; + u32 vni; + u8 vni_set = 0; + f64 timeout; + + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "local %U", unformat_ip4_address, &local4)) + { + local_set = 1; + ipv4_set = 1; + } + else if (unformat (line_input, "remote %U", + unformat_ip4_address, &remote4)) + { + remote_set = 1; + ipv4_set = 1; + } + else if (unformat (line_input, "local %U", + unformat_ip6_address, &local6)) + { + local_set = 1; + ipv6_set = 1; + } + else if (unformat (line_input, "remote %U", + unformat_ip6_address, &remote6)) + { + remote_set = 1; + ipv6_set = 1; + } + + else if (unformat (line_input, "vni %d", &vni)) + vni_set = 1; + else + { + errmsg ("parse error '%U'\n", format_unformat_error, line_input); + return -99; + } + } + + if (local_set == 0) + { + errmsg ("tunnel local address not specified\n"); + return -99; + } + if (remote_set == 0) + { + errmsg ("tunnel remote address not specified\n"); + return -99; + } + if (ipv4_set && ipv6_set) + { + errmsg ("both IPv4 and IPv6 addresses specified"); + return -99; + } + + if (vni_set == 0) + { + errmsg ("vni not specified\n"); + return -99; + } + + M (VXLAN_GPE_IOAM_VNI_ENABLE, vxlan_gpe_ioam_vni_enable); + + + if (ipv6_set) + { + clib_memcpy (&mp->local, &local6, sizeof (local6)); + clib_memcpy (&mp->remote, &remote6, sizeof (remote6)); + } + else + { + clib_memcpy (&mp->local, &local4, sizeof (local4)); + clib_memcpy (&mp->remote, &remote4, sizeof (remote4)); + } + + mp->vni = ntohl (vni); + mp->is_ipv6 = ipv6_set; + + S; + W; + + return (0); +} + +static int +api_vxlan_gpe_ioam_vni_disable (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + + unformat_input_t *line_input = vam->input; + vl_api_vxlan_gpe_ioam_vni_disable_t *mp; + ip4_address_t local4, remote4; + ip6_address_t local6, remote6; + u8 ipv4_set = 0, ipv6_set = 0; + u8 local_set = 0; + u8 remote_set = 0; + u32 vni; + u8 vni_set = 0; + f64 timeout; + + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "local %U", unformat_ip4_address, &local4)) + { + local_set = 1; + ipv4_set = 1; + } + else if (unformat (line_input, "remote %U", + unformat_ip4_address, &remote4)) + { + remote_set = 1; + ipv4_set = 1; + } + else if (unformat (line_input, "local %U", + unformat_ip6_address, &local6)) + { + local_set = 1; + ipv6_set = 1; + } + else if (unformat (line_input, "remote %U", + unformat_ip6_address, &remote6)) + { + remote_set = 1; + ipv6_set = 1; + } + + else if (unformat (line_input, "vni %d", &vni)) + vni_set = 1; + else + { + errmsg ("parse error '%U'\n", format_unformat_error, line_input); + return -99; + } + } + + if (local_set == 0) + { + errmsg ("tunnel local address not specified\n"); + return -99; + } + if (remote_set == 0) + { + errmsg ("tunnel remote address not specified\n"); + return -99; + } + if (ipv4_set && ipv6_set) + { + errmsg ("both IPv4 and IPv6 addresses specified"); + return -99; + } + + if (vni_set == 0) + { + errmsg ("vni not specified\n"); + return -99; + } + + M (VXLAN_GPE_IOAM_VNI_DISABLE, vxlan_gpe_ioam_vni_disable); + + + if (ipv6_set) + { + clib_memcpy (&mp->local, &local6, sizeof (local6)); + clib_memcpy (&mp->remote, &remote6, sizeof (remote6)); + } + else + { + clib_memcpy (&mp->local, &local4, sizeof (local4)); + clib_memcpy (&mp->remote, &remote4, sizeof (remote4)); + } + + mp->vni = ntohl (vni); + mp->is_ipv6 = ipv6_set; + + S; + W; + + return 0; +} + +static int +api_vxlan_gpe_ioam_transit_enable (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + + unformat_input_t *line_input = vam->input; + vl_api_vxlan_gpe_ioam_transit_enable_t *mp; + ip4_address_t local4; + ip6_address_t local6; + u8 ipv4_set = 0, ipv6_set = 0; + u8 local_set = 0; + u32 outer_fib_index = 0; + f64 timeout; + + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "dst-ip %U", unformat_ip4_address, &local4)) + { + local_set = 1; + ipv4_set = 1; + } + else if (unformat (line_input, "dst-ip %U", + unformat_ip6_address, &local6)) + { + local_set = 1; + ipv6_set = 1; + } + + else if (unformat (line_input, "outer-fib-index %d", &outer_fib_index)) + ; + else + { + errmsg ("parse error '%U'\n", format_unformat_error, line_input); + return -99; + } + } + + if (local_set == 0) + { + errmsg ("destination address not specified\n"); + return -99; + } + if (ipv4_set && ipv6_set) + { + errmsg ("both IPv4 and IPv6 addresses specified"); + return -99; + } + + + M (VXLAN_GPE_IOAM_TRANSIT_ENABLE, vxlan_gpe_ioam_transit_enable); + + + if (ipv6_set) + { + errmsg ("IPv6 currently unsupported"); + return -1; + } + else + { + clib_memcpy (&mp->dst_addr, &local4, sizeof (local4)); + } + + mp->outer_fib_index = htonl (outer_fib_index); + mp->is_ipv6 = ipv6_set; + + S; + W; + + return (0); +} + +static int +api_vxlan_gpe_ioam_transit_disable (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + + unformat_input_t *line_input = vam->input; + vl_api_vxlan_gpe_ioam_transit_disable_t *mp; + ip4_address_t local4; + ip6_address_t local6; + u8 ipv4_set = 0, ipv6_set = 0; + u8 local_set = 0; + u32 outer_fib_index; + f64 timeout; + + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "dst-ip %U", unformat_ip4_address, &local4)) + { + local_set = 1; + ipv4_set = 1; + } + else if (unformat (line_input, "dst-ip %U", + unformat_ip6_address, &local6)) + { + local_set = 1; + ipv6_set = 1; + } + + else if (unformat (line_input, "outer-fib-index %d", &outer_fib_index)) + ; + else + { + errmsg ("parse error '%U'\n", format_unformat_error, line_input); + return -99; + } + } + + if (local_set == 0) + { + errmsg ("destination address not specified\n"); + return -99; + } + if (ipv4_set && ipv6_set) + { + errmsg ("both IPv4 and IPv6 addresses specified"); + return -99; + } + + + M (VXLAN_GPE_IOAM_TRANSIT_DISABLE, vxlan_gpe_ioam_transit_disable); + + + if (ipv6_set) + { + return -1; + } + else + { + clib_memcpy (&mp->dst_addr, &local4, sizeof (local4)); + } + + mp->outer_fib_index = htonl (outer_fib_index); + mp->is_ipv6 = ipv6_set; + + S; + W; + + + return (0); +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(vxlan_gpe_ioam_enable, ""\ + "[trace] [pow] [ppc ]") \ +_(vxlan_gpe_ioam_disable, "") \ +_(vxlan_gpe_ioam_vni_enable, ""\ + "local remote vni ") \ +_(vxlan_gpe_ioam_vni_disable, ""\ + "local remote vni ") \ +_(vxlan_gpe_ioam_transit_enable, ""\ + "dst-ip [outer-fib-index ]") \ +_(vxlan_gpe_ioam_transit_disable, ""\ + "dst-ip [outer-fib-index ]") \ + + +void +vat_api_hookup (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * +vat_plugin_register (vat_main_t * vam) +{ + vxlan_gpe_test_main_t *sm = &vxlan_gpe_test_main; + u8 *name; + + sm->vat_main = vam; + + name = format (0, "ioam_vxlan_gpe_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~ 0) + vat_api_hookup (vam); + + vec_free (name); + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/lb.am b/src/plugins/lb.am new file mode 100644 index 00000000..352358fa --- /dev/null +++ b/src/plugins/lb.am @@ -0,0 +1,42 @@ +# Copyright (c) 2016 Cisco Systems, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +vppapitestplugins_LTLIBRARIES += lb_test_plugin.la +vppplugins_LTLIBRARIES += lb_plugin.la + +lb_plugin_la_SOURCES = \ + lb/lb.c \ + lb/node.c \ + lb/cli.c \ + lb/util.c \ + lb/refcount.c \ + lb/api.c + +BUILT_SOURCES += \ + lb/lb.api.h \ + lb/lb.api.json + +API_FILES += lb/lb.api + +noinst_HEADERS += \ + lb/lb.h \ + lb/util.h \ + lb/refcount.h \ + lb/lbhash.h \ + lb/lb.api.h + +lb_test_plugin_la_SOURCES = \ + lb/lb_test.c \ + lb/lb_plugin.api.h + +# vi:syntax=automake diff --git a/src/plugins/lb/api.c b/src/plugins/lb/api.c new file mode 100644 index 00000000..06c53fa1 --- /dev/null +++ b/src/plugins/lb/api.c @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (lb_main_t * lbm, api_main_t * am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + lbm->msg_id_base); + foreach_vl_msg_name_crc_lb; +#undef _ +} + +/* Macro to finish up custom dump fns */ +#define FINISH \ + vec_add1 (s, 0); \ + vl_print (handle, (char *)s); \ + vec_free (s); \ + return handle; + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+lbm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +static void +vl_api_lb_conf_t_handler +(vl_api_lb_conf_t * mp) +{ + lb_main_t *lbm = &lb_main; + vl_api_lb_conf_reply_t * rmp; + int rv = 0; + + rv = lb_conf((ip4_address_t *)&mp->ip4_src_address, + (ip6_address_t *)mp->ip6_src_address, + mp->sticky_buckets_per_core, + mp->flow_timeout); + + REPLY_MACRO (VL_API_LB_CONF_REPLY); +} + +static void *vl_api_lb_conf_t_print +(vl_api_lb_conf_t *mp, void * handle) +{ + u8 * s; + s = format (0, "SCRIPT: lb_conf "); + s = format (s, "%U ", format_ip4_address, (ip4_address_t *)&mp->ip4_src_address); + s = format (s, "%U ", format_ip6_address, (ip6_address_t *)mp->ip6_src_address); + s = format (s, "%u ", mp->sticky_buckets_per_core); + s = format (s, "%u ", mp->flow_timeout); + FINISH; +} + + +static void +vl_api_lb_add_del_vip_t_handler +(vl_api_lb_add_del_vip_t * mp) +{ + lb_main_t *lbm = &lb_main; + vl_api_lb_conf_reply_t * rmp; + int rv = 0; + ip46_address_t prefix; + memcpy(&prefix.ip6, mp->ip_prefix, sizeof(prefix.ip6)); + + if (mp->is_del) { + u32 vip_index; + if (!(rv = lb_vip_find_index(&prefix, mp->prefix_length, &vip_index))) + rv = lb_vip_del(vip_index); + } else { + u32 vip_index; + lb_vip_type_t type; + if (ip46_prefix_is_ip4(&prefix, mp->prefix_length)) { + type = mp->is_gre4?LB_VIP_TYPE_IP4_GRE4:LB_VIP_TYPE_IP4_GRE6; + } else { + type = mp->is_gre4?LB_VIP_TYPE_IP6_GRE4:LB_VIP_TYPE_IP6_GRE6; + } + + rv = lb_vip_add(&prefix, mp->prefix_length, type, + mp->new_flows_table_length, &vip_index); + } + REPLY_MACRO (VL_API_LB_CONF_REPLY); +} + +static void *vl_api_lb_add_del_vip_t_print +(vl_api_lb_add_del_vip_t *mp, void * handle) +{ + u8 * s; + s = format (0, "SCRIPT: lb_add_del_vip "); + s = format (s, "%U ", format_ip46_prefix, + (ip46_address_t *)mp->ip_prefix, mp->prefix_length, IP46_TYPE_ANY); + s = format (s, "%s ", mp->is_gre4?"gre4":"gre6"); + s = format (s, "%u ", mp->new_flows_table_length); + s = format (s, "%s ", mp->is_del?"del":"add"); + FINISH; +} + +static void +vl_api_lb_add_del_as_t_handler +(vl_api_lb_add_del_as_t * mp) +{ + lb_main_t *lbm = &lb_main; + vl_api_lb_conf_reply_t * rmp; + int rv = 0; + u32 vip_index; + if ((rv = lb_vip_find_index((ip46_address_t *)mp->vip_ip_prefix, + mp->vip_prefix_length, &vip_index))) + goto done; + + if (mp->is_del) + rv = lb_vip_del_ass(vip_index, (ip46_address_t *)mp->as_address, 1); + else + rv = lb_vip_add_ass(vip_index, (ip46_address_t *)mp->as_address, 1); + +done: + REPLY_MACRO (VL_API_LB_CONF_REPLY); +} + +static void *vl_api_lb_add_del_as_t_print +(vl_api_lb_add_del_as_t *mp, void * handle) +{ + u8 * s; + s = format (0, "SCRIPT: lb_add_del_as "); + s = format (s, "%U ", format_ip46_prefix, + (ip46_address_t *)mp->vip_ip_prefix, mp->vip_prefix_length, IP46_TYPE_ANY); + s = format (s, "%U ", format_ip46_address, + (ip46_address_t *)mp->as_address, IP46_TYPE_ANY); + s = format (s, "%s ", mp->is_del?"del":"add"); + FINISH; +} + +/* List of message types that this plugin understands */ +#define foreach_lb_plugin_api_msg \ +_(LB_CONF, lb_conf) \ +_(LB_ADD_DEL_VIP, lb_add_del_vip) \ +_(LB_ADD_DEL_AS, lb_add_del_as) + +static clib_error_t * lb_api_init (vlib_main_t * vm) +{ + lb_main_t *lbm = &lb_main; + u8 *name = format (0, "lb_%08x%c", api_version, 0); + lbm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + lbm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_lb_plugin_api_msg; +#undef _ + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (lbm, &api_main); + + return 0; +} + +VLIB_INIT_FUNCTION (lb_api_init); diff --git a/src/plugins/lb/cli.c b/src/plugins/lb/cli.c new file mode 100644 index 00000000..b59c6426 --- /dev/null +++ b/src/plugins/lb/cli.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +static clib_error_t * +lb_vip_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + ip46_address_t prefix; + u8 plen; + u32 new_len = 1024; + u8 del = 0; + int ret; + u32 gre4 = 0; + lb_vip_type_t type; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + if (!unformat(line_input, "%U", unformat_ip46_prefix, &prefix, &plen, IP46_TYPE_ANY, &plen)) + return clib_error_return (0, "invalid vip prefix: '%U'", + format_unformat_error, line_input); + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(line_input, "new_len %d", &new_len)) + ; + else if (unformat(line_input, "del")) + del = 1; + else if (unformat(line_input, "encap gre4")) + gre4 = 1; + else if (unformat(line_input, "encap gre6")) + gre4 = 0; + else + return clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + } + + unformat_free (line_input); + + + if (ip46_prefix_is_ip4(&prefix, plen)) { + type = (gre4)?LB_VIP_TYPE_IP4_GRE4:LB_VIP_TYPE_IP4_GRE6; + } else { + type = (gre4)?LB_VIP_TYPE_IP6_GRE4:LB_VIP_TYPE_IP6_GRE6; + } + + lb_garbage_collection(); + + u32 index; + if (!del) { + if ((ret = lb_vip_add(&prefix, plen, type, new_len, &index))) { + return clib_error_return (0, "lb_vip_add error %d", ret); + } else { + vlib_cli_output(vm, "lb_vip_add ok %d", index); + } + } else { + if ((ret = lb_vip_find_index(&prefix, plen, &index))) + return clib_error_return (0, "lb_vip_find_index error %d", ret); + else if ((ret = lb_vip_del(index))) + return clib_error_return (0, "lb_vip_del error %d", ret); + } + return NULL; +} + +VLIB_CLI_COMMAND (lb_vip_command, static) = +{ + .path = "lb vip", + .short_help = "lb vip [encap (gre6|gre4)] [new_len ] [del]", + .function = lb_vip_command_fn, +}; + +static clib_error_t * +lb_as_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + ip46_address_t vip_prefix, as_addr; + u8 vip_plen; + ip46_address_t *as_array = 0; + u32 vip_index; + u8 del = 0; + int ret; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + if (!unformat(line_input, "%U", unformat_ip46_prefix, &vip_prefix, &vip_plen, IP46_TYPE_ANY)) + return clib_error_return (0, "invalid as address: '%U'", + format_unformat_error, line_input); + + if ((ret = lb_vip_find_index(&vip_prefix, vip_plen, &vip_index))) + return clib_error_return (0, "lb_vip_find_index error %d", ret); + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(line_input, "%U", unformat_ip46_address, &as_addr, IP46_TYPE_ANY)) { + vec_add1(as_array, as_addr); + } else if (unformat(line_input, "del")) { + del = 1; + } else { + vec_free(as_array); + return clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + } + } + + if (!vec_len(as_array)) { + vec_free(as_array); + return clib_error_return (0, "No AS address provided"); + } + + lb_garbage_collection(); + clib_warning("vip index is %d", vip_index); + + if (del) { + if ((ret = lb_vip_del_ass(vip_index, as_array, vec_len(as_array)))) { + vec_free(as_array); + return clib_error_return (0, "lb_vip_del_ass error %d", ret); + } + } else { + if ((ret = lb_vip_add_ass(vip_index, as_array, vec_len(as_array)))) { + vec_free(as_array); + return clib_error_return (0, "lb_vip_add_ass error %d", ret); + } + } + + vec_free(as_array); + return 0; +} + +VLIB_CLI_COMMAND (lb_as_command, static) = +{ + .path = "lb as", + .short_help = "lb as [
[
[...]]] [del]", + .function = lb_as_command_fn, +}; + +static clib_error_t * +lb_conf_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + lb_main_t *lbm = &lb_main; + unformat_input_t _line_input, *line_input = &_line_input; + ip4_address_t ip4 = lbm->ip4_src_address; + ip6_address_t ip6 = lbm->ip6_src_address; + u32 per_cpu_sticky_buckets = lbm->per_cpu_sticky_buckets; + u32 per_cpu_sticky_buckets_log2 = 0; + u32 flow_timeout = lbm->flow_timeout; + int ret; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat(line_input, "ip4-src-address %U", unformat_ip4_address, &ip4)) + ; + else if (unformat(line_input, "ip6-src-address %U", unformat_ip6_address, &ip6)) + ; + else if (unformat(line_input, "buckets %d", &per_cpu_sticky_buckets)) + ; + else if (unformat(line_input, "buckets-log2 %d", &per_cpu_sticky_buckets_log2)) { + if (per_cpu_sticky_buckets_log2 >= 32) + return clib_error_return (0, "buckets-log2 value is too high"); + per_cpu_sticky_buckets = 1 << per_cpu_sticky_buckets_log2; + } else if (unformat(line_input, "timeout %d", &flow_timeout)) + ; + else + return clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + } + + unformat_free (line_input); + + lb_garbage_collection(); + + if ((ret = lb_conf(&ip4, &ip6, per_cpu_sticky_buckets, flow_timeout))) + return clib_error_return (0, "lb_conf error %d", ret); + + return NULL; +} + +VLIB_CLI_COMMAND (lb_conf_command, static) = +{ + .path = "lb conf", + .short_help = "lb conf [ip4-src-address ] [ip6-src-address ] [buckets ] [timeout ]", + .function = lb_conf_command_fn, +}; + +static clib_error_t * +lb_show_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + vlib_cli_output(vm, "%U", format_lb_main); + return NULL; +} + + +VLIB_CLI_COMMAND (lb_show_command, static) = +{ + .path = "show lb", + .short_help = "show lb", + .function = lb_show_command_fn, +}; + +static clib_error_t * +lb_show_vips_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + unformat_input_t line_input; + lb_main_t *lbm = &lb_main; + lb_vip_t *vip; + u8 verbose = 0; + + if (!unformat_user (input, unformat_line_input, &line_input)) + return 0; + + if (unformat(&line_input, "verbose")) + verbose = 1; + + pool_foreach(vip, lbm->vips, { + vlib_cli_output(vm, "%U\n", verbose?format_lb_vip_detailed:format_lb_vip, vip); + }); + + unformat_free (&line_input); + return NULL; +} + +VLIB_CLI_COMMAND (lb_show_vips_command, static) = +{ + .path = "show lb vips", + .short_help = "show lb vips [verbose]", + .function = lb_show_vips_command_fn, +}; diff --git a/src/plugins/lb/lb.api b/src/plugins/lb/lb.api new file mode 100644 index 00000000..39ee3c8f --- /dev/null +++ b/src/plugins/lb/lb.api @@ -0,0 +1,71 @@ +/** \brief Configure Load-Balancer global parameters + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param ip4_src_address - IPv4 address to be used as source for IPv4 GRE traffic. + @param ip6_src_address - IPv6 address to be used as source for IPv6 GRE traffic. + @param n_sticky_buckets - Number of buckets *per worker thread* in the + established flow table (must be power of 2). + @param flow_timeout - Time in seconds after which, if no packet is received + for a given flow, the flow is removed from the established flow table. +*/ +define lb_conf +{ + u32 client_index; + u32 context; + u32 ip4_src_address; + u8 ip6_src_address[16]; + u32 sticky_buckets_per_core; + u32 flow_timeout; +}; + +define lb_conf_reply { + u32 context; + i32 retval; +}; + +/** \brief Add a virtual address (or prefix) + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param ip_prefix - IP address (IPv4 in lower order 32 bits). + @param prefix_length - IP prefix length (96 + 'IPv4 prefix length' for IPv4). + @param is_gre4 - Encap is ip4 GRE (ip6 GRE otherwise). + @param new_flows_table_length - Size of the new connections flow table used + for this VIP (must be power of 2). + @param is_del - The VIP should be removed. +*/ +define lb_add_del_vip { + u32 client_index; + u32 context; + u8 ip_prefix[16]; + u8 prefix_length; + u8 is_gre4; + u32 new_flows_table_length; + u8 is_del; +}; + +define lb_add_del_vip_reply { + u32 context; + i32 retval; +}; + +/** \brief Add an application server for a given VIP + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param vip_ip_prefix - VIP IP address (IPv4 in lower order 32 bits). + @param vip_ip_prefix - VIP IP prefix length (96 + 'IPv4 prefix length' for IPv4). + @param as_address - The application server address (IPv4 in lower order 32 bits). + @param is_del - The AS should be removed. +*/ +define lb_add_del_as { + u32 client_index; + u32 context; + u8 vip_ip_prefix[16]; + u8 vip_prefix_length; + u8 as_address[16]; + u8 is_del; +}; + +define lb_add_del_as_reply { + u32 context; + i32 retval; +}; diff --git a/src/plugins/lb/lb.c b/src/plugins/lb/lb.c new file mode 100644 index 00000000..1d9b9870 --- /dev/null +++ b/src/plugins/lb/lb.c @@ -0,0 +1,844 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +//GC runs at most once every so many seconds +#define LB_GARBAGE_RUN 60 + +//After so many seconds. It is assumed that inter-core race condition will not occur. +#define LB_CONCURRENCY_TIMEOUT 10 + +lb_main_t lb_main; + +#define lb_get_writer_lock() do {} while(__sync_lock_test_and_set (lb_main.writer_lock, 1)) +#define lb_put_writer_lock() lb_main.writer_lock[0] = 0 + +static void lb_as_stack (lb_as_t *as); + + +const static char * const lb_dpo_gre4_ip4[] = { "lb4-gre4" , NULL }; +const static char * const lb_dpo_gre4_ip6[] = { "lb6-gre4" , NULL }; +const static char* const * const lb_dpo_gre4_nodes[DPO_PROTO_NUM] = + { + [DPO_PROTO_IP4] = lb_dpo_gre4_ip4, + [DPO_PROTO_IP6] = lb_dpo_gre4_ip6, + }; + +const static char * const lb_dpo_gre6_ip4[] = { "lb4-gre6" , NULL }; +const static char * const lb_dpo_gre6_ip6[] = { "lb6-gre6" , NULL }; +const static char* const * const lb_dpo_gre6_nodes[DPO_PROTO_NUM] = + { + [DPO_PROTO_IP4] = lb_dpo_gre6_ip4, + [DPO_PROTO_IP6] = lb_dpo_gre6_ip6, + }; + +u32 lb_hash_time_now(vlib_main_t * vm) +{ + return (u32) (vlib_time_now(vm) + 10000); +} + +u8 *format_lb_main (u8 * s, va_list * args) +{ + vlib_thread_main_t *tm = vlib_get_thread_main(); + lb_main_t *lbm = &lb_main; + s = format(s, "lb_main"); + s = format(s, " ip4-src-address: %U \n", format_ip4_address, &lbm->ip4_src_address); + s = format(s, " ip6-src-address: %U \n", format_ip6_address, &lbm->ip6_src_address); + s = format(s, " #vips: %u\n", pool_elts(lbm->vips)); + s = format(s, " #ass: %u\n", pool_elts(lbm->ass) - 1); + + u32 cpu_index; + for(cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++ ) { + lb_hash_t *h = lbm->per_cpu[cpu_index].sticky_ht; + if (h) { + s = format(s, "core %d\n", cpu_index); + s = format(s, " timeout: %ds\n", h->timeout); + s = format(s, " usage: %d / %d\n", lb_hash_elts(h, lb_hash_time_now(vlib_get_main())), lb_hash_size(h)); + } + } + + return s; +} + +static char *lb_vip_type_strings[] = { + [LB_VIP_TYPE_IP6_GRE6] = "ip6-gre6", + [LB_VIP_TYPE_IP6_GRE4] = "ip6-gre4", + [LB_VIP_TYPE_IP4_GRE6] = "ip4-gre6", + [LB_VIP_TYPE_IP4_GRE4] = "ip4-gre4", +}; + +u8 *format_lb_vip_type (u8 * s, va_list * args) +{ + lb_vip_type_t vipt = va_arg (*args, lb_vip_type_t); + u32 i; + for (i=0; itype, + format_ip46_prefix, &vip->prefix, vip->plen, IP46_TYPE_ANY, + vip->new_flow_table_mask + 1, + pool_elts(vip->as_indexes), + (vip->flags & LB_VIP_FLAGS_USED)?"":" removed"); +} + +u8 *format_lb_as (u8 * s, va_list * args) +{ + lb_as_t *as = va_arg (*args, lb_as_t *); + return format(s, "%U %s", format_ip46_address, + &as->address, IP46_TYPE_ANY, + (as->flags & LB_AS_FLAGS_USED)?"used":"removed"); +} + +u8 *format_lb_vip_detailed (u8 * s, va_list * args) +{ + lb_main_t *lbm = &lb_main; + lb_vip_t *vip = va_arg (*args, lb_vip_t *); + uword indent = format_get_indent (s); + + s = format(s, "%U %U [%u] %U%s\n" + "%U new_size:%u\n", + format_white_space, indent, + format_lb_vip_type, vip->type, + vip - lbm->vips, format_ip46_prefix, &vip->prefix, vip->plen, IP46_TYPE_ANY, + (vip->flags & LB_VIP_FLAGS_USED)?"":" removed", + format_white_space, indent, + vip->new_flow_table_mask + 1); + + //Print counters + s = format(s, "%U counters:\n", + format_white_space, indent); + u32 i; + for (i=0; ivip_counters[i].name, + vlib_get_simple_counter(&lbm->vip_counters[i], vip - lbm->vips)); + + + s = format(s, "%U #as:%u\n", + format_white_space, indent, + pool_elts(vip->as_indexes)); + + //Let's count the buckets for each AS + u32 *count = 0; + vec_validate(count, pool_len(lbm->ass)); //Possibly big alloc for not much... + lb_new_flow_entry_t *nfe; + vec_foreach(nfe, vip->new_flow_table) + count[nfe->as_index]++; + + lb_as_t *as; + u32 *as_index; + pool_foreach(as_index, vip->as_indexes, { + as = &lbm->ass[*as_index]; + s = format(s, "%U %U %d buckets %d flows dpo:%u %s\n", + format_white_space, indent, + format_ip46_address, &as->address, IP46_TYPE_ANY, + count[as - lbm->ass], + vlib_refcount_get(&lbm->as_refcount, as - lbm->ass), + as->dpo.dpoi_index, + (as->flags & LB_AS_FLAGS_USED)?"used":" removed"); + }); + + vec_free(count); + + /* + s = format(s, "%U new flows table:\n", format_white_space, indent); + lb_new_flow_entry_t *nfe; + vec_foreach(nfe, vip->new_flow_table) { + s = format(s, "%U %d: %d\n", format_white_space, indent, nfe - vip->new_flow_table, nfe->as_index); + } + */ + return s; +} + +typedef struct { + u32 as_index; + u32 last; + u32 skip; +} lb_pseudorand_t; + +static int lb_pseudorand_compare(void *a, void *b) +{ + lb_as_t *asa, *asb; + lb_main_t *lbm = &lb_main; + asa = &lbm->ass[((lb_pseudorand_t *)a)->as_index]; + asb = &lbm->ass[((lb_pseudorand_t *)b)->as_index]; + return memcmp(&asa->address, &asb->address, sizeof(asb->address)); +} + +static void lb_vip_garbage_collection(lb_vip_t *vip) +{ + lb_main_t *lbm = &lb_main; + ASSERT (lbm->writer_lock[0]); + + u32 now = (u32) vlib_time_now(vlib_get_main()); + if (!clib_u32_loop_gt(now, vip->last_garbage_collection + LB_GARBAGE_RUN)) + return; + + vip->last_garbage_collection = now; + lb_as_t *as; + u32 *as_index; + pool_foreach(as_index, vip->as_indexes, { + as = &lbm->ass[*as_index]; + if (!(as->flags & LB_AS_FLAGS_USED) && //Not used + clib_u32_loop_gt(now, as->last_used + LB_CONCURRENCY_TIMEOUT) && //Not recently used + (vlib_refcount_get(&lbm->as_refcount, as - lbm->ass) == 0)) + { //Not referenced + fib_entry_child_remove(as->next_hop_fib_entry_index, + as->next_hop_child_index); + fib_table_entry_delete_index(as->next_hop_fib_entry_index, + FIB_SOURCE_RR); + as->next_hop_fib_entry_index = FIB_NODE_INDEX_INVALID; + + pool_put(vip->as_indexes, as_index); + pool_put(lbm->ass, as); + } + }); +} + +void lb_garbage_collection() +{ + lb_main_t *lbm = &lb_main; + lb_get_writer_lock(); + lb_vip_t *vip; + u32 *to_be_removed_vips = 0, *i; + pool_foreach(vip, lbm->vips, { + lb_vip_garbage_collection(vip); + + if (!(vip->flags & LB_VIP_FLAGS_USED) && + (pool_elts(vip->as_indexes) == 0)) { + vec_add1(to_be_removed_vips, vip - lbm->vips); + } + }); + + vec_foreach(i, to_be_removed_vips) { + vip = &lbm->vips[*i]; + pool_put(lbm->vips, vip); + pool_free(vip->as_indexes); + } + + vec_free(to_be_removed_vips); + lb_put_writer_lock(); +} + +static void lb_vip_update_new_flow_table(lb_vip_t *vip) +{ + lb_main_t *lbm = &lb_main; + lb_new_flow_entry_t *old_table; + u32 i, *as_index; + lb_new_flow_entry_t *new_flow_table = 0; + lb_as_t *as; + lb_pseudorand_t *pr, *sort_arr = 0; + u32 count; + + ASSERT (lbm->writer_lock[0]); //We must have the lock + + //Check if some AS is configured or not + i = 0; + pool_foreach(as_index, vip->as_indexes, { + as = &lbm->ass[*as_index]; + if (as->flags & LB_AS_FLAGS_USED) { //Not used anymore + i = 1; + goto out; //Not sure 'break' works in this macro-loop + } + }); + +out: + if (i == 0) { + //Only the default. i.e. no AS + vec_validate(new_flow_table, vip->new_flow_table_mask); + for (i=0; ias_indexes)); + + i = 0; + pool_foreach(as_index, vip->as_indexes, { + as = &lbm->ass[*as_index]; + if (!(as->flags & LB_AS_FLAGS_USED)) //Not used anymore + continue; + + sort_arr[i].as_index = as - lbm->ass; + i++; + }); + _vec_len(sort_arr) = i; + + vec_sort_with_function(sort_arr, lb_pseudorand_compare); + + //Now let's pseudo-randomly generate permutations + vec_foreach(pr, sort_arr) { + lb_as_t *as = &lbm->ass[pr->as_index]; + + u64 seed = clib_xxhash(as->address.as_u64[0] ^ + as->address.as_u64[1]); + /* We have 2^n buckets. + * skip must be prime with 2^n. + * So skip must be odd. + * MagLev actually state that M should be prime, + * but this has a big computation cost (% operation). + * Using 2^n is more better (& operation). + */ + pr->skip = ((seed & 0xffffffff) | 1) & vip->new_flow_table_mask; + pr->last = (seed >> 32) & vip->new_flow_table_mask; + } + + //Let's create a new flow table + vec_validate(new_flow_table, vip->new_flow_table_mask); + for (i=0; ilast; + pr->last = (pr->last + pr->skip) & vip->new_flow_table_mask; + if (new_flow_table[last].as_index == ~0) { + new_flow_table[last].as_index = pr->as_index; + break; + } + } + done++; + if (done == vec_len(new_flow_table)) + goto finished; + } + } + + vec_free(sort_arr); + +finished: + +//Count number of changed entries + count = 0; + for (i=0; inew_flow_table == 0 || + new_flow_table[i].as_index != vip->new_flow_table[i].as_index) + count++; + + old_table = vip->new_flow_table; + vip->new_flow_table = new_flow_table; + vec_free(old_table); +} + +int lb_conf(ip4_address_t *ip4_address, ip6_address_t *ip6_address, + u32 per_cpu_sticky_buckets, u32 flow_timeout) +{ + lb_main_t *lbm = &lb_main; + + if (!is_pow2(per_cpu_sticky_buckets)) + return VNET_API_ERROR_INVALID_MEMORY_SIZE; + + lb_get_writer_lock(); //Not exactly necessary but just a reminder that it exists for my future self + lbm->ip4_src_address = *ip4_address; + lbm->ip6_src_address = *ip6_address; + lbm->per_cpu_sticky_buckets = per_cpu_sticky_buckets; + lbm->flow_timeout = flow_timeout; + lb_put_writer_lock(); + return 0; +} + +static +int lb_vip_find_index_with_lock(ip46_address_t *prefix, u8 plen, u32 *vip_index) +{ + lb_main_t *lbm = &lb_main; + lb_vip_t *vip; + ASSERT (lbm->writer_lock[0]); //This must be called with the lock owned + ip46_prefix_normalize(prefix, plen); + pool_foreach(vip, lbm->vips, { + if ((vip->flags & LB_AS_FLAGS_USED) && + vip->plen == plen && + vip->prefix.as_u64[0] == prefix->as_u64[0] && + vip->prefix.as_u64[1] == prefix->as_u64[1]) { + *vip_index = vip - lbm->vips; + return 0; + } + }); + return VNET_API_ERROR_NO_SUCH_ENTRY; +} + +int lb_vip_find_index(ip46_address_t *prefix, u8 plen, u32 *vip_index) +{ + int ret; + lb_get_writer_lock(); + ret = lb_vip_find_index_with_lock(prefix, plen, vip_index); + lb_put_writer_lock(); + return ret; +} + +static int lb_as_find_index_vip(lb_vip_t *vip, ip46_address_t *address, u32 *as_index) +{ + lb_main_t *lbm = &lb_main; + ASSERT (lbm->writer_lock[0]); //This must be called with the lock owned + lb_as_t *as; + u32 *asi; + pool_foreach(asi, vip->as_indexes, { + as = &lbm->ass[*asi]; + if (as->vip_index == (vip - lbm->vips) && + as->address.as_u64[0] == address->as_u64[0] && + as->address.as_u64[1] == address->as_u64[1]) { + *as_index = as - lbm->ass; + return 0; + } + }); + return -1; +} + +int lb_vip_add_ass(u32 vip_index, ip46_address_t *addresses, u32 n) +{ + lb_main_t *lbm = &lb_main; + lb_get_writer_lock(); + lb_vip_t *vip; + if (!(vip = lb_vip_get_by_index(vip_index))) { + lb_put_writer_lock(); + return VNET_API_ERROR_NO_SUCH_ENTRY; + } + + ip46_type_t type = lb_vip_is_gre4(vip)?IP46_TYPE_IP4:IP46_TYPE_IP6; + u32 *to_be_added = 0; + u32 *to_be_updated = 0; + u32 i; + u32 *ip; + + //Sanity check + while (n--) { + + if (!lb_as_find_index_vip(vip, &addresses[n], &i)) { + if (lbm->ass[i].flags & LB_AS_FLAGS_USED) { + vec_free(to_be_added); + vec_free(to_be_updated); + lb_put_writer_lock(); + return VNET_API_ERROR_VALUE_EXIST; + } + vec_add1(to_be_updated, i); + goto next; + } + + if (ip46_address_type(&addresses[n]) != type) { + vec_free(to_be_added); + vec_free(to_be_updated); + lb_put_writer_lock(); + return VNET_API_ERROR_INVALID_ADDRESS_FAMILY; + } + + if (n) { + u32 n2 = n; + while(n2--) //Check for duplicates + if (addresses[n2].as_u64[0] == addresses[n].as_u64[0] && + addresses[n2].as_u64[1] == addresses[n].as_u64[1]) + goto next; + } + + vec_add1(to_be_added, n); + +next: + continue; + } + + //Update reused ASs + vec_foreach(ip, to_be_updated) { + lbm->ass[*ip].flags = LB_AS_FLAGS_USED; + } + vec_free(to_be_updated); + + //Create those who have to be created + vec_foreach(ip, to_be_added) { + lb_as_t *as; + u32 *as_index; + pool_get(lbm->ass, as); + as->address = addresses[*ip]; + as->flags = LB_AS_FLAGS_USED; + as->vip_index = vip_index; + pool_get(vip->as_indexes, as_index); + *as_index = as - lbm->ass; + + /* + * become a child of the FIB entry + * so we are informed when its forwarding changes + */ + fib_prefix_t nh = {}; + if (lb_vip_is_gre4(vip)) { + nh.fp_addr.ip4 = as->address.ip4; + nh.fp_len = 32; + nh.fp_proto = FIB_PROTOCOL_IP4; + } else { + nh.fp_addr.ip6 = as->address.ip6; + nh.fp_len = 128; + nh.fp_proto = FIB_PROTOCOL_IP6; + } + + as->next_hop_fib_entry_index = + fib_table_entry_special_add(0, + &nh, + FIB_SOURCE_RR, + FIB_ENTRY_FLAG_NONE, + ADJ_INDEX_INVALID); + as->next_hop_child_index = + fib_entry_child_add(as->next_hop_fib_entry_index, + lbm->fib_node_type, + as - lbm->ass); + + lb_as_stack(as); + } + vec_free(to_be_added); + + //Recompute flows + lb_vip_update_new_flow_table(vip); + + //Garbage collection maybe + lb_vip_garbage_collection(vip); + + lb_put_writer_lock(); + return 0; +} + +int lb_vip_del_ass_withlock(u32 vip_index, ip46_address_t *addresses, u32 n) +{ + lb_main_t *lbm = &lb_main; + u32 now = (u32) vlib_time_now(vlib_get_main()); + u32 *ip = 0; + + lb_vip_t *vip; + if (!(vip = lb_vip_get_by_index(vip_index))) { + return VNET_API_ERROR_NO_SUCH_ENTRY; + } + + u32 *indexes = NULL; + while (n--) { + u32 i; + if (lb_as_find_index_vip(vip, &addresses[n], &i)) { + vec_free(indexes); + return VNET_API_ERROR_NO_SUCH_ENTRY; + } + + if (n) { //Check for duplicates + u32 n2 = n - 1; + while(n2--) { + if (addresses[n2].as_u64[0] == addresses[n].as_u64[0] && + addresses[n2].as_u64[1] == addresses[n].as_u64[1]) + goto next; + } + } + + vec_add1(indexes, i); +next: + continue; + } + + //Garbage collection maybe + lb_vip_garbage_collection(vip); + + if (indexes != NULL) { + vec_foreach(ip, indexes) { + lbm->ass[*ip].flags &= ~LB_AS_FLAGS_USED; + lbm->ass[*ip].last_used = now; + } + + //Recompute flows + lb_vip_update_new_flow_table(vip); + } + + vec_free(indexes); + return 0; +} + +int lb_vip_del_ass(u32 vip_index, ip46_address_t *addresses, u32 n) +{ + lb_get_writer_lock(); + int ret = lb_vip_del_ass_withlock(vip_index, addresses, n); + lb_put_writer_lock(); + return ret; +} + +/** + * Add the VIP adjacency to the ip4 or ip6 fib + */ +static void lb_vip_add_adjacency(lb_main_t *lbm, lb_vip_t *vip) +{ + dpo_proto_t proto = 0; + dpo_id_t dpo = DPO_INVALID; + fib_prefix_t pfx = {}; + if (lb_vip_is_ip4(vip)) { + pfx.fp_addr.ip4 = vip->prefix.ip4; + pfx.fp_len = vip->plen - 96; + pfx.fp_proto = FIB_PROTOCOL_IP4; + proto = DPO_PROTO_IP4; + } else { + pfx.fp_addr.ip6 = vip->prefix.ip6; + pfx.fp_len = vip->plen; + pfx.fp_proto = FIB_PROTOCOL_IP6; + proto = DPO_PROTO_IP6; + } + dpo_set(&dpo, lb_vip_is_gre4(vip)?lbm->dpo_gre4_type:lbm->dpo_gre6_type, + proto, vip - lbm->vips); + fib_table_entry_special_dpo_add(0, + &pfx, + FIB_SOURCE_PLUGIN_HI, + FIB_ENTRY_FLAG_EXCLUSIVE, + &dpo); + dpo_reset(&dpo); +} + +/** + * Deletes the adjacency associated with the VIP + */ +static void lb_vip_del_adjacency(lb_main_t *lbm, lb_vip_t *vip) +{ + fib_prefix_t pfx = {}; + if (lb_vip_is_ip4(vip)) { + pfx.fp_addr.ip4 = vip->prefix.ip4; + pfx.fp_len = vip->plen - 96; + pfx.fp_proto = FIB_PROTOCOL_IP4; + } else { + pfx.fp_addr.ip6 = vip->prefix.ip6; + pfx.fp_len = vip->plen; + pfx.fp_proto = FIB_PROTOCOL_IP6; + } + fib_table_entry_special_remove(0, &pfx, FIB_SOURCE_PLUGIN_HI); +} + +int lb_vip_add(ip46_address_t *prefix, u8 plen, lb_vip_type_t type, u32 new_length, u32 *vip_index) +{ + lb_main_t *lbm = &lb_main; + lb_vip_t *vip; + lb_get_writer_lock(); + ip46_prefix_normalize(prefix, plen); + + if (!lb_vip_find_index_with_lock(prefix, plen, vip_index)) { + lb_put_writer_lock(); + return VNET_API_ERROR_VALUE_EXIST; + } + + if (!is_pow2(new_length)) { + lb_put_writer_lock(); + return VNET_API_ERROR_INVALID_MEMORY_SIZE; + } + + if (ip46_prefix_is_ip4(prefix, plen) && + (type != LB_VIP_TYPE_IP4_GRE4) && + (type != LB_VIP_TYPE_IP4_GRE6)) + return VNET_API_ERROR_INVALID_ADDRESS_FAMILY; + + + //Allocate + pool_get(lbm->vips, vip); + + //Init + vip->prefix = *prefix; + vip->plen = plen; + vip->last_garbage_collection = (u32) vlib_time_now(vlib_get_main()); + vip->type = type; + vip->flags = LB_VIP_FLAGS_USED; + vip->as_indexes = 0; + + //Validate counters + u32 i; + for (i = 0; i < LB_N_VIP_COUNTERS; i++) { + vlib_validate_simple_counter(&lbm->vip_counters[i], vip - lbm->vips); + vlib_zero_simple_counter(&lbm->vip_counters[i], vip - lbm->vips); + } + + //Configure new flow table + vip->new_flow_table_mask = new_length - 1; + vip->new_flow_table = 0; + + //Create a new flow hash table full of the default entry + lb_vip_update_new_flow_table(vip); + + //Create adjacency to direct traffic + lb_vip_add_adjacency(lbm, vip); + + //Return result + *vip_index = vip - lbm->vips; + + lb_put_writer_lock(); + return 0; +} + +int lb_vip_del(u32 vip_index) +{ + lb_main_t *lbm = &lb_main; + lb_vip_t *vip; + lb_get_writer_lock(); + if (!(vip = lb_vip_get_by_index(vip_index))) { + lb_put_writer_lock(); + return VNET_API_ERROR_NO_SUCH_ENTRY; + } + + //FIXME: This operation is actually not working + //We will need to remove state before performing this. + + { + //Remove all ASs + ip46_address_t *ass = 0; + lb_as_t *as; + u32 *as_index; + pool_foreach(as_index, vip->as_indexes, { + as = &lbm->ass[*as_index]; + vec_add1(ass, as->address); + }); + if (vec_len(ass)) + lb_vip_del_ass_withlock(vip_index, ass, vec_len(ass)); + vec_free(ass); + } + + //Delete adjacency + lb_vip_del_adjacency(lbm, vip); + + //Set the VIP as unused + vip->flags &= ~LB_VIP_FLAGS_USED; + + lb_put_writer_lock(); + return 0; +} + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, + vnet_plugin_handoff_t * h, + int from_early_init) +{ + clib_error_t *error = 0; + return error; +} + + +u8 *format_lb_dpo (u8 * s, va_list * va) +{ + index_t index = va_arg (*va, index_t); + CLIB_UNUSED(u32 indent) = va_arg (*va, u32); + lb_main_t *lbm = &lb_main; + lb_vip_t *vip = pool_elt_at_index (lbm->vips, index); + return format (s, "%U", format_lb_vip, vip); +} + +static void lb_dpo_lock (dpo_id_t *dpo) {} +static void lb_dpo_unlock (dpo_id_t *dpo) {} + +static fib_node_t * +lb_fib_node_get_node (fib_node_index_t index) +{ + lb_main_t *lbm = &lb_main; + lb_as_t *as = pool_elt_at_index (lbm->ass, index); + return (&as->fib_node); +} + +static void +lb_fib_node_last_lock_gone (fib_node_t *node) +{ +} + +static lb_as_t * +lb_as_from_fib_node (fib_node_t *node) +{ + return ((lb_as_t*)(((char*)node) - + STRUCT_OFFSET_OF(lb_as_t, fib_node))); +} + +static void +lb_as_stack (lb_as_t *as) +{ + lb_main_t *lbm = &lb_main; + lb_vip_t *vip = &lbm->vips[as->vip_index]; + dpo_stack(lb_vip_is_gre4(vip)?lbm->dpo_gre4_type:lbm->dpo_gre6_type, + lb_vip_is_ip4(vip)?DPO_PROTO_IP4:DPO_PROTO_IP6, + &as->dpo, + fib_entry_contribute_ip_forwarding( + as->next_hop_fib_entry_index)); +} + +static fib_node_back_walk_rc_t +lb_fib_node_back_walk_notify (fib_node_t *node, + fib_node_back_walk_ctx_t *ctx) +{ + lb_as_stack(lb_as_from_fib_node(node)); + return (FIB_NODE_BACK_WALK_CONTINUE); +} + +clib_error_t * +lb_init (vlib_main_t * vm) +{ + vlib_thread_main_t *tm = vlib_get_thread_main (); + lb_main_t *lbm = &lb_main; + lb_as_t *default_as; + fib_node_vft_t lb_fib_node_vft = { + .fnv_get = lb_fib_node_get_node, + .fnv_last_lock = lb_fib_node_last_lock_gone, + .fnv_back_walk = lb_fib_node_back_walk_notify, + }; + dpo_vft_t lb_vft = { + .dv_lock = lb_dpo_lock, + .dv_unlock = lb_dpo_unlock, + .dv_format = format_lb_dpo, + }; + + lbm->vips = 0; + lbm->per_cpu = 0; + vec_validate(lbm->per_cpu, tm->n_vlib_mains - 1); + lbm->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); + lbm->writer_lock[0] = 0; + lbm->per_cpu_sticky_buckets = LB_DEFAULT_PER_CPU_STICKY_BUCKETS; + lbm->flow_timeout = LB_DEFAULT_FLOW_TIMEOUT; + lbm->ip4_src_address.as_u32 = 0xffffffff; + lbm->ip6_src_address.as_u64[0] = 0xffffffffffffffffL; + lbm->ip6_src_address.as_u64[1] = 0xffffffffffffffffL; + lbm->dpo_gre4_type = dpo_register_new_type(&lb_vft, lb_dpo_gre4_nodes); + lbm->dpo_gre6_type = dpo_register_new_type(&lb_vft, lb_dpo_gre6_nodes); + lbm->fib_node_type = fib_node_register_new_type(&lb_fib_node_vft); + + //Init AS reference counters + vlib_refcount_init(&lbm->as_refcount); + + //Allocate and init default AS. + lbm->ass = 0; + pool_get(lbm->ass, default_as); + default_as->flags = 0; + default_as->dpo.dpoi_next_node = LB_NEXT_DROP; + default_as->vip_index = ~0; + default_as->address.ip6.as_u64[0] = 0xffffffffffffffffL; + default_as->address.ip6.as_u64[1] = 0xffffffffffffffffL; + +#define _(a,b,c) lbm->vip_counters[c].name = b; + lb_foreach_vip_counter +#undef _ + return NULL; +} + +VLIB_INIT_FUNCTION (lb_init); diff --git a/src/plugins/lb/lb.h b/src/plugins/lb/lb.h new file mode 100644 index 00000000..882b9b30 --- /dev/null +++ b/src/plugins/lb/lb.h @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * lb-plugin implements a MagLev-like load balancer. + * http://research.google.com/pubs/pub44824.html + * + * It hasn't been tested for interoperability with the original MagLev + * but intends to provide similar functionality. + * The load-balancer receives traffic destined to VIP (Virtual IP) + * addresses from one or multiple(ECMP) routers. + * The load-balancer tunnels the traffic toward many application servers + * ensuring session stickyness (i.e. that a single sessions is tunneled + * towards a single application server). + * + */ + +#ifndef LB_PLUGIN_LB_LB_H_ +#define LB_PLUGIN_LB_LB_H_ + +#include +#include + +#include +#include +#include +#include + +#include + +#define LB_DEFAULT_PER_CPU_STICKY_BUCKETS 1 << 10 +#define LB_DEFAULT_FLOW_TIMEOUT 40 + +typedef enum { + LB_NEXT_DROP, + LB_N_NEXT, +} lb_next_t; + +/** + * Each VIP is configured with a set of + * application server. + */ +typedef struct { + /** + * Registration to FIB event. + */ + fib_node_t fib_node; + + /** + * Destination address used to tunnel traffic towards + * that application server. + * The address is also used as ID and pseudo-random + * seed for the load-balancing process. + */ + ip46_address_t address; + + /** + * ASs are indexed by address and VIP Index. + * Which means there will be duplicated if the same server + * address is used for multiple VIPs. + */ + u32 vip_index; + + /** + * Some per-AS flags. + * For now only LB_AS_FLAGS_USED is defined. + */ + u8 flags; + +#define LB_AS_FLAGS_USED 0x1 + + /** + * Rotating timestamp of when LB_AS_FLAGS_USED flag was last set. + * + * AS removal is based on garbage collection and reference counting. + * When an AS is removed, there is a race between configuration core + * and worker cores which may still add a reference while it should not + * be used. This timestamp is used to not remove the AS while a race condition + * may happen. + */ + u32 last_used; + + /** + * The FIB entry index for the next-hop + */ + fib_node_index_t next_hop_fib_entry_index; + + /** + * The child index on the FIB entry + */ + u32 next_hop_child_index; + + /** + * The next DPO in the graph to follow. + */ + dpo_id_t dpo; + +} lb_as_t; + +format_function_t format_lb_as; + +typedef struct { + u32 as_index; +} lb_new_flow_entry_t; + +#define lb_foreach_vip_counter \ + _(NEXT_PACKET, "packet from existing sessions", 0) \ + _(FIRST_PACKET, "first session packet", 1) \ + _(UNTRACKED_PACKET, "untracked packet", 2) \ + _(NO_SERVER, "no server configured", 3) + +typedef enum { +#define _(a,b,c) LB_VIP_COUNTER_##a = c, + lb_foreach_vip_counter +#undef _ + LB_N_VIP_COUNTERS +} lb_vip_counter_t; + +/** + * The load balancer supports IPv4 and IPv6 traffic + * and GRE4 and GRE6 encap. + */ +typedef enum { + LB_VIP_TYPE_IP6_GRE6, + LB_VIP_TYPE_IP6_GRE4, + LB_VIP_TYPE_IP4_GRE6, + LB_VIP_TYPE_IP4_GRE4, + LB_VIP_N_TYPES, +} lb_vip_type_t; + +format_function_t format_lb_vip_type; +unformat_function_t unformat_lb_vip_type; + +/** + * Load balancing service is provided per VIP. + * In this data model, a VIP can be a whole prefix. + * But load balancing only + * occurs on a per-source-address/port basis. Meaning that if a given source + * reuses the same port for multiple destinations within the same VIP, + * they will be considered as a single flow. + */ +typedef struct { + + //Runtime + + /** + * Vector mapping (flow-hash & new_connect_table_mask) to AS index. + * This is used for new flows. + */ + lb_new_flow_entry_t *new_flow_table; + + /** + * New flows table length - 1 + * (length MUST be a power of 2) + */ + u32 new_flow_table_mask; + + /** + * Last time garbage collection was run to free the ASs. + */ + u32 last_garbage_collection; + + //Not runtime + + /** + * A Virtual IP represents a given service delivered + * by a set of application servers. It can be a single + * address or a prefix. + * IPv4 prefixes are encoded using IPv4-in-IPv6 embedded address + * (i.e. ::/96 prefix). + */ + ip46_address_t prefix; + + /** + * The VIP prefix length. + * In case of IPv4, plen = 96 + ip4_plen. + */ + u8 plen; + + /** + * The type of traffic for this. + * LB_TYPE_UNDEFINED if unknown. + */ + lb_vip_type_t type; + + /** + * Flags related to this VIP. + * LB_VIP_FLAGS_USED means the VIP is active. + * When it is not set, the VIP in the process of being removed. + * We cannot immediately remove a VIP because the VIP index still may be stored + * in the adjacency index. + */ + u8 flags; +#define LB_VIP_FLAGS_USED 0x1 + + /** + * Pool of AS indexes used for this VIP. + * This also includes ASs that have been removed (but are still referenced). + */ + u32 *as_indexes; +} lb_vip_t; + +#define lb_vip_is_ip4(vip) ((vip)->type == LB_VIP_TYPE_IP4_GRE6 || (vip)->type == LB_VIP_TYPE_IP4_GRE4) +#define lb_vip_is_gre4(vip) ((vip)->type == LB_VIP_TYPE_IP6_GRE4 || (vip)->type == LB_VIP_TYPE_IP4_GRE4) +format_function_t format_lb_vip; +format_function_t format_lb_vip_detailed; + +typedef struct { + /** + * Each CPU has its own sticky flow hash table. + * One single table is used for all VIPs. + */ + lb_hash_t *sticky_ht; +} lb_per_cpu_t; + +typedef struct { + /** + * Pool of all Virtual IPs + */ + lb_vip_t *vips; + + /** + * Pool of ASs. + * ASs are referenced by address and vip index. + * The first element (index 0) is special and used only to fill + * new_flow_tables when no AS has been configured. + */ + lb_as_t *ass; + + /** + * Each AS has an associated reference counter. + * As ass[0] has a special meaning, its associated counter + * starts at 0 and is decremented instead. i.e. do not use it. + */ + vlib_refcount_t as_refcount; + + /** + * Some global data is per-cpu + */ + lb_per_cpu_t *per_cpu; + + /** + * Node next index for IP adjacencies, for each of the traffic types. + */ + u32 ip_lookup_next_index[LB_VIP_N_TYPES]; + + /** + * Source address used in IPv6 encapsulated traffic + */ + ip6_address_t ip6_src_address; + + /** + * Source address used for IPv4 encapsulated traffic + */ + ip4_address_t ip4_src_address; + + /** + * Number of buckets in the per-cpu sticky hash table. + */ + u32 per_cpu_sticky_buckets; + + /** + * Flow timeout in seconds. + */ + u32 flow_timeout; + + /** + * Per VIP counter + */ + vlib_simple_counter_main_t vip_counters[LB_N_VIP_COUNTERS]; + + /** + * DPO used to send packet from IP4/6 lookup to LB node. + */ + dpo_type_t dpo_gre4_type; + dpo_type_t dpo_gre6_type; + + /** + * Node type for registering to fib changes. + */ + fib_node_type_t fib_node_type; + + /** + * API dynamically registered base ID. + */ + u16 msg_id_base; + + volatile u32 *writer_lock; +} lb_main_t; + +extern lb_main_t lb_main; +extern vlib_node_registration_t lb6_node; +extern vlib_node_registration_t lb4_node; + +/** + * Fix global load-balancer parameters. + * @param ip4_address IPv4 source address used for encapsulated traffic + * @param ip6_address IPv6 source address used for encapsulated traffic + * @return 0 on success. VNET_LB_ERR_XXX on error + */ +int lb_conf(ip4_address_t *ip4_address, ip6_address_t *ip6_address, + u32 sticky_buckets, u32 flow_timeout); + +int lb_vip_add(ip46_address_t *prefix, u8 plen, lb_vip_type_t type, + u32 new_length, u32 *vip_index); +int lb_vip_del(u32 vip_index); + +int lb_vip_find_index(ip46_address_t *prefix, u8 plen, u32 *vip_index); + +#define lb_vip_get_by_index(index) (pool_is_free_index(lb_main.vips, index)?NULL:pool_elt_at_index(lb_main.vips, index)) + +int lb_vip_add_ass(u32 vip_index, ip46_address_t *addresses, u32 n); +int lb_vip_del_ass(u32 vip_index, ip46_address_t *addresses, u32 n); + +u32 lb_hash_time_now(vlib_main_t * vm); + +void lb_garbage_collection(); + +format_function_t format_lb_main; + +#endif /* LB_PLUGIN_LB_LB_H_ */ diff --git a/src/plugins/lb/lb_plugin_doc.md b/src/plugins/lb/lb_plugin_doc.md new file mode 100644 index 00000000..c7885ffb --- /dev/null +++ b/src/plugins/lb/lb_plugin_doc.md @@ -0,0 +1,141 @@ +# Load Balancer plugin for VPP {#lb_plugin_doc} + +## Version + +The load balancer plugin is currently in *beta* version. +Both CLIs and APIs are subject to *heavy* changes. +Wich also means feedback is really welcome regarding features, apis, etc... + +## Overview + +This plugin provides load balancing for VPP in a way that is largely inspired +from Google's MagLev: http://research.google.com/pubs/pub44824.html + +The load balancer is configured with a set of Virtual IPs (VIP, which can be +prefixes), and for each VIP, with a set of Application Server addresses (ASs). + +Traffic received for a given VIP (or VIP prefix) is tunneled using GRE towards +the different ASs in a way that (tries to) ensure that a given session will +always be tunneled to the same AS. + +Both VIPs or ASs can be IPv4 or IPv6, but for a given VIP, all ASs must be using +the same encap. type (i.e. IPv4+GRE or IPv6+GRE). Meaning that for a given VIP, +all AS addresses must be of the same family. + +## Performances + +The load balancer has been tested up to 1 millions flows and still forwards more +than 3Mpps per core in such circumstances. +Although 3Mpps seems already good, it is likely that performances will be improved +in next versions. + +## Configuration + +### Global LB parameters + +The load balancer needs to be configured with some parameters: + + lb conf [ip4-src-address ] [ip6-src-address ] + [buckets ] [timeout ] + +ip4-src-address: the source address used to send encap. packets using IPv4. + +ip6-src-address: the source address used to send encap. packets using IPv6. + +buckets: the *per-thread* established-connexions-table number of buckets. + +timeout: the number of seconds a connection will remain in the + established-connexions-table while no packet for this flow + is received. + + +### Configure the VIPs + + lb vip [encap (gre6|gre4)] [new_len ] [del] + +new_len is the size of the new-connection-table. It should be 1 or 2 orders of +magnitude bigger than the number of ASs for the VIP in order to ensure a good +load balancing. + +Examples: + + lb vip 2002::/16 encap gre6 new_len 1024 + lb vip 2003::/16 encap gre4 new_len 2048 + lb vip 80.0.0.0/8 encap gre6 new_len 16 + lb vip 90.0.0.0/8 encap gre4 new_len 1024 + +### Configure the ASs (for each VIP) + + lb as [
[
[...]]] [del] + +You can add (or delete) as many ASs at a time (for a single VIP). +Note that the AS address family must correspond to the VIP encap. IP family. + +Examples: + + lb as 2002::/16 2001::2 2001::3 2001::4 + lb as 2003::/16 10.0.0.1 10.0.0.2 + lb as 80.0.0.0/8 2001::2 + lb as 90.0.0.0/8 10.0.0.1 + + + +## Monitoring + +The plugin provides quite a bunch of counters and information. +These are still subject to quite significant changes. + + show lb + show lb vip + show lb vip verbose + + show node counters + + +## Design notes + +### Multi-Threading + +MagLev is a distributed system which pseudo-randomly generates a +new-connections-table based on AS names such that each server configured with +the same set of ASs ends up with the same table. Connection stickyness is then +ensured with an established-connections-table. Using ECMP, it is assumed (but +not relied on) that servers will mostly receive traffic for different flows. + +This implementation pushes the parallelism a little bit further by using +one established-connections table per thread. This is equivalent to assuming +that RSS will make a job similar to ECMP, and is pretty useful as threads don't +need to get a lock in order to write in the table. + +### Hash Table + +A load balancer requires an efficient read and write hash table. The hash table +used by ip6-forward is very read-efficient, but not so much for writing. In +addition, it is not a big deal if writing into the hash table fails (again, +MagLev uses a flow table but does not heaviliy relies on it). + +The plugin therefore uses a very specific (and stupid) hash table. + - Fixed (and power of 2) number of buckets (configured at runtime) + - Fixed (and power of 2) elements per buckets (configured at compilation time) + +### Reference counting + +When an AS is removed, there is two possible ways to react. + - Keep using the AS for established connections + - Change AS for established connections (likely to cause error for TCP) + +In the first case, although an AS is removed from the configuration, its +associated state needs to stay around as long as it is used by at least one +thread. + +In order to avoid locks, a specific reference counter is used. The design is quite +similar to clib counters but: + - It is possible to decrease the value + - Summing will not zero the per-thread counters + - Only the thread can reallocate its own counters vector (to avoid concurrency issues) + +This reference counter is lock free, but reading a count of 0 does not mean +the value can be freed unless it is ensured by *other* means that no other thread +is concurrently referencing the object. In the case of this plugin, it is assumed +that no concurrent event will take place after a few seconds. + diff --git a/src/plugins/lb/lb_test.c b/src/plugins/lb/lb_test.c new file mode 100644 index 00000000..8c2eaa91 --- /dev/null +++ b/src/plugins/lb/lb_test.c @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +//TODO: Move that to vat/plugin_api.c +////////////////////////// +uword unformat_ip46_address (unformat_input_t * input, va_list * args) +{ + ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); + ip46_type_t type = va_arg (*args, ip46_type_t); + if ((type != IP46_TYPE_IP6) && + unformat(input, "%U", unformat_ip4_address, &ip46->ip4)) { + ip46_address_mask_ip4(ip46); + return 1; + } else if ((type != IP46_TYPE_IP4) && + unformat(input, "%U", unformat_ip6_address, &ip46->ip6)) { + return 1; + } + return 0; +} +uword unformat_ip46_prefix (unformat_input_t * input, va_list * args) +{ + ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); + u8 *len = va_arg (*args, u8 *); + ip46_type_t type = va_arg (*args, ip46_type_t); + + u32 l; + if ((type != IP46_TYPE_IP6) && unformat(input, "%U/%u", unformat_ip4_address, &ip46->ip4, &l)) { + if (l > 32) + return 0; + *len = l + 96; + ip46->pad[0] = ip46->pad[1] = ip46->pad[2] = 0; + } else if ((type != IP46_TYPE_IP4) && unformat(input, "%U/%u", unformat_ip6_address, &ip46->ip6, &l)) { + if (l > 128) + return 0; + *len = l; + } else { + return 0; + } + return 1; +} +///////////////////////// + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} lb_test_main_t; + +lb_test_main_t lb_test_main; + +#define foreach_standard_reply_retval_handler \ +_(lb_conf_reply) \ +_(lb_add_del_vip_reply) \ +_(lb_add_del_as_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = lb_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ + _(LB_CONF_REPLY, lb_conf_reply) \ + _(LB_ADD_DEL_VIP_REPLY, lb_add_del_vip_reply) \ + _(LB_ADD_DEL_AS_REPLY, lb_add_del_as_reply) + +/* M: construct, but don't yet send a message */ +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memcpy (mp, &mps, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + lbtm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int api_lb_conf (vat_main_t * vam) +{ + lb_test_main_t *lbtm = &lb_test_main; + unformat_input_t *i = vam->input; + f64 timeout; + vl_api_lb_conf_t mps, *mp; + + if (!unformat(i, "%U %U %u %u", + unformat_ip4_address, &mps.ip4_src_address, + unformat_ip6_address, mps.ip6_src_address, + &mps.sticky_buckets_per_core, + &mps.flow_timeout)) { + errmsg ("invalid arguments\n"); + return -99; + } + + M(LB_CONF, lb_conf); S; W; + + /* NOTREACHED */ + return 0; +} + +static int api_lb_add_del_vip (vat_main_t * vam) +{ + lb_test_main_t *lbtm = &lb_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_lb_add_del_vip_t mps, *mp; + mps.is_del = 0; + mps.is_gre4 = 0; + + if (!unformat(i, "%U", + unformat_ip46_prefix, mps.ip_prefix, &mps.prefix_length, IP46_TYPE_ANY)) { + errmsg ("invalid prefix\n"); + return -99; + } + + if (unformat(i, "gre4")) { + mps.is_gre4 = 1; + } else if (unformat(i, "gre6")) { + mps.is_gre4 = 0; + } else { + errmsg ("no encap\n"); + return -99; + } + + if (!unformat(i, "%d", &mps.new_flows_table_length)) { + errmsg ("no table lentgh\n"); + return -99; + } + + if (unformat(i, "del")) { + mps.is_del = 1; + } + + M(LB_ADD_DEL_VIP, lb_add_del_vip); S; W; + /* NOTREACHED */ + return 0; +} + +static int api_lb_add_del_as (vat_main_t * vam) +{ + lb_test_main_t *lbtm = &lb_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_lb_add_del_as_t mps, *mp; + mps.is_del = 0; + + if (!unformat(i, "%U %U", + unformat_ip46_prefix, mps.vip_ip_prefix, &mps.vip_prefix_length, IP46_TYPE_ANY, + unformat_ip46_address, mps.as_address)) { + errmsg ("invalid prefix or address\n"); + return -99; + } + + if (unformat(i, "del")) { + mps.is_del = 1; + } + + M(LB_ADD_DEL_AS, lb_add_del_as); S; W; + /* NOTREACHED */ + return 0; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(lb_conf, " ") \ +_(lb_add_del_vip, " [gre4|gre6] [del]") \ +_(lb_add_del_as, "
[del]") + +void vat_api_hookup (vat_main_t *vam) +{ + lb_test_main_t * lbtm = &lb_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + lbtm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * vat_plugin_register (vat_main_t *vam) +{ + lb_test_main_t * lbtm = &lb_test_main; + + u8 * name; + + lbtm->vat_main = vam; + + /* Ask the vpp engine for the first assigned message-id */ + name = format (0, "lb_%08x%c", api_version, 0); + lbtm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (lbtm->msg_id_base != (u16) ~0) + vat_api_hookup (vam); + + vec_free(name); + + return 0; +} diff --git a/src/plugins/lb/lbhash.h b/src/plugins/lb/lbhash.h new file mode 100644 index 00000000..ca3cc143 --- /dev/null +++ b/src/plugins/lb/lbhash.h @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2012 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * vppinfra already includes tons of different hash tables. + * MagLev flow table is a bit different. It has to be very efficient + * for both writing and reading operations. But it does not need to + * be 100% reliable (write can fail). It also needs to recycle + * old entries in a lazy way. + * + * This hash table is the most dummy hash table you can do. + * Fixed total size, fixed bucket size. + * Advantage is that it could be very efficient (maybe). + * + */ + +#ifndef LB_PLUGIN_LB_LBHASH_H_ +#define LB_PLUGIN_LB_LBHASH_H_ + +#include + +#if defined (__SSE4_2__) +#include +#endif + +/* + * @brief Number of entries per bucket. + */ +#define LBHASH_ENTRY_PER_BUCKET 4 + +#define LB_HASH_DO_NOT_USE_SSE_BUCKETS 0 + +/* + * @brief One bucket contains 4 entries. + * Each bucket takes one 64B cache line in memory. + */ +typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u32 hash[LBHASH_ENTRY_PER_BUCKET]; + u32 timeout[LBHASH_ENTRY_PER_BUCKET]; + u32 vip[LBHASH_ENTRY_PER_BUCKET]; + u32 value[LBHASH_ENTRY_PER_BUCKET]; +} lb_hash_bucket_t; + +typedef struct { + u32 buckets_mask; + u32 timeout; + lb_hash_bucket_t buckets[]; +} lb_hash_t; + +#define lb_hash_nbuckets(h) (((h)->buckets_mask) + 1) +#define lb_hash_size(h) ((h)->buckets_mask + LBHASH_ENTRY_PER_BUCKET) + +#define lb_hash_foreach_bucket(h, bucket) \ + for (bucket = (h)->buckets; \ + bucket < (h)->buckets + lb_hash_nbuckets(h); \ + bucket++) + +#define lb_hash_foreach_entry(h, bucket, i) \ + lb_hash_foreach_bucket(h, bucket) \ + for (i = 0; i < LBHASH_ENTRY_PER_BUCKET; i++) + +#define lb_hash_foreach_valid_entry(h, bucket, i, now) \ + lb_hash_foreach_entry(h, bucket, i) \ + if (!clib_u32_loop_gt((now), bucket->timeout[i])) + +static_always_inline +lb_hash_t *lb_hash_alloc(u32 buckets, u32 timeout) +{ + if (!is_pow2(buckets)) + return NULL; + + // Allocate 1 more bucket for prefetch + u32 size = ((u64)&((lb_hash_t *)(0))->buckets[0]) + + sizeof(lb_hash_bucket_t) * (buckets + 1); + u8 *mem = 0; + lb_hash_t *h; + vec_alloc_aligned(mem, size, CLIB_CACHE_LINE_BYTES); + h = (lb_hash_t *)mem; + h->buckets_mask = (buckets - 1); + h->timeout = timeout; + return h; +} + +static_always_inline +void lb_hash_free(lb_hash_t *h) +{ + u8 *mem = (u8 *)h; + vec_free(mem); +} + +#if __SSE4_2__ +static_always_inline +u32 lb_hash_hash(u64 k0, u64 k1, u64 k2, u64 k3, u64 k4) +{ + u64 val = 0; + val = _mm_crc32_u64(val, k0); + val = _mm_crc32_u64(val, k1); + val = _mm_crc32_u64(val, k2); + val = _mm_crc32_u64(val, k3); + val = _mm_crc32_u64(val, k4); + return (u32) val; +} +#else +static_always_inline +u32 lb_hash_hash(u64 k0, u64 k1, u64 k2, u64 k3, u64 k4) +{ + u64 tmp = k0 ^ k1 ^ k2 ^ k3 ^ k4; + return (u32)clib_xxhash (tmp); +} +#endif + +static_always_inline +void lb_hash_prefetch_bucket(lb_hash_t *ht, u32 hash) +{ + lb_hash_bucket_t *bucket = &ht->buckets[hash & ht->buckets_mask]; + CLIB_PREFETCH(bucket, sizeof(*bucket), READ); +} + +static_always_inline +void lb_hash_get(lb_hash_t *ht, u32 hash, u32 vip, u32 time_now, + u32 *available_index, u32 *found_value) +{ + lb_hash_bucket_t *bucket = &ht->buckets[hash & ht->buckets_mask]; + *found_value = ~0; + *available_index = ~0; +#if __SSE4_2__ && LB_HASH_DO_NOT_USE_SSE_BUCKETS == 0 + u32 bitmask, found_index; + __m128i mask; + + // mask[*] = timeout[*] > now + mask = _mm_cmpgt_epi32(_mm_loadu_si128 ((__m128i *) bucket->timeout), + _mm_set1_epi32 (time_now)); + // bitmask[*] = now <= timeout[*/4] + bitmask = (~_mm_movemask_epi8(mask)) & 0xffff; + // Get first index with now <= timeout[*], if any. + *available_index = (bitmask)?__builtin_ctz(bitmask)/4:*available_index; + + // mask[*] = (timeout[*] > now) && (hash[*] == hash) + mask = _mm_and_si128(mask, + _mm_cmpeq_epi32( + _mm_loadu_si128 ((__m128i *) bucket->hash), + _mm_set1_epi32 (hash))); + + // Load the array of vip values + // mask[*] = (timeout[*] > now) && (hash[*] == hash) && (vip[*] == vip) + mask = _mm_and_si128(mask, + _mm_cmpeq_epi32( + _mm_loadu_si128 ((__m128i *) bucket->vip), + _mm_set1_epi32 (vip))); + + // mask[*] = (timeout[*x4] > now) && (hash[*x4] == hash) && (vip[*x4] == vip) + bitmask = _mm_movemask_epi8(mask); + // Get first index, if any + found_index = (bitmask)?__builtin_ctzll(bitmask)/4:0; + ASSERT(found_index < 4); + *found_value = (bitmask)?bucket->value[found_index]:*found_value; + bucket->timeout[found_index] = + (bitmask)?time_now + ht->timeout:bucket->timeout[found_index]; +#else + u32 i; + for (i = 0; i < LBHASH_ENTRY_PER_BUCKET; i++) { + u8 cmp = (bucket->hash[i] == hash && bucket->vip[i] == vip); + u8 timeouted = clib_u32_loop_gt(time_now, bucket->timeout[i]); + *found_value = (cmp || timeouted)?*found_value:bucket->value[i]; + bucket->timeout[i] = (cmp || timeouted)?time_now + ht->timeout:bucket->timeout[i]; + *available_index = (timeouted && (*available_index == ~0))?i:*available_index; + + if (!cmp) + return; + } +#endif +} + +static_always_inline +u32 lb_hash_available_value(lb_hash_t *h, u32 hash, u32 available_index) +{ + return h->buckets[hash & h->buckets_mask].value[available_index]; +} + +static_always_inline +void lb_hash_put(lb_hash_t *h, u32 hash, u32 value, u32 vip, + u32 available_index, u32 time_now) +{ + lb_hash_bucket_t *bucket = &h->buckets[hash & h->buckets_mask]; + bucket->hash[available_index] = hash; + bucket->value[available_index] = value; + bucket->timeout[available_index] = time_now + h->timeout; + bucket->vip[available_index] = vip; +} + +static_always_inline +u32 lb_hash_elts(lb_hash_t *h, u32 time_now) +{ + u32 tot = 0; + lb_hash_bucket_t *bucket; + u32 i; + lb_hash_foreach_valid_entry(h, bucket, i, time_now) { + tot++; + } + return tot; +} + +#endif /* LB_PLUGIN_LB_LBHASH_H_ */ diff --git a/src/plugins/lb/node.c b/src/plugins/lb/node.c new file mode 100644 index 00000000..8b763c53 --- /dev/null +++ b/src/plugins/lb/node.c @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#define foreach_lb_error \ + _(NONE, "no error") \ + _(PROTO_NOT_SUPPORTED, "protocol not supported") + +typedef enum { +#define _(sym,str) LB_ERROR_##sym, + foreach_lb_error +#undef _ + LB_N_ERROR, +} lb_error_t; + +static char *lb_error_strings[] = { +#define _(sym,string) string, + foreach_lb_error +#undef _ +}; + +typedef struct { + u32 vip_index; + u32 as_index; +} lb_trace_t; + +u8 * +format_lb_trace (u8 * s, va_list * args) +{ + lb_main_t *lbm = &lb_main; + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + lb_trace_t *t = va_arg (*args, lb_trace_t *); + if (pool_is_free_index(lbm->vips, t->vip_index)) { + s = format(s, "lb vip[%d]: This VIP was freed since capture\n"); + } else { + s = format(s, "lb vip[%d]: %U\n", t->vip_index, format_lb_vip, &lbm->vips[t->vip_index]); + } + if (pool_is_free_index(lbm->ass, t->as_index)) { + s = format(s, "lb as[%d]: This AS was freed since capture\n"); + } else { + s = format(s, "lb as[%d]: %U\n", t->as_index, format_lb_as, &lbm->ass[t->as_index]); + } + return s; +} + +lb_hash_t *lb_get_sticky_table(u32 cpu_index) +{ + lb_main_t *lbm = &lb_main; + lb_hash_t *sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; + //Check if size changed + if (PREDICT_FALSE(sticky_ht && (lbm->per_cpu_sticky_buckets != lb_hash_nbuckets(sticky_ht)))) + { + //Dereference everything in there + lb_hash_bucket_t *b; + u32 i; + lb_hash_foreach_entry(sticky_ht, b, i) { + vlib_refcount_add(&lbm->as_refcount, cpu_index, b->value[i], -1); + vlib_refcount_add(&lbm->as_refcount, cpu_index, 0, 1); + } + + lb_hash_free(sticky_ht); + sticky_ht = NULL; + } + + //Create if necessary + if (PREDICT_FALSE(sticky_ht == NULL)) { + lbm->per_cpu[cpu_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout); + sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; + clib_warning("Regenerated sticky table %p", sticky_ht); + } + + ASSERT(sticky_ht); + + //Update timeout + sticky_ht->timeout = lbm->flow_timeout; + return sticky_ht; +} + +u64 +lb_node_get_other_ports4(ip4_header_t *ip40) +{ + return 0; +} + +u64 +lb_node_get_other_ports6(ip6_header_t *ip60) +{ + return 0; +} + +static_always_inline u32 +lb_node_get_hash(vlib_buffer_t *p, u8 is_input_v4) +{ + u32 hash; + if (is_input_v4) + { + ip4_header_t *ip40; + u64 ports; + ip40 = vlib_buffer_get_current (p); + if (PREDICT_TRUE (ip40->protocol == IP_PROTOCOL_TCP || + ip40->protocol == IP_PROTOCOL_UDP)) + ports = ((u64)((udp_header_t *)(ip40 + 1))->src_port << 16) | + ((u64)((udp_header_t *)(ip40 + 1))->dst_port); + else + ports = lb_node_get_other_ports4(ip40); + + hash = lb_hash_hash(*((u64 *)&ip40->address_pair), ports, + 0, 0, 0); + } + else + { + ip6_header_t *ip60; + ip60 = vlib_buffer_get_current (p); + u64 ports; + if (PREDICT_TRUE (ip60->protocol == IP_PROTOCOL_TCP || + ip60->protocol == IP_PROTOCOL_UDP)) + ports = ((u64)((udp_header_t *)(ip60 + 1))->src_port << 16) | + ((u64)((udp_header_t *)(ip60 + 1))->dst_port); + else + ports = lb_node_get_other_ports6(ip60); + + hash = lb_hash_hash(ip60->src_address.as_u64[0], + ip60->src_address.as_u64[1], + ip60->dst_address.as_u64[0], + ip60->dst_address.as_u64[1], + ports); + } + return hash; +} + +static_always_inline uword +lb_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame, + u8 is_input_v4, //Compile-time parameter stating that is input is v4 (or v6) + u8 is_encap_v4) //Compile-time parameter stating that is GRE encap is v4 (or v6) +{ + lb_main_t *lbm = &lb_main; + u32 n_left_from, *from, next_index, *to_next, n_left_to_next; + u32 cpu_index = os_get_cpu_number(); + u32 lb_time = lb_hash_time_now(vm); + + lb_hash_t *sticky_ht = lb_get_sticky_table(cpu_index); + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + u32 nexthash0 = 0; + if (PREDICT_TRUE(n_left_from > 0)) + nexthash0 = lb_node_get_hash(vlib_get_buffer (vm, from[0]), is_input_v4); + + while (n_left_from > 0) + { + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 pi0; + vlib_buffer_t *p0; + lb_vip_t *vip0; + u32 asindex0; + u16 len0; + u32 available_index0; + u8 counter = 0; + u32 hash0 = nexthash0; + + if (PREDICT_TRUE(n_left_from > 1)) + { + vlib_buffer_t *p1 = vlib_get_buffer (vm, from[1]); + //Compute next hash and prefetch bucket + nexthash0 = lb_node_get_hash(p1, is_input_v4); + lb_hash_prefetch_bucket(sticky_ht, nexthash0); + //Prefetch for encap, next + CLIB_PREFETCH (vlib_buffer_get_current(p1) - 64, 64, STORE); + } + + if (PREDICT_TRUE(n_left_from > 2)) + { + vlib_buffer_t *p2; + p2 = vlib_get_buffer(vm, from[2]); + /* prefetch packet header and data */ + vlib_prefetch_buffer_header(p2, STORE); + CLIB_PREFETCH (vlib_buffer_get_current(p2), 64, STORE); + } + + pi0 = to_next[0] = from[0]; + from += 1; + n_left_from -= 1; + to_next += 1; + n_left_to_next -= 1; + + p0 = vlib_get_buffer (vm, pi0); + vip0 = pool_elt_at_index (lbm->vips, + vnet_buffer (p0)->ip.adj_index[VLIB_TX]); + + if (is_input_v4) + { + ip4_header_t *ip40; + ip40 = vlib_buffer_get_current (p0); + len0 = clib_net_to_host_u16(ip40->length); + } + else + { + ip6_header_t *ip60; + ip60 = vlib_buffer_get_current (p0); + len0 = clib_net_to_host_u16(ip60->payload_length) + sizeof(ip6_header_t); + } + + lb_hash_get(sticky_ht, hash0, vnet_buffer (p0)->ip.adj_index[VLIB_TX], + lb_time, &available_index0, &asindex0); + + if (PREDICT_TRUE(asindex0 != ~0)) + { + //Found an existing entry + counter = LB_VIP_COUNTER_NEXT_PACKET; + } + else if (PREDICT_TRUE(available_index0 != ~0)) + { + //There is an available slot for a new flow + asindex0 = vip0->new_flow_table[hash0 & vip0->new_flow_table_mask].as_index; + counter = LB_VIP_COUNTER_FIRST_PACKET; + counter = (asindex0 == 0)?LB_VIP_COUNTER_NO_SERVER:counter; + + //TODO: There are race conditions with as0 and vip0 manipulation. + //Configuration may be changed, vectors resized, etc... + + //Dereference previously used + vlib_refcount_add(&lbm->as_refcount, cpu_index, + lb_hash_available_value(sticky_ht, hash0, available_index0), -1); + vlib_refcount_add(&lbm->as_refcount, cpu_index, + asindex0, 1); + + //Add sticky entry + //Note that when there is no AS configured, an entry is configured anyway. + //But no configured AS is not something that should happen + lb_hash_put(sticky_ht, hash0, asindex0, + vnet_buffer (p0)->ip.adj_index[VLIB_TX], + available_index0, lb_time); + } + else + { + //Could not store new entry in the table + asindex0 = vip0->new_flow_table[hash0 & vip0->new_flow_table_mask].as_index; + counter = LB_VIP_COUNTER_UNTRACKED_PACKET; + } + + vlib_increment_simple_counter(&lbm->vip_counters[counter], + cpu_index, + vnet_buffer (p0)->ip.adj_index[VLIB_TX], + 1); + + //Now let's encap + { + gre_header_t *gre0; + if (is_encap_v4) + { + ip4_header_t *ip40; + vlib_buffer_advance(p0, - sizeof(ip4_header_t) - sizeof(gre_header_t)); + ip40 = vlib_buffer_get_current(p0); + gre0 = (gre_header_t *)(ip40 + 1); + ip40->src_address = lbm->ip4_src_address; + ip40->dst_address = lbm->ass[asindex0].address.ip4; + ip40->ip_version_and_header_length = 0x45; + ip40->ttl = 128; + ip40->length = clib_host_to_net_u16(len0 + sizeof(gre_header_t) + sizeof(ip4_header_t)); + ip40->protocol = IP_PROTOCOL_GRE; + ip40->checksum = ip4_header_checksum (ip40); + } + else + { + ip6_header_t *ip60; + vlib_buffer_advance(p0, - sizeof(ip6_header_t) - sizeof(gre_header_t)); + ip60 = vlib_buffer_get_current(p0); + gre0 = (gre_header_t *)(ip60 + 1); + ip60->dst_address = lbm->ass[asindex0].address.ip6; + ip60->src_address = lbm->ip6_src_address; + ip60->hop_limit = 128; + ip60->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6<<28); + ip60->payload_length = clib_host_to_net_u16(len0 + sizeof(gre_header_t)); + ip60->protocol = IP_PROTOCOL_GRE; + } + + gre0->flags_and_version = 0; + gre0->protocol = (is_input_v4)? + clib_host_to_net_u16(0x0800): + clib_host_to_net_u16(0x86DD); + } + + if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED)) + { + lb_trace_t *tr = vlib_add_trace (vm, node, p0, sizeof (*tr)); + tr->as_index = asindex0; + tr->vip_index = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; + } + + //Enqueue to next + //Note that this is going to error if asindex0 == 0 + vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbm->ass[asindex0].dpo.dpoi_index; + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, pi0, + lbm->ass[asindex0].dpo.dpoi_next_node); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return frame->n_vectors; +} + +static uword +lb6_gre6_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + return lb_node_fn(vm, node, frame, 0, 0); +} + +static uword +lb6_gre4_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + return lb_node_fn(vm, node, frame, 0, 1); +} + +static uword +lb4_gre6_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + return lb_node_fn(vm, node, frame, 1, 0); +} + +static uword +lb4_gre4_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + return lb_node_fn(vm, node, frame, 1, 1); +} + +VLIB_REGISTER_NODE (lb6_gre6_node) = +{ + .function = lb6_gre6_node_fn, + .name = "lb6-gre6", + .vector_size = sizeof (u32), + .format_trace = format_lb_trace, + + .n_errors = LB_N_ERROR, + .error_strings = lb_error_strings, + + .n_next_nodes = LB_N_NEXT, + .next_nodes = + { + [LB_NEXT_DROP] = "error-drop" + }, +}; + +VLIB_REGISTER_NODE (lb6_gre4_node) = +{ + .function = lb6_gre4_node_fn, + .name = "lb6-gre4", + .vector_size = sizeof (u32), + .format_trace = format_lb_trace, + + .n_errors = LB_N_ERROR, + .error_strings = lb_error_strings, + + .n_next_nodes = LB_N_NEXT, + .next_nodes = + { + [LB_NEXT_DROP] = "error-drop" + }, +}; + +VLIB_REGISTER_NODE (lb4_gre6_node) = +{ + .function = lb4_gre6_node_fn, + .name = "lb4-gre6", + .vector_size = sizeof (u32), + .format_trace = format_lb_trace, + + .n_errors = LB_N_ERROR, + .error_strings = lb_error_strings, + + .n_next_nodes = LB_N_NEXT, + .next_nodes = + { + [LB_NEXT_DROP] = "error-drop" + }, +}; + +VLIB_REGISTER_NODE (lb4_gre4_node) = +{ + .function = lb4_gre4_node_fn, + .name = "lb4-gre4", + .vector_size = sizeof (u32), + .format_trace = format_lb_trace, + + .n_errors = LB_N_ERROR, + .error_strings = lb_error_strings, + + .n_next_nodes = LB_N_NEXT, + .next_nodes = + { + [LB_NEXT_DROP] = "error-drop" + }, +}; + diff --git a/src/plugins/lb/refcount.c b/src/plugins/lb/refcount.c new file mode 100644 index 00000000..22415c88 --- /dev/null +++ b/src/plugins/lb/refcount.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size) +{ + u32 *new_counter = 0, *old_counter; + vec_validate(new_counter, size); + memcpy(new_counter, per_cpu->counters, per_cpu->length); + old_counter = per_cpu->counters; + per_cpu->counters = new_counter; + CLIB_MEMORY_BARRIER(); + per_cpu->length = vec_len(new_counter); + vec_free(old_counter); +} + +u64 vlib_refcount_get(vlib_refcount_t *r, u32 index) +{ + u64 count = 0; + vlib_thread_main_t *tm = vlib_get_thread_main (); + u32 cpu_index; + for (cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++) { + if (r->per_cpu[cpu_index].length > index) + count += r->per_cpu[cpu_index].counters[index]; + } + return count; +} + diff --git a/src/plugins/lb/refcount.h b/src/plugins/lb/refcount.h new file mode 100644 index 00000000..8c26e7be --- /dev/null +++ b/src/plugins/lb/refcount.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * vlib provides lock-free counters but those + * - Have 16bits per-CPU counter, which may overflow. + * - Would only increment. + * + * This is very similar to vlib counters, but may be used to count reference. + * Such a counter includes an arbitrary number of counters. Each counter + * is identified by its index. This is used to aggregate per-cpu memory. + * + * Warning: + * This reference counter is lock-free but is not race-condition free. + * The counting result is approximate and another mechanism needs to be used + * in order to ensure that an object may be freed. + * + */ + +#include + +typedef struct { + u32 *counters; + u32 length; + u32 *reader_lengths; + CLIB_CACHE_LINE_ALIGN_MARK(o); +} vlib_refcount_per_cpu_t; + +typedef struct { + vlib_refcount_per_cpu_t *per_cpu; +} vlib_refcount_t; + +void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size); + +static_always_inline +void vlib_refcount_add(vlib_refcount_t *r, u32 cpu_index, u32 counter_index, i32 v) +{ + vlib_refcount_per_cpu_t *per_cpu = &r->per_cpu[cpu_index]; + if (PREDICT_FALSE(counter_index >= per_cpu->length)) + __vlib_refcount_resize(per_cpu, clib_max(counter_index + 16, per_cpu->length * 2)); + + per_cpu->counters[counter_index] += v; +} + +u64 vlib_refcount_get(vlib_refcount_t *r, u32 index); + +static_always_inline +void vlib_refcount_init(vlib_refcount_t *r) +{ + vlib_thread_main_t *tm = vlib_get_thread_main (); + r->per_cpu = 0; + vec_validate (r->per_cpu, tm->n_vlib_mains - 1); +} + + diff --git a/src/plugins/lb/util.c b/src/plugins/lb/util.c new file mode 100644 index 00000000..d969d168 --- /dev/null +++ b/src/plugins/lb/util.c @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +void ip46_prefix_normalize(ip46_address_t *prefix, u8 plen) +{ + if (plen == 0) { + prefix->as_u64[0] = 0; + prefix->as_u64[1] = 0; + } else if (plen <= 64) { + prefix->as_u64[0] &= clib_host_to_net_u64(0xffffffffffffffffL << (64 - plen)); + prefix->as_u64[1] = 0; + } else { + prefix->as_u64[1] &= clib_host_to_net_u64(0xffffffffffffffffL << (128 - plen)); + } + +} + +uword unformat_ip46_prefix (unformat_input_t * input, va_list * args) +{ + ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); + u8 *len = va_arg (*args, u8 *); + ip46_type_t type = va_arg (*args, ip46_type_t); + + u32 l; + if ((type != IP46_TYPE_IP6) && unformat(input, "%U/%u", unformat_ip4_address, &ip46->ip4, &l)) { + if (l > 32) + return 0; + *len = l + 96; + ip46->pad[0] = ip46->pad[1] = ip46->pad[2] = 0; + } else if ((type != IP46_TYPE_IP4) && unformat(input, "%U/%u", unformat_ip6_address, &ip46->ip6, &l)) { + if (l > 128) + return 0; + *len = l; + } else { + return 0; + } + return 1; +} + +u8 *format_ip46_prefix (u8 * s, va_list * args) +{ + ip46_address_t *ip46 = va_arg (*args, ip46_address_t *); + u32 len = va_arg (*args, u32); //va_arg cannot use u8 or u16 + ip46_type_t type = va_arg (*args, ip46_type_t); + + int is_ip4 = 0; + if (type == IP46_TYPE_IP4) + is_ip4 = 1; + else if (type == IP46_TYPE_IP6) + is_ip4 = 0; + else + is_ip4 = (len >= 96) && ip46_address_is_ip4(ip46); + + return is_ip4 ? + format(s, "%U/%d", format_ip4_address, &ip46->ip4, len - 96): + format(s, "%U/%d", format_ip6_address, &ip46->ip6, len); +} + diff --git a/src/plugins/lb/util.h b/src/plugins/lb/util.h new file mode 100644 index 00000000..3f082310 --- /dev/null +++ b/src/plugins/lb/util.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Non-LB specific stuff comes here + */ + +#ifndef LB_PLUGIN_LB_UTIL_H_ +#define LB_PLUGIN_LB_UTIL_H_ + +#include +#include + +#define ip46_address_type(ip46) (ip46_address_is_ip4(ip46)?IP46_TYPE_IP4:IP46_TYPE_IP6) +#define ip46_prefix_is_ip4(ip46, len) ((len) >= 96 && ip46_address_is_ip4(ip46)) +#define ip46_prefix_type(ip46, len) (ip46_prefix_is_ip4(ip46, len)?IP46_TYPE_IP4:IP46_TYPE_IP6) + +void ip46_prefix_normalize(ip46_address_t *prefix, u8 plen); +uword unformat_ip46_prefix (unformat_input_t * input, va_list * args); +u8 *format_ip46_prefix (u8 * s, va_list * args); + +/** + * 32 bits integer comparison for running values. + * 1 > 0 is true. But 1 > 0xffffffff also is. + */ +#define clib_u32_loop_gt(a, b) (((u32)(a)) - ((u32)(b)) < 0x7fffffff) + +#endif /* LB_PLUGIN_LB_UTIL_H_ */ diff --git a/src/plugins/snat.am b/src/plugins/snat.am new file mode 100644 index 00000000..7ff2386e --- /dev/null +++ b/src/plugins/snat.am @@ -0,0 +1,33 @@ + +# Copyright (c) +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +vppapitestplugins_LTLIBRARIES += snat_test_plugin.la +vppplugins_LTLIBRARIES += snat_plugin.la + +snat_plugin_la_SOURCES = snat/snat.c \ + snat/in2out.c \ + snat/out2in.c \ + snat/snat_plugin.api.h + +API_FILES += snat/snat.api + +nobase_apiinclude_HEADERS += \ + snat/snat_all_api_h.h \ + snat/snat_msg_enum.h \ + snat/snat.api.h + +snat_test_plugin_la_SOURCES = \ + snat/snat_test.c snat/snat_plugin.api.h + +# vi:syntax=automake diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c new file mode 100644 index 00000000..c78fdd76 --- /dev/null +++ b/src/plugins/snat/in2out.c @@ -0,0 +1,1597 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +typedef struct { + u32 sw_if_index; + u32 next_index; + u32 session_index; + u32 is_slow_path; +} snat_in2out_trace_t; + +typedef struct { + u32 next_worker_index; + u8 do_handoff; +} snat_in2out_worker_handoff_trace_t; + +/* packet trace format function */ +static u8 * format_snat_in2out_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *); + char * tag; + + tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH"; + + s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag, + t->sw_if_index, t->next_index, t->session_index); + + return s; +} + +static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *); + + s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d", + t->sw_if_index, t->next_index); + + return s; +} + +static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + snat_in2out_worker_handoff_trace_t * t = + va_arg (*args, snat_in2out_worker_handoff_trace_t *); + char * m; + + m = t->do_handoff ? "next worker" : "same worker"; + s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index); + + return s; +} + +vlib_node_registration_t snat_in2out_node; +vlib_node_registration_t snat_in2out_slowpath_node; +vlib_node_registration_t snat_in2out_fast_node; +vlib_node_registration_t snat_in2out_worker_handoff_node; + +#define foreach_snat_in2out_error \ +_(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \ +_(IN2OUT_PACKETS, "Good in2out packets processed") \ +_(OUT_OF_PORTS, "Out of ports") \ +_(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \ +_(BAD_ICMP_TYPE, "icmp type not echo-request") \ +_(NO_TRANSLATION, "No translation") + +typedef enum { +#define _(sym,str) SNAT_IN2OUT_ERROR_##sym, + foreach_snat_in2out_error +#undef _ + SNAT_IN2OUT_N_ERROR, +} snat_in2out_error_t; + +static char * snat_in2out_error_strings[] = { +#define _(sym,string) string, + foreach_snat_in2out_error +#undef _ +}; + +typedef enum { + SNAT_IN2OUT_NEXT_LOOKUP, + SNAT_IN2OUT_NEXT_DROP, + SNAT_IN2OUT_NEXT_SLOW_PATH, + SNAT_IN2OUT_N_NEXT, +} snat_in2out_next_t; + +static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, + ip4_header_t * ip0, + u32 rx_fib_index0, + snat_session_key_t * key0, + snat_session_t ** sessionp, + vlib_node_runtime_t * node, + u32 next0, + u32 cpu_index) +{ + snat_user_t *u; + snat_user_key_t user_key; + snat_session_t *s; + clib_bihash_kv_8_8_t kv0, value0; + u32 oldest_per_user_translation_list_index; + dlist_elt_t * oldest_per_user_translation_list_elt; + dlist_elt_t * per_user_translation_list_elt; + dlist_elt_t * per_user_list_head_elt; + u32 session_index; + snat_session_key_t key1; + u32 address_index = ~0; + u32 outside_fib_index; + uword * p; + snat_static_mapping_key_t worker_by_out_key; + + p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id); + if (! p) + { + b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB]; + return SNAT_IN2OUT_NEXT_DROP; + } + outside_fib_index = p[0]; + + user_key.addr = ip0->src_address; + user_key.fib_index = rx_fib_index0; + kv0.key = user_key.as_u64; + + /* Ever heard of the "user" = src ip4 address before? */ + if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) + { + /* no, make a new one */ + pool_get (sm->per_thread_data[cpu_index].users, u); + memset (u, 0, sizeof (*u)); + u->addr = ip0->src_address; + + pool_get (sm->per_thread_data[cpu_index].list_pool, per_user_list_head_elt); + + u->sessions_per_user_list_head_index = per_user_list_head_elt - + sm->per_thread_data[cpu_index].list_pool; + + clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + u->sessions_per_user_list_head_index); + + kv0.value = u - sm->per_thread_data[cpu_index].users; + + /* add user */ + clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); + } + else + { + u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, + value0.value); + } + + /* Over quota? Recycle the least recently used dynamic translation */ + if (u->nsessions >= sm->max_translations_per_user) + { + /* Remove the oldest dynamic translation */ + do { + oldest_per_user_translation_list_index = + clib_dlist_remove_head (sm->per_thread_data[cpu_index].list_pool, + u->sessions_per_user_list_head_index); + + ASSERT (oldest_per_user_translation_list_index != ~0); + + /* add it back to the end of the LRU list */ + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + u->sessions_per_user_list_head_index, + oldest_per_user_translation_list_index); + /* Get the list element */ + oldest_per_user_translation_list_elt = + pool_elt_at_index (sm->per_thread_data[cpu_index].list_pool, + oldest_per_user_translation_list_index); + + /* Get the session index from the list element */ + session_index = oldest_per_user_translation_list_elt->value; + + /* Get the session */ + s = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + session_index); + } while (snat_is_session_static (s)); + + /* Remove in2out, out2in keys */ + kv0.key = s->in2out.as_u64; + if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */)) + clib_warning ("in2out key delete failed"); + kv0.key = s->out2in.as_u64; + if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */)) + clib_warning ("out2in key delete failed"); + + snat_free_outside_address_and_port + (sm, &s->out2in, s->outside_address_index); + s->outside_address_index = ~0; + + if (snat_alloc_outside_address_and_port (sm, &key1, &address_index)) + { + ASSERT(0); + + b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS]; + return SNAT_IN2OUT_NEXT_DROP; + } + s->outside_address_index = address_index; + } + else + { + u8 static_mapping = 1; + + /* First try to match static mapping by local address and port */ + if (snat_static_mapping_match (sm, *key0, &key1, 0)) + { + static_mapping = 0; + /* Try to create dynamic translation */ + if (snat_alloc_outside_address_and_port (sm, &key1, &address_index)) + { + b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS]; + return SNAT_IN2OUT_NEXT_DROP; + } + } + + /* Create a new session */ + pool_get (sm->per_thread_data[cpu_index].sessions, s); + memset (s, 0, sizeof (*s)); + + s->outside_address_index = address_index; + + if (static_mapping) + { + u->nstaticsessions++; + s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING; + } + else + { + u->nsessions++; + } + + /* Create list elts */ + pool_get (sm->per_thread_data[cpu_index].list_pool, + per_user_translation_list_elt); + clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + per_user_translation_list_elt - + sm->per_thread_data[cpu_index].list_pool); + + per_user_translation_list_elt->value = + s - sm->per_thread_data[cpu_index].sessions; + s->per_user_index = per_user_translation_list_elt - + sm->per_thread_data[cpu_index].list_pool; + s->per_user_list_head_index = u->sessions_per_user_list_head_index; + + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s->per_user_list_head_index, + per_user_translation_list_elt - + sm->per_thread_data[cpu_index].list_pool); + } + + s->in2out = *key0; + s->out2in = key1; + s->out2in.protocol = key0->protocol; + s->out2in.fib_index = outside_fib_index; + *sessionp = s; + + /* Add to translation hashes */ + kv0.key = s->in2out.as_u64; + kv0.value = s - sm->per_thread_data[cpu_index].sessions; + if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) + clib_warning ("in2out key add failed"); + + kv0.key = s->out2in.as_u64; + kv0.value = s - sm->per_thread_data[cpu_index].sessions; + + if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) + clib_warning ("out2in key add failed"); + + /* Add to translated packets worker lookup */ + worker_by_out_key.addr = s->out2in.addr; + worker_by_out_key.port = s->out2in.port; + worker_by_out_key.fib_index = s->out2in.fib_index; + kv0.key = worker_by_out_key.as_u64; + kv0.value = cpu_index; + clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1); + return next0; +} + +static inline u32 icmp_in2out_slow_path (snat_main_t *sm, + vlib_buffer_t * b0, + ip4_header_t * ip0, + icmp46_header_t * icmp0, + u32 sw_if_index0, + u32 rx_fib_index0, + vlib_node_runtime_t * node, + u32 next0, + f64 now, + u32 cpu_index) +{ + snat_session_key_t key0; + icmp_echo_header_t *echo0; + clib_bihash_kv_8_8_t kv0, value0; + snat_session_t * s0; + u32 new_addr0, old_addr0; + u16 old_id0, new_id0; + ip_csum_t sum0; + snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; + + if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request)) + { + b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE]; + return SNAT_IN2OUT_NEXT_DROP; + } + + echo0 = (icmp_echo_header_t *)(icmp0+1); + + key0.addr = ip0->src_address; + key0.port = echo0->identifier; + key0.protocol = SNAT_PROTOCOL_ICMP; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0)) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + return next0; + + next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, + &s0, node, next0, cpu_index); + + if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) + return next0; + } + else + s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value0.value); + + old_addr0 = ip0->src_address.as_u32; + ip0->src_address = s0->out2in.addr; + new_addr0 = ip0->src_address.as_u32; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + src_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + old_id0 = echo0->identifier; + new_id0 = s0->out2in.port; + echo0->identifier = new_id0; + + sum0 = icmp0->checksum; + sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, + identifier); + icmp0->checksum = ip_csum_fold (sum0); + + /* Accounting */ + s0->last_heard = now; + s0->total_pkts++; + s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0); + /* Per-user LRU list maintenance for dynamic translations */ + if (!snat_is_session_static (s0)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_list_head_index, + s0->per_user_index); + } + + return next0; +} + +/** + * @brief Hairpinning + * + * Hairpinning allows two endpoints on the internal side of the NAT to + * communicate even if they only use each other's external IP addresses + * and ports. + * + * @param sm SNAT main. + * @param b0 Vlib buffer. + * @param ip0 IP header. + * @param udp0 UDP header. + * @param tcp0 TCP header. + * @param proto0 SNAT protocol. + */ +static inline void +snat_hairpinning (snat_main_t *sm, + vlib_buffer_t * b0, + ip4_header_t * ip0, + udp_header_t * udp0, + tcp_header_t * tcp0, + u32 proto0) +{ + snat_session_key_t key0, sm0; + snat_static_mapping_key_t k0; + snat_session_t * s0; + clib_bihash_kv_8_8_t kv0, value0; + ip_csum_t sum0; + u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si; + u16 new_dst_port0, old_dst_port0; + + key0.addr = ip0->dst_address; + key0.port = udp0->dst_port; + key0.protocol = proto0; + key0.fib_index = sm->outside_fib_index; + kv0.key = key0.as_u64; + + /* Check if destination is in active sessions */ + if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) + { + /* or static mappings */ + if (!snat_static_mapping_match(sm, key0, &sm0, 1)) + { + new_dst_addr0 = sm0.addr.as_u32; + new_dst_port0 = sm0.port; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; + } + } + else + { + si = value0.value; + if (sm->num_workers > 1) + { + k0.addr = ip0->dst_address; + k0.port = udp0->dst_port; + k0.fib_index = sm->outside_fib_index; + kv0.key = k0.as_u64; + if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0)) + ASSERT(0); + else + ti = value0.value; + } + else + ti = sm->num_workers; + + s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si); + new_dst_addr0 = s0->in2out.addr.as_u32; + new_dst_port0 = s0->in2out.port; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; + } + + /* Destination is behind the same NAT, use internal address and port */ + if (new_dst_addr0) + { + old_dst_addr0 = ip0->dst_address.as_u32; + ip0->dst_address.as_u32 = new_dst_addr0; + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, + ip4_header_t, dst_address); + ip0->checksum = ip_csum_fold (sum0); + + old_dst_port0 = tcp0->ports.dst; + if (PREDICT_TRUE(new_dst_port0 != old_dst_port0)) + { + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + tcp0->ports.dst = new_dst_port0; + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, + ip4_header_t, dst_address); + sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0, + ip4_header_t /* cheat */, length); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + udp0->dst_port = new_dst_port0; + udp0->checksum = 0; + } + } + } +} + +static inline uword +snat_in2out_node_fn_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame, int is_slow_path) +{ + u32 n_left_from, * from, * to_next; + snat_in2out_next_t next_index; + u32 pkts_processed = 0; + snat_main_t * sm = &snat_main; + snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; + f64 now = vlib_time_now (vm); + u32 stats_node_index; + u32 cpu_index = os_get_cpu_number (); + + stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index : + snat_in2out_node.index; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t * b0, * b1; + u32 next0, next1; + u32 sw_if_index0, sw_if_index1; + ip4_header_t * ip0, * ip1; + ip_csum_t sum0, sum1; + u32 new_addr0, old_addr0, new_addr1, old_addr1; + u16 old_port0, new_port0, old_port1, new_port1; + udp_header_t * udp0, * udp1; + tcp_header_t * tcp0, * tcp1; + icmp46_header_t * icmp0, * icmp1; + snat_session_key_t key0, key1; + u32 rx_fib_index0, rx_fib_index1; + u32 proto0, proto1; + snat_session_t * s0 = 0, * s1 = 0; + clib_bihash_kv_8_8_t kv0, value0, kv1, value1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * p2, * p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + /* speculatively enqueue b0 and b1 to the current next frame */ + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + tcp0 = (tcp_header_t *) udp0; + icmp0 = (icmp46_header_t *) udp0; + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, + sw_if_index0); + + next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP; + + proto0 = ~0; + proto0 = (ip0->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto0; + + /* Next configured feature, probably ip4-lookup */ + if (is_slow_path) + { + if (PREDICT_FALSE (proto0 == ~0)) + goto trace00; + + if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = icmp_in2out_slow_path + (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, + node, next0, now, cpu_index); + goto trace00; + } + } + else + { + if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; + goto trace00; + } + } + + key0.addr = ip0->src_address; + key0.port = udp0->src_port; + key0.protocol = proto0; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0)) + { + if (is_slow_path) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + goto trace00; + + next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, + &s0, node, next0, cpu_index); + if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) + goto trace00; + } + else + { + next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; + goto trace00; + } + } + else + s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value0.value); + + old_addr0 = ip0->src_address.as_u32; + ip0->src_address = s0->out2in.addr; + new_addr0 = ip0->src_address.as_u32; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + src_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + old_port0 = tcp0->ports.src; + tcp0->ports.src = s0->out2in.port; + new_port0 = tcp0->ports.src; + + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + sum0 = ip_csum_update (sum0, old_port0, new_port0, + ip4_header_t /* cheat */, + length /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + old_port0 = udp0->src_port; + udp0->src_port = s0->out2in.port; + udp0->checksum = 0; + } + + /* Hairpinning */ + snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0); + + /* Accounting */ + s0->last_heard = now; + s0->total_pkts++; + s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s0)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_list_head_index, + s0->per_user_index); + } + trace00: + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_in2out_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->is_slow_path = is_slow_path; + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->session_index = ~0; + if (s0) + t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + } + + pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; + + ip1 = vlib_buffer_get_current (b1); + udp1 = ip4_next_header (ip1); + tcp1 = (tcp_header_t *) udp1; + icmp1 = (icmp46_header_t *) udp1; + + sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX]; + rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, + sw_if_index1); + + proto1 = ~0; + proto1 = (ip1->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto1; + proto1 = (ip1->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto1; + proto1 = (ip1->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto1; + + /* Next configured feature, probably ip4-lookup */ + if (is_slow_path) + { + if (PREDICT_FALSE (proto1 == ~0)) + goto trace01; + + if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP)) + { + next1 = icmp_in2out_slow_path + (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, + next1, now, cpu_index); + goto trace01; + } + } + else + { + if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP)) + { + next1 = SNAT_IN2OUT_NEXT_SLOW_PATH; + goto trace01; + } + } + + key1.addr = ip1->src_address; + key1.port = udp1->src_port; + key1.protocol = proto1; + key1.fib_index = rx_fib_index1; + + kv1.key = key1.as_u64; + + if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0)) + { + if (is_slow_path) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index1)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index1, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index1; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip1->dst_address.as_u32 == + rt->cached_ip4_address)) + goto trace01; + + next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1, + &s1, node, next1, cpu_index); + if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP)) + goto trace01; + } + else + { + next1 = SNAT_IN2OUT_NEXT_SLOW_PATH; + goto trace01; + } + } + else + s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value1.value); + + old_addr1 = ip1->src_address.as_u32; + ip1->src_address = s1->out2in.addr; + new_addr1 = ip1->src_address.as_u32; + vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index; + + sum1 = ip1->checksum; + sum1 = ip_csum_update (sum1, old_addr1, new_addr1, + ip4_header_t, + src_address /* changed member */); + ip1->checksum = ip_csum_fold (sum1); + + if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP)) + { + old_port1 = tcp1->ports.src; + tcp1->ports.src = s1->out2in.port; + new_port1 = tcp1->ports.src; + + sum1 = tcp1->checksum; + sum1 = ip_csum_update (sum1, old_addr1, new_addr1, + ip4_header_t, + dst_address /* changed member */); + sum1 = ip_csum_update (sum1, old_port1, new_port1, + ip4_header_t /* cheat */, + length /* changed member */); + tcp1->checksum = ip_csum_fold(sum1); + } + else + { + old_port1 = udp1->src_port; + udp1->src_port = s1->out2in.port; + udp1->checksum = 0; + } + + /* Hairpinning */ + snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1); + + /* Accounting */ + s1->last_heard = now; + s1->total_pkts++; + s1->total_bytes += vlib_buffer_length_in_chain (vm, b1); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s1)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s1->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s1->per_user_list_head_index, + s1->per_user_index); + } + trace01: + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b1->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_in2out_trace_t *t = + vlib_add_trace (vm, node, b1, sizeof (*t)); + t->sw_if_index = sw_if_index1; + t->next_index = next1; + t->session_index = ~0; + if (s1) + t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; + } + + pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP; + + /* verify speculative enqueues, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, + to_next, n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t * b0; + u32 next0; + u32 sw_if_index0; + ip4_header_t * ip0; + ip_csum_t sum0; + u32 new_addr0, old_addr0; + u16 old_port0, new_port0; + udp_header_t * udp0; + tcp_header_t * tcp0; + icmp46_header_t * icmp0; + snat_session_key_t key0; + u32 rx_fib_index0; + u32 proto0; + snat_session_t * s0 = 0; + clib_bihash_kv_8_8_t kv0, value0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + next0 = SNAT_IN2OUT_NEXT_LOOKUP; + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + tcp0 = (tcp_header_t *) udp0; + icmp0 = (icmp46_header_t *) udp0; + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, + sw_if_index0); + + proto0 = ~0; + proto0 = (ip0->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto0; + + /* Next configured feature, probably ip4-lookup */ + if (is_slow_path) + { + if (PREDICT_FALSE (proto0 == ~0)) + goto trace0; + + if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = icmp_in2out_slow_path + (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, + next0, now, cpu_index); + goto trace0; + } + } + else + { + if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; + goto trace0; + } + } + + key0.addr = ip0->src_address; + key0.port = udp0->src_port; + key0.protocol = proto0; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0)) + { + if (is_slow_path) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + goto trace0; + + next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, + &s0, node, next0, cpu_index); + if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) + goto trace0; + } + else + { + next0 = SNAT_IN2OUT_NEXT_SLOW_PATH; + goto trace0; + } + } + else + s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value0.value); + + old_addr0 = ip0->src_address.as_u32; + ip0->src_address = s0->out2in.addr; + new_addr0 = ip0->src_address.as_u32; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + src_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + old_port0 = tcp0->ports.src; + tcp0->ports.src = s0->out2in.port; + new_port0 = tcp0->ports.src; + + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + sum0 = ip_csum_update (sum0, old_port0, new_port0, + ip4_header_t /* cheat */, + length /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + old_port0 = udp0->src_port; + udp0->src_port = s0->out2in.port; + udp0->checksum = 0; + } + + /* Hairpinning */ + snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0); + + /* Accounting */ + s0->last_heard = now; + s0->total_pkts++; + s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s0)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_list_head_index, + s0->per_user_index); + } + + trace0: + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_in2out_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->is_slow_path = is_slow_path; + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->session_index = ~0; + if (s0) + t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + } + + pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, stats_node_index, + SNAT_IN2OUT_ERROR_IN2OUT_PACKETS, + pkts_processed); + return frame->n_vectors; +} + +static uword +snat_in2out_fast_path_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */); +} + +VLIB_REGISTER_NODE (snat_in2out_node) = { + .function = snat_in2out_fast_path_fn, + .name = "snat-in2out", + .vector_size = sizeof (u32), + .format_trace = format_snat_in2out_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(snat_in2out_error_strings), + .error_strings = snat_in2out_error_strings, + + .runtime_data_bytes = sizeof (snat_runtime_t), + + .n_next_nodes = SNAT_IN2OUT_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [SNAT_IN2OUT_NEXT_DROP] = "error-drop", + [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup", + [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn); + +static uword +snat_in2out_slow_path_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */); +} + +VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = { + .function = snat_in2out_slow_path_fn, + .name = "snat-in2out-slowpath", + .vector_size = sizeof (u32), + .format_trace = format_snat_in2out_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(snat_in2out_error_strings), + .error_strings = snat_in2out_error_strings, + + .runtime_data_bytes = sizeof (snat_runtime_t), + + .n_next_nodes = SNAT_IN2OUT_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [SNAT_IN2OUT_NEXT_DROP] = "error-drop", + [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup", + [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node, snat_in2out_slow_path_fn); + +static uword +snat_in2out_worker_handoff_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + snat_main_t *sm = &snat_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + u32 n_left_from, *from, *to_next = 0; + static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index; + static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index + = 0; + vlib_frame_queue_elt_t *hf = 0; + vlib_frame_t *f = 0; + int i; + u32 n_left_to_next_worker = 0, *to_next_worker = 0; + u32 next_worker_index = 0; + u32 current_worker_index = ~0; + u32 cpu_index = os_get_cpu_number (); + + ASSERT (vec_len (sm->workers)); + + if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0)) + { + vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1); + + vec_validate_init_empty (congested_handoff_queue_by_worker_index, + sm->first_worker_index + sm->num_workers - 1, + (vlib_frame_queue_t *) (~0)); + } + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + while (n_left_from > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 sw_if_index0; + u32 rx_fib_index0; + ip4_header_t * ip0; + snat_user_key_t key0; + clib_bihash_kv_8_8_t kv0, value0; + u8 do_handoff; + + bi0 = from[0]; + from += 1; + n_left_from -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); + + ip0 = vlib_buffer_get_current (b0); + + key0.addr = ip0->src_address; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + /* Ever heard of of the "user" before? */ + if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv0, &value0)) + { + /* No, assign next available worker (RR) */ + next_worker_index = sm->first_worker_index + + sm->workers[sm->next_worker++ % vec_len (sm->workers)]; + + /* add non-traslated packets worker lookup */ + kv0.value = next_worker_index; + clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1); + } + else + next_worker_index = value0.value; + + if (PREDICT_FALSE (next_worker_index != cpu_index)) + { + do_handoff = 1; + + if (next_worker_index != current_worker_index) + { + if (hf) + hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; + + hf = vlib_get_worker_handoff_queue_elt (sm->fq_in2out_index, + next_worker_index, + handoff_queue_elt_by_worker_index); + + n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors; + to_next_worker = &hf->buffer_index[hf->n_vectors]; + current_worker_index = next_worker_index; + } + + /* enqueue to correct worker thread */ + to_next_worker[0] = bi0; + to_next_worker++; + n_left_to_next_worker--; + + if (n_left_to_next_worker == 0) + { + hf->n_vectors = VLIB_FRAME_SIZE; + vlib_put_frame_queue_elt (hf); + current_worker_index = ~0; + handoff_queue_elt_by_worker_index[next_worker_index] = 0; + hf = 0; + } + } + else + { + do_handoff = 0; + /* if this is 1st frame */ + if (!f) + { + f = vlib_get_frame_to_node (vm, snat_in2out_node.index); + to_next = vlib_frame_vector_args (f); + } + + to_next[0] = bi0; + to_next += 1; + f->n_vectors++; + } + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_in2out_worker_handoff_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_worker_index = next_worker_index; + t->do_handoff = do_handoff; + } + } + + if (f) + vlib_put_frame_to_node (vm, snat_in2out_node.index, f); + + if (hf) + hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; + + /* Ship frames to the worker nodes */ + for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++) + { + if (handoff_queue_elt_by_worker_index[i]) + { + hf = handoff_queue_elt_by_worker_index[i]; + /* + * It works better to let the handoff node + * rate-adapt, always ship the handoff queue element. + */ + if (1 || hf->n_vectors == hf->last_n_vectors) + { + vlib_put_frame_queue_elt (hf); + handoff_queue_elt_by_worker_index[i] = 0; + } + else + hf->last_n_vectors = hf->n_vectors; + } + congested_handoff_queue_by_worker_index[i] = + (vlib_frame_queue_t *) (~0); + } + hf = 0; + current_worker_index = ~0; + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = { + .function = snat_in2out_worker_handoff_fn, + .name = "snat-in2out-worker-handoff", + .vector_size = sizeof (u32), + .format_trace = format_snat_in2out_worker_handoff_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_next_nodes = 1, + + .next_nodes = { + [0] = "error-drop", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node, snat_in2out_worker_handoff_fn); + +static inline u32 icmp_in2out_static_map (snat_main_t *sm, + vlib_buffer_t * b0, + ip4_header_t * ip0, + icmp46_header_t * icmp0, + u32 sw_if_index0, + vlib_node_runtime_t * node, + u32 next0, + u32 rx_fib_index0) +{ + snat_session_key_t key0, sm0; + icmp_echo_header_t *echo0; + u32 new_addr0, old_addr0; + u16 old_id0, new_id0; + ip_csum_t sum0; + snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; + + echo0 = (icmp_echo_header_t *)(icmp0+1); + + key0.addr = ip0->src_address; + key0.port = echo0->identifier; + key0.fib_index = rx_fib_index0; + + if (snat_static_mapping_match(sm, key0, &sm0, 0)) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + return next0; + + b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION]; + return SNAT_IN2OUT_NEXT_DROP; + } + + new_addr0 = sm0.addr.as_u32; + new_id0 = sm0.port; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; + old_addr0 = ip0->src_address.as_u32; + ip0->src_address.as_u32 = new_addr0; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + src_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_FALSE(new_id0 != echo0->identifier)) + { + old_id0 = echo0->identifier; + echo0->identifier = new_id0; + + sum0 = icmp0->checksum; + sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, + identifier); + icmp0->checksum = ip_csum_fold (sum0); + } + + return next0; +} + +static uword +snat_in2out_fast_static_map_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + u32 n_left_from, * from, * to_next; + snat_in2out_next_t next_index; + u32 pkts_processed = 0; + snat_main_t * sm = &snat_main; + snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; + u32 stats_node_index; + + stats_node_index = snat_in2out_fast_node.index; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t * b0; + u32 next0; + u32 sw_if_index0; + ip4_header_t * ip0; + ip_csum_t sum0; + u32 new_addr0, old_addr0; + u16 old_port0, new_port0; + udp_header_t * udp0; + tcp_header_t * tcp0; + icmp46_header_t * icmp0; + snat_session_key_t key0, sm0; + u32 proto0; + u32 rx_fib_index0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + next0 = SNAT_IN2OUT_NEXT_LOOKUP; + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + tcp0 = (tcp_header_t *) udp0; + icmp0 = (icmp46_header_t *) udp0; + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); + + proto0 = ~0; + proto0 = (ip0->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto0; + + if (PREDICT_FALSE (proto0 == ~0)) + goto trace0; + + if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + goto trace0; + + next0 = icmp_in2out_static_map + (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0); + goto trace0; + } + + key0.addr = ip0->src_address; + key0.port = udp0->src_port; + key0.fib_index = rx_fib_index0; + + if (snat_static_mapping_match(sm, key0, &sm0, 0)) + { + b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION]; + next0= SNAT_IN2OUT_NEXT_DROP; + goto trace0; + } + + new_addr0 = sm0.addr.as_u32; + new_port0 = sm0.port; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; + old_addr0 = ip0->src_address.as_u32; + ip0->src_address.as_u32 = new_addr0; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + src_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_FALSE(new_port0 != udp0->dst_port)) + { + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + old_port0 = tcp0->ports.src; + tcp0->ports.src = new_port0; + + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + sum0 = ip_csum_update (sum0, old_port0, new_port0, + ip4_header_t /* cheat */, + length /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + old_port0 = udp0->src_port; + udp0->src_port = new_port0; + udp0->checksum = 0; + } + } + else + { + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + } + + /* Hairpinning */ + snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0); + + trace0: + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_in2out_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + } + + pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, stats_node_index, + SNAT_IN2OUT_ERROR_IN2OUT_PACKETS, + pkts_processed); + return frame->n_vectors; +} + + +VLIB_REGISTER_NODE (snat_in2out_fast_node) = { + .function = snat_in2out_fast_static_map_fn, + .name = "snat-in2out-fast", + .vector_size = sizeof (u32), + .format_trace = format_snat_in2out_fast_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(snat_in2out_error_strings), + .error_strings = snat_in2out_error_strings, + + .runtime_data_bytes = sizeof (snat_runtime_t), + + .n_next_nodes = SNAT_IN2OUT_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [SNAT_IN2OUT_NEXT_DROP] = "error-drop", + [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup", + [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn); diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c new file mode 100644 index 00000000..f1f4159c --- /dev/null +++ b/src/plugins/snat/out2in.c @@ -0,0 +1,1261 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +typedef struct { + u32 sw_if_index; + u32 next_index; + u32 session_index; +} snat_out2in_trace_t; + +typedef struct { + u32 next_worker_index; + u8 do_handoff; +} snat_out2in_worker_handoff_trace_t; + +/* packet trace format function */ +static u8 * format_snat_out2in_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *); + + s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d", + t->sw_if_index, t->next_index, t->session_index); + return s; +} + +static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *); + + s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d", + t->sw_if_index, t->next_index); + return s; +} + +static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + snat_out2in_worker_handoff_trace_t * t = + va_arg (*args, snat_out2in_worker_handoff_trace_t *); + char * m; + + m = t->do_handoff ? "next worker" : "same worker"; + s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index); + + return s; +} + +vlib_node_registration_t snat_out2in_node; +vlib_node_registration_t snat_out2in_fast_node; +vlib_node_registration_t snat_out2in_worker_handoff_node; + +#define foreach_snat_out2in_error \ +_(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \ +_(OUT2IN_PACKETS, "Good out2in packets processed") \ +_(BAD_ICMP_TYPE, "icmp type not echo-reply") \ +_(NO_TRANSLATION, "No translation") + +typedef enum { +#define _(sym,str) SNAT_OUT2IN_ERROR_##sym, + foreach_snat_out2in_error +#undef _ + SNAT_OUT2IN_N_ERROR, +} snat_out2in_error_t; + +static char * snat_out2in_error_strings[] = { +#define _(sym,string) string, + foreach_snat_out2in_error +#undef _ +}; + +typedef enum { + SNAT_OUT2IN_NEXT_DROP, + SNAT_OUT2IN_NEXT_LOOKUP, + SNAT_OUT2IN_N_NEXT, +} snat_out2in_next_t; + +/** + * @brief Create session for static mapping. + * + * Create NAT session initiated by host from external network with static + * mapping. + * + * @param sm SNAT main. + * @param b0 Vlib buffer. + * @param in2out In2out SNAT session key. + * @param out2in Out2in SNAT session key. + * @param node Vlib node. + * + * @returns SNAT session if successfully created otherwise 0. + */ +static inline snat_session_t * +create_session_for_static_mapping (snat_main_t *sm, + vlib_buffer_t *b0, + snat_session_key_t in2out, + snat_session_key_t out2in, + vlib_node_runtime_t * node, + u32 cpu_index) +{ + snat_user_t *u; + snat_user_key_t user_key; + snat_session_t *s; + clib_bihash_kv_8_8_t kv0, value0; + dlist_elt_t * per_user_translation_list_elt; + dlist_elt_t * per_user_list_head_elt; + + user_key.addr = in2out.addr; + user_key.fib_index = in2out.fib_index; + kv0.key = user_key.as_u64; + + /* Ever heard of the "user" = inside ip4 address before? */ + if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) + { + /* no, make a new one */ + pool_get (sm->per_thread_data[cpu_index].users, u); + memset (u, 0, sizeof (*u)); + u->addr = in2out.addr; + + pool_get (sm->per_thread_data[cpu_index].list_pool, + per_user_list_head_elt); + + u->sessions_per_user_list_head_index = per_user_list_head_elt - + sm->per_thread_data[cpu_index].list_pool; + + clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + u->sessions_per_user_list_head_index); + + kv0.value = u - sm->per_thread_data[cpu_index].users; + + /* add user */ + clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); + + /* add non-traslated packets worker lookup */ + kv0.value = cpu_index; + clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1); + } + else + { + u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, + value0.value); + } + + pool_get (sm->per_thread_data[cpu_index].sessions, s); + memset (s, 0, sizeof (*s)); + + s->outside_address_index = ~0; + s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING; + u->nstaticsessions++; + + /* Create list elts */ + pool_get (sm->per_thread_data[cpu_index].list_pool, + per_user_translation_list_elt); + clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + per_user_translation_list_elt - + sm->per_thread_data[cpu_index].list_pool); + + per_user_translation_list_elt->value = + s - sm->per_thread_data[cpu_index].sessions; + s->per_user_index = + per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool; + s->per_user_list_head_index = u->sessions_per_user_list_head_index; + + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s->per_user_list_head_index, + per_user_translation_list_elt - + sm->per_thread_data[cpu_index].list_pool); + + s->in2out = in2out; + s->out2in = out2in; + s->in2out.protocol = out2in.protocol; + + /* Add to translation hashes */ + kv0.key = s->in2out.as_u64; + kv0.value = s - sm->per_thread_data[cpu_index].sessions; + if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) + clib_warning ("in2out key add failed"); + + kv0.key = s->out2in.as_u64; + kv0.value = s - sm->per_thread_data[cpu_index].sessions; + + if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) + clib_warning ("out2in key add failed"); + + return s; +} + +static inline u32 icmp_out2in_slow_path (snat_main_t *sm, + vlib_buffer_t * b0, + ip4_header_t * ip0, + icmp46_header_t * icmp0, + u32 sw_if_index0, + u32 rx_fib_index0, + vlib_node_runtime_t * node, + u32 next0, f64 now, + u32 cpu_index) +{ + snat_session_key_t key0, sm0; + icmp_echo_header_t *echo0; + clib_bihash_kv_8_8_t kv0, value0; + snat_session_t * s0; + u32 new_addr0, old_addr0; + u16 old_id0, new_id0; + ip_csum_t sum0; + snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; + + echo0 = (icmp_echo_header_t *)(icmp0+1); + + key0.addr = ip0->dst_address; + key0.port = echo0->identifier; + key0.protocol = SNAT_PROTOCOL_ICMP; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) + { + /* Try to match static mapping by external address and port, + destination address and port in packet */ + if (snat_static_mapping_match(sm, key0, &sm0, 1)) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + return next0; + + b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; + return SNAT_OUT2IN_NEXT_DROP; + } + + /* Create session initiated by host from external network */ + s0 = create_session_for_static_mapping(sm, b0, sm0, key0, + node, cpu_index); + if (!s0) + return SNAT_OUT2IN_NEXT_DROP; + } + else + s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value0.value); + + old_addr0 = ip0->dst_address.as_u32; + ip0->dst_address = s0->in2out.addr; + new_addr0 = ip0->dst_address.as_u32; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + old_id0 = echo0->identifier; + new_id0 = s0->in2out.port; + echo0->identifier = new_id0; + + sum0 = icmp0->checksum; + sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, + identifier); + icmp0->checksum = ip_csum_fold (sum0); + + /* Accounting */ + s0->last_heard = now; + s0->total_pkts++; + s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s0)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_list_head_index, + s0->per_user_index); + } + + return next0; +} + +static uword +snat_out2in_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + u32 n_left_from, * from, * to_next; + snat_out2in_next_t next_index; + u32 pkts_processed = 0; + snat_main_t * sm = &snat_main; + f64 now = vlib_time_now (vm); + u32 cpu_index = os_get_cpu_number (); + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t * b0, * b1; + u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP; + u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP; + u32 sw_if_index0, sw_if_index1; + ip4_header_t * ip0, *ip1; + ip_csum_t sum0, sum1; + u32 new_addr0, old_addr0; + u16 new_port0, old_port0; + u32 new_addr1, old_addr1; + u16 new_port1, old_port1; + udp_header_t * udp0, * udp1; + tcp_header_t * tcp0, * tcp1; + icmp46_header_t * icmp0, * icmp1; + snat_session_key_t key0, key1, sm0, sm1; + u32 rx_fib_index0, rx_fib_index1; + u32 proto0, proto1; + snat_session_t * s0 = 0, * s1 = 0; + clib_bihash_kv_8_8_t kv0, kv1, value0, value1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * p2, * p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + /* speculatively enqueue b0 and b1 to the current next frame */ + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + tcp0 = (tcp_header_t *) udp0; + icmp0 = (icmp46_header_t *) udp0; + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, + sw_if_index0); + + proto0 = ~0; + proto0 = (ip0->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto0; + + if (PREDICT_FALSE (proto0 == ~0)) + goto trace0; + + if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = icmp_out2in_slow_path + (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, + next0, now, cpu_index); + goto trace0; + } + + key0.addr = ip0->dst_address; + key0.port = udp0->dst_port; + key0.protocol = proto0; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) + { + /* Try to match static mapping by external address and port, + destination address and port in packet */ + if (snat_static_mapping_match(sm, key0, &sm0, 1)) + { + b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; + goto trace0; + } + + /* Create session initiated by host from external network */ + s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, + cpu_index); + if (!s0) + goto trace0; + } + else + s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value0.value); + + old_addr0 = ip0->dst_address.as_u32; + ip0->dst_address = s0->in2out.addr; + new_addr0 = ip0->dst_address.as_u32; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + old_port0 = tcp0->ports.dst; + tcp0->ports.dst = s0->in2out.port; + new_port0 = tcp0->ports.dst; + + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + + sum0 = ip_csum_update (sum0, old_port0, new_port0, + ip4_header_t /* cheat */, + length /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + old_port0 = udp0->dst_port; + udp0->dst_port = s0->in2out.port; + udp0->checksum = 0; + } + + /* Accounting */ + s0->last_heard = now; + s0->total_pkts++; + s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s0)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_list_head_index, + s0->per_user_index); + } + trace0: + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_out2in_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->session_index = ~0; + if (s0) + t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + } + + pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; + + + ip1 = vlib_buffer_get_current (b1); + udp1 = ip4_next_header (ip1); + tcp1 = (tcp_header_t *) udp1; + icmp1 = (icmp46_header_t *) udp1; + + sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX]; + rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, + sw_if_index1); + + proto1 = ~0; + proto1 = (ip1->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto1; + proto1 = (ip1->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto1; + proto1 = (ip1->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto1; + + if (PREDICT_FALSE (proto1 == ~0)) + goto trace1; + + if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP)) + { + next1 = icmp_out2in_slow_path + (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, + next1, now, cpu_index); + goto trace1; + } + + key1.addr = ip1->dst_address; + key1.port = udp1->dst_port; + key1.protocol = proto1; + key1.fib_index = rx_fib_index1; + + kv1.key = key1.as_u64; + + if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1)) + { + /* Try to match static mapping by external address and port, + destination address and port in packet */ + if (snat_static_mapping_match(sm, key1, &sm1, 1)) + { + b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; + goto trace1; + } + + /* Create session initiated by host from external network */ + s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node, + cpu_index); + if (!s1) + goto trace1; + } + else + s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value1.value); + + old_addr1 = ip1->dst_address.as_u32; + ip1->dst_address = s1->in2out.addr; + new_addr1 = ip1->dst_address.as_u32; + vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index; + + sum1 = ip1->checksum; + sum1 = ip_csum_update (sum1, old_addr1, new_addr1, + ip4_header_t, + dst_address /* changed member */); + ip1->checksum = ip_csum_fold (sum1); + + if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP)) + { + old_port1 = tcp1->ports.dst; + tcp1->ports.dst = s1->in2out.port; + new_port1 = tcp1->ports.dst; + + sum1 = tcp1->checksum; + sum1 = ip_csum_update (sum1, old_addr1, new_addr1, + ip4_header_t, + dst_address /* changed member */); + + sum1 = ip_csum_update (sum1, old_port1, new_port1, + ip4_header_t /* cheat */, + length /* changed member */); + tcp1->checksum = ip_csum_fold(sum1); + } + else + { + old_port1 = udp1->dst_port; + udp1->dst_port = s1->in2out.port; + udp1->checksum = 0; + } + + /* Accounting */ + s1->last_heard = now; + s1->total_pkts++; + s1->total_bytes += vlib_buffer_length_in_chain (vm, b1); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s1)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s1->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s1->per_user_list_head_index, + s1->per_user_index); + } + trace1: + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b1->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_out2in_trace_t *t = + vlib_add_trace (vm, node, b1, sizeof (*t)); + t->sw_if_index = sw_if_index1; + t->next_index = next1; + t->session_index = ~0; + if (s1) + t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; + } + + pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP; + + /* verify speculative enqueues, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, + to_next, n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t * b0; + u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP; + u32 sw_if_index0; + ip4_header_t * ip0; + ip_csum_t sum0; + u32 new_addr0, old_addr0; + u16 new_port0, old_port0; + udp_header_t * udp0; + tcp_header_t * tcp0; + icmp46_header_t * icmp0; + snat_session_key_t key0, sm0; + u32 rx_fib_index0; + u32 proto0; + snat_session_t * s0 = 0; + clib_bihash_kv_8_8_t kv0, value0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + tcp0 = (tcp_header_t *) udp0; + icmp0 = (icmp46_header_t *) udp0; + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, + sw_if_index0); + + proto0 = ~0; + proto0 = (ip0->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto0; + + if (PREDICT_FALSE (proto0 == ~0)) + goto trace00; + + if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = icmp_out2in_slow_path + (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, + next0, now, cpu_index); + goto trace00; + } + + key0.addr = ip0->dst_address; + key0.port = udp0->dst_port; + key0.protocol = proto0; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0)) + { + /* Try to match static mapping by external address and port, + destination address and port in packet */ + if (snat_static_mapping_match(sm, key0, &sm0, 1)) + { + b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; + goto trace00; + } + + /* Create session initiated by host from external network */ + s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, + cpu_index); + if (!s0) + goto trace00; + } + else + s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + value0.value); + + old_addr0 = ip0->dst_address.as_u32; + ip0->dst_address = s0->in2out.addr; + new_addr0 = ip0->dst_address.as_u32; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + old_port0 = tcp0->ports.dst; + tcp0->ports.dst = s0->in2out.port; + new_port0 = tcp0->ports.dst; + + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + + sum0 = ip_csum_update (sum0, old_port0, new_port0, + ip4_header_t /* cheat */, + length /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + old_port0 = udp0->dst_port; + udp0->dst_port = s0->in2out.port; + udp0->checksum = 0; + } + + /* Accounting */ + s0->last_heard = now; + s0->total_pkts++; + s0->total_bytes += vlib_buffer_length_in_chain (vm, b0); + /* Per-user LRU list maintenance for dynamic translation */ + if (!snat_is_session_static (s0)) + { + clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_index); + clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + s0->per_user_list_head_index, + s0->per_user_index); + } + trace00: + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_out2in_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + t->session_index = ~0; + if (s0) + t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + } + + pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, snat_out2in_node.index, + SNAT_OUT2IN_ERROR_OUT2IN_PACKETS, + pkts_processed); + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (snat_out2in_node) = { + .function = snat_out2in_node_fn, + .name = "snat-out2in", + .vector_size = sizeof (u32), + .format_trace = format_snat_out2in_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(snat_out2in_error_strings), + .error_strings = snat_out2in_error_strings, + + .runtime_data_bytes = sizeof (snat_runtime_t), + + .n_next_nodes = SNAT_OUT2IN_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [SNAT_OUT2IN_NEXT_DROP] = "error-drop", + [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", + }, +}; +VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn); + +static uword +snat_out2in_worker_handoff_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + snat_main_t *sm = &snat_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + u32 n_left_from, *from, *to_next = 0; + static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index; + static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index + = 0; + vlib_frame_queue_elt_t *hf = 0; + vlib_frame_t *f = 0; + int i; + u32 n_left_to_next_worker = 0, *to_next_worker = 0; + u32 next_worker_index = 0; + u32 current_worker_index = ~0; + u32 cpu_index = os_get_cpu_number (); + + ASSERT (vec_len (sm->workers)); + + if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0)) + { + vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1); + + vec_validate_init_empty (congested_handoff_queue_by_worker_index, + sm->first_worker_index + sm->num_workers - 1, + (vlib_frame_queue_t *) (~0)); + } + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + while (n_left_from > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 sw_if_index0; + u32 rx_fib_index0; + ip4_header_t * ip0; + udp_header_t * udp0; + snat_static_mapping_key_t key0; + clib_bihash_kv_8_8_t kv0, value0; + u8 do_handoff; + + bi0 = from[0]; + from += 1; + n_left_from -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + + key0.addr = ip0->dst_address; + key0.port = udp0->dst_port; + key0.fib_index = rx_fib_index0; + + kv0.key = key0.as_u64; + + /* Ever heard of of the "user" before? */ + if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0)) + { + key0.port = 0; + kv0.key = key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0)) + { + /* No, assign next available worker (RR) */ + next_worker_index = sm->first_worker_index + + sm->workers[sm->next_worker++ % vec_len (sm->workers)]; + } + else + { + /* Static mapping without port */ + next_worker_index = value0.value; + } + + /* Add to translated packets worker lookup */ + kv0.value = next_worker_index; + clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1); + } + else + next_worker_index = value0.value; + + if (PREDICT_FALSE (next_worker_index != cpu_index)) + { + do_handoff = 1; + + if (next_worker_index != current_worker_index) + { + if (hf) + hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; + + hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index, + next_worker_index, + handoff_queue_elt_by_worker_index); + + n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors; + to_next_worker = &hf->buffer_index[hf->n_vectors]; + current_worker_index = next_worker_index; + } + + /* enqueue to correct worker thread */ + to_next_worker[0] = bi0; + to_next_worker++; + n_left_to_next_worker--; + + if (n_left_to_next_worker == 0) + { + hf->n_vectors = VLIB_FRAME_SIZE; + vlib_put_frame_queue_elt (hf); + current_worker_index = ~0; + handoff_queue_elt_by_worker_index[next_worker_index] = 0; + hf = 0; + } + } + else + { + do_handoff = 0; + /* if this is 1st frame */ + if (!f) + { + f = vlib_get_frame_to_node (vm, snat_out2in_node.index); + to_next = vlib_frame_vector_args (f); + } + + to_next[0] = bi0; + to_next += 1; + f->n_vectors++; + } + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_out2in_worker_handoff_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->next_worker_index = next_worker_index; + t->do_handoff = do_handoff; + } + } + + if (f) + vlib_put_frame_to_node (vm, snat_out2in_node.index, f); + + if (hf) + hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker; + + /* Ship frames to the worker nodes */ + for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++) + { + if (handoff_queue_elt_by_worker_index[i]) + { + hf = handoff_queue_elt_by_worker_index[i]; + /* + * It works better to let the handoff node + * rate-adapt, always ship the handoff queue element. + */ + if (1 || hf->n_vectors == hf->last_n_vectors) + { + vlib_put_frame_queue_elt (hf); + handoff_queue_elt_by_worker_index[i] = 0; + } + else + hf->last_n_vectors = hf->n_vectors; + } + congested_handoff_queue_by_worker_index[i] = + (vlib_frame_queue_t *) (~0); + } + hf = 0; + current_worker_index = ~0; + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = { + .function = snat_out2in_worker_handoff_fn, + .name = "snat-out2in-worker-handoff", + .vector_size = sizeof (u32), + .format_trace = format_snat_out2in_worker_handoff_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_next_nodes = 1, + + .next_nodes = { + [0] = "error-drop", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn); + +static inline u32 icmp_out2in_fast (snat_main_t *sm, + vlib_buffer_t * b0, + ip4_header_t * ip0, + icmp46_header_t * icmp0, + u32 sw_if_index0, + vlib_node_runtime_t * node, + u32 next0, + u32 rx_fib_index0) +{ + snat_session_key_t key0, sm0; + icmp_echo_header_t *echo0; + u32 new_addr0, old_addr0; + u16 old_id0, new_id0; + ip_csum_t sum0; + snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; + + echo0 = (icmp_echo_header_t *)(icmp0+1); + + key0.addr = ip0->dst_address; + key0.port = echo0->identifier; + key0.fib_index = rx_fib_index0; + + if (snat_static_mapping_match(sm, key0, &sm0, 1)) + { + ip4_address_t * first_int_addr; + + if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0)) + { + first_int_addr = + ip4_interface_first_address (sm->ip4_main, sw_if_index0, + 0 /* just want the address */); + rt->cached_sw_if_index = sw_if_index0; + rt->cached_ip4_address = first_int_addr->as_u32; + } + + /* Don't NAT packet aimed at the intfc address */ + if (PREDICT_FALSE(ip0->dst_address.as_u32 == + rt->cached_ip4_address)) + return next0; + + b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; + return SNAT_OUT2IN_NEXT_DROP; + } + + new_addr0 = sm0.addr.as_u32; + new_id0 = sm0.port; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; + + old_addr0 = ip0->dst_address.as_u32; + ip0->dst_address.as_u32 = new_addr0; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_FALSE(new_id0 != echo0->identifier)) + { + old_id0 = echo0->identifier; + echo0->identifier = new_id0; + + sum0 = icmp0->checksum; + sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t, + identifier); + icmp0->checksum = ip_csum_fold (sum0); + } + + return next0; +} + +static uword +snat_out2in_fast_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + u32 n_left_from, * from, * to_next; + snat_out2in_next_t next_index; + u32 pkts_processed = 0; + snat_main_t * sm = &snat_main; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t * b0; + u32 next0 = SNAT_OUT2IN_NEXT_DROP; + u32 sw_if_index0; + ip4_header_t * ip0; + ip_csum_t sum0; + u32 new_addr0, old_addr0; + u16 new_port0, old_port0; + udp_header_t * udp0; + tcp_header_t * tcp0; + icmp46_header_t * icmp0; + snat_session_key_t key0, sm0; + u32 proto0; + u32 rx_fib_index0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + ip0 = vlib_buffer_get_current (b0); + udp0 = ip4_next_header (ip0); + tcp0 = (tcp_header_t *) udp0; + icmp0 = (icmp46_header_t *) udp0; + + sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0); + + vnet_feature_next (sw_if_index0, &next0, b0); + + proto0 = ~0; + proto0 = (ip0->protocol == IP_PROTOCOL_UDP) + ? SNAT_PROTOCOL_UDP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_TCP) + ? SNAT_PROTOCOL_TCP : proto0; + proto0 = (ip0->protocol == IP_PROTOCOL_ICMP) + ? SNAT_PROTOCOL_ICMP : proto0; + + if (PREDICT_FALSE (proto0 == ~0)) + goto trace00; + + if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP)) + { + next0 = icmp_out2in_fast + (sm, b0, ip0, icmp0, sw_if_index0, node, next0, rx_fib_index0); + goto trace00; + } + + key0.addr = ip0->dst_address; + key0.port = udp0->dst_port; + key0.fib_index = rx_fib_index0; + + if (snat_static_mapping_match(sm, key0, &sm0, 1)) + { + b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; + goto trace00; + } + + new_addr0 = sm0.addr.as_u32; + new_port0 = sm0.port; + vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index; + old_addr0 = ip0->dst_address.as_u32; + ip0->dst_address.as_u32 = new_addr0; + + sum0 = ip0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + ip0->checksum = ip_csum_fold (sum0); + + if (PREDICT_FALSE(new_port0 != udp0->dst_port)) + { + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + old_port0 = tcp0->ports.dst; + tcp0->ports.dst = new_port0; + + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + + sum0 = ip_csum_update (sum0, old_port0, new_port0, + ip4_header_t /* cheat */, + length /* changed member */); + tcp0->checksum = ip_csum_fold(sum0); + } + else + { + old_port0 = udp0->dst_port; + udp0->dst_port = new_port0; + udp0->checksum = 0; + } + } + else + { + if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP)) + { + sum0 = tcp0->checksum; + sum0 = ip_csum_update (sum0, old_addr0, new_addr0, + ip4_header_t, + dst_address /* changed member */); + + tcp0->checksum = ip_csum_fold(sum0); + } + } + + trace00: + + if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + snat_out2in_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = sw_if_index0; + t->next_index = next0; + } + + pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, snat_out2in_fast_node.index, + SNAT_OUT2IN_ERROR_OUT2IN_PACKETS, + pkts_processed); + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (snat_out2in_fast_node) = { + .function = snat_out2in_fast_node_fn, + .name = "snat-out2in-fast", + .vector_size = sizeof (u32), + .format_trace = format_snat_out2in_fast_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(snat_out2in_error_strings), + .error_strings = snat_out2in_error_strings, + + .runtime_data_bytes = sizeof (snat_runtime_t), + + .n_next_nodes = SNAT_OUT2IN_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup", + [SNAT_OUT2IN_NEXT_DROP] = "error-drop", + }, +}; +VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn); diff --git a/src/plugins/snat/snat.api b/src/plugins/snat/snat.api new file mode 100644 index 00000000..a191eed5 --- /dev/null +++ b/src/plugins/snat/snat.api @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @file snat.api + * @brief VPP control-plane API messages. + * + * This file defines VPP control-plane API messages which are generally + * called through a shared memory interface. + */ + +/** \brief Add/del S-NAT address range + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param is_ip4 - 1 if address type is IPv4 + @first_ip_address - first IP address + @last_ip_address - last IP address + @is_add - 1 if add, 0 if delete +*/ +define snat_add_address_range { + u32 client_index; + u32 context; + u8 is_ip4; + u8 first_ip_address[16]; + u8 last_ip_address[16]; + u8 is_add; +}; + +/** \brief Add S-NAT address range reply + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return code +*/ +define snat_add_address_range_reply { + u32 context; + i32 retval; +}; + +/** \brief Dump S-NAT addresses + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define snat_address_dump { + u32 client_index; + u32 context; +}; + +/** \brief S-NAT address details response + @param context - sender context, to match reply w/ request + @param is_ip4 - 1 if address type is IPv4 + @param ip_address - IP address +*/ +define snat_address_details { + u32 context; + u8 is_ip4; + u8 ip_address[16]; +}; + +/** \brief Enable/disable S-NAT feature on the interface + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param is_add - 1 if add, 0 if delete + @param is_inside - 1 if inside, 0 if outside + @param sw_if_index - software index of the interface +*/ +define snat_interface_add_del_feature { + u32 client_index; + u32 context; + u8 is_add; + u8 is_inside; + u32 sw_if_index; +}; + +/** \brief Enable/disable S-NAT feature on the interface reply + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return code +*/ +define snat_interface_add_del_feature_reply { + u32 context; + i32 retval; +}; + +/** \brief Dump interfaces with S-NAT feature + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define snat_interface_dump { + u32 client_index; + u32 context; +}; + +/** \brief S-NAT interface details response + @param context - sender context, to match reply w/ request + @param is_inside - 1 if inside, 0 if outside + @param sw_if_index - software index of the interface +*/ +define snat_interface_details { + u32 context; + u8 is_inside; + u32 sw_if_index; +}; + +/** \brief Add/delete S-NAT static mapping + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param is_add - 1 if add, 0 if delete + @param is_ip4 - 1 if address type is IPv4 + @param addr_only - 1 if address only mapping + @param local_ip_address - local IP address + @param external_ip_address - external IP address + @param local_port - local port number + @param external_port - external port number + @param vfr_id - VRF ID +*/ +define snat_add_static_mapping { + u32 client_index; + u32 context; + u8 is_add; + u8 is_ip4; + u8 addr_only; + u8 local_ip_address[16]; + u8 external_ip_address[16]; + u16 local_port; + u16 external_port; + u32 vrf_id; +}; + +/** \brief Add/delete S-NAT static mapping reply + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return code +*/ +define snat_add_static_mapping_reply { + u32 context; + i32 retval; +}; + +/** \brief Dump S-NAT static mappings + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define snat_static_mapping_dump { + u32 client_index; + u32 context; +}; + +/** \brief S-NAT static mapping details response + @param context - sender context, to match reply w/ request + @param is_ip4 - 1 if address type is IPv4 + @param addr_only - 1 if address only mapping + @param local_ip_address - local IP address + @param external_ip_address - external IP address + @param local_port - local port number + @param external_port - external port number + @param vfr_id - VRF ID +*/ +define snat_static_mapping_details { + u32 context; + u8 is_ip4; + u8 addr_only; + u8 local_ip_address[16]; + u8 external_ip_address[16]; + u16 local_port; + u16 external_port; + u32 vrf_id; +}; + +/** \brief Control ping from client to api server request + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define snat_control_ping +{ + u32 client_index; + u32 context; +}; + +/** \brief Control ping from the client to the server response + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return code for the request + @param vpe_pid - the pid of the vpe, returned by the server +*/ +define snat_control_ping_reply +{ + u32 context; + i32 retval; + u32 client_index; + u32 vpe_pid; +}; + +/** \brief Show S-NAT plugin startup config + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define snat_show_config +{ + u32 client_index; + u32 context; +}; + +/** \brief Show S-NAT plugin startup config reply + @param context - sender context, to match reply w/ request + @param retval - return code for the request + @param static_mapping_only - if 1 dynamic translations disabled + @param static_mapping_connection_tracking - if 1 create session data + @param translation_buckets - number of translation hash buckets + @param translation_memory_size - translation hash memory size + @param user_buckets - number of user hash buckets + @param user_memory_size - user hash memory size + @param max_translations_per_user - maximum number of translations per user + @param outside_vrf_id - outside VRF id + @param inside_vrf_id - default inside VRF id +*/ +define snat_show_config_reply +{ + u32 context; + i32 retval; + u8 static_mapping_only; + u8 static_mapping_connection_tracking; + u32 translation_buckets; + u32 translation_memory_size; + u32 user_buckets; + u32 user_memory_size; + u32 max_translations_per_user; + u32 outside_vrf_id; + u32 inside_vrf_id; +}; + +/** \brief Set S-NAT workers + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param worker_mask - S-NAT workers mask +*/ +define snat_set_workers { + u32 client_index; + u32 context; + u64 worker_mask; +}; + +/** \brief Set S-NAT workers reply + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param retval - return code +*/ +define snat_set_workers_reply { + u32 context; + i32 retval; +}; + +/** \brief Dump S-NAT workers + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request +*/ +define snat_worker_dump { + u32 client_index; + u32 context; +}; + +/** \brief S-NAT workers details response + @param context - sender context, to match reply w/ request + @param worker_index - worker index + @param lcore_id - lcore ID + @param name - worker name +*/ +define snat_worker_details { + u32 context; + u32 worker_index; + u32 lcore_id; + u8 name[64]; +}; diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c new file mode 100644 index 00000000..bc995684 --- /dev/null +++ b/src/plugins/snat/snat.c @@ -0,0 +1,1957 @@ +/* + * snat.c - simple nat plugin + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include + +snat_main_t snat_main; + +/* define message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include +#undef vl_endianfun + +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +/* Macro to finish up custom dump fns */ +#define FINISH \ + vec_add1 (s, 0); \ + vl_print (handle, (char *)s); \ + vec_free (s); \ + return handle; + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ + +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +#define REPLY_MACRO2(t, body) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+sm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + do {body;} while (0); \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + + +/* Hook up input features */ +VNET_FEATURE_INIT (ip4_snat_in2out, static) = { + .arc_name = "ip4-unicast", + .node_name = "snat-in2out", + .runs_before = VNET_FEATURES ("snat-out2in"), +}; +VNET_FEATURE_INIT (ip4_snat_out2in, static) = { + .arc_name = "ip4-unicast", + .node_name = "snat-out2in", + .runs_before = VNET_FEATURES ("ip4-lookup"), +}; +VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = { + .arc_name = "ip4-unicast", + .node_name = "snat-in2out-worker-handoff", + .runs_before = VNET_FEATURES ("snat-out2in-worker-handoff"), +}; +VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = { + .arc_name = "ip4-unicast", + .node_name = "snat-out2in-worker-handoff", + .runs_before = VNET_FEATURES ("ip4-lookup"), +}; +VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = { + .arc_name = "ip4-unicast", + .node_name = "snat-in2out-fast", + .runs_before = VNET_FEATURES ("snat-out2in-fast"), +}; +VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = { + .arc_name = "ip4-unicast", + .node_name = "snat-out2in-fast", + .runs_before = VNET_FEATURES ("ip4-lookup"), +}; + + +/* + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ + +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + snat_main_t * sm = &snat_main; + clib_error_t * error = 0; + + sm->vlib_main = vm; + sm->vnet_main = h->vnet_main; + sm->ethernet_main = h->ethernet_main; + + return error; +} + +/*$$$$$ move to an installed header file */ +#if (1 || CLIB_DEBUG > 0) /* "trust, but verify" */ + +#define VALIDATE_SW_IF_INDEX(mp) \ + do { u32 __sw_if_index = ntohl(mp->sw_if_index); \ + vnet_main_t *__vnm = vnet_get_main(); \ + if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ + __sw_if_index)) { \ + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ + goto bad_sw_if_index; \ + } \ +} while(0); + +#define BAD_SW_IF_INDEX_LABEL \ +do { \ +bad_sw_if_index: \ + ; \ +} while (0); + +#define VALIDATE_RX_SW_IF_INDEX(mp) \ + do { u32 __rx_sw_if_index = ntohl(mp->rx_sw_if_index); \ + vnet_main_t *__vnm = vnet_get_main(); \ + if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ + __rx_sw_if_index)) { \ + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ + goto bad_rx_sw_if_index; \ + } \ +} while(0); + +#define BAD_RX_SW_IF_INDEX_LABEL \ +do { \ +bad_rx_sw_if_index: \ + ; \ +} while (0); + +#define VALIDATE_TX_SW_IF_INDEX(mp) \ + do { u32 __tx_sw_if_index = ntohl(mp->tx_sw_if_index); \ + vnet_main_t *__vnm = vnet_get_main(); \ + if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ + __tx_sw_if_index)) { \ + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ + goto bad_tx_sw_if_index; \ + } \ +} while(0); + +#define BAD_TX_SW_IF_INDEX_LABEL \ +do { \ +bad_tx_sw_if_index: \ + ; \ +} while (0); + +#else + +#define VALIDATE_SW_IF_INDEX(mp) +#define BAD_SW_IF_INDEX_LABEL +#define VALIDATE_RX_SW_IF_INDEX(mp) +#define BAD_RX_SW_IF_INDEX_LABEL +#define VALIDATE_TX_SW_IF_INDEX(mp) +#define BAD_TX_SW_IF_INDEX_LABEL + +#endif /* CLIB_DEBUG > 0 */ + +void snat_add_address (snat_main_t *sm, ip4_address_t *addr) +{ + snat_address_t * ap; + + /* Check if address already exists */ + vec_foreach (ap, sm->addresses) + { + if (ap->addr.as_u32 == addr->as_u32) + return; + } + + vec_add2 (sm->addresses, ap, 1); + ap->addr = *addr; + clib_bitmap_alloc (ap->busy_port_bitmap, 65535); +} + +static int is_snat_address_used_in_static_mapping (snat_main_t *sm, + ip4_address_t addr) +{ + snat_static_mapping_t *m; + pool_foreach (m, sm->static_mappings, + ({ + if (m->external_addr.as_u32 == addr.as_u32) + return 1; + })); + + return 0; +} + +int snat_del_address (snat_main_t *sm, ip4_address_t addr) +{ + snat_address_t *a = 0; + snat_session_t *ses; + u32 *ses_to_be_removed = 0, *ses_index; + clib_bihash_kv_8_8_t kv, value; + snat_user_key_t user_key; + snat_user_t *u; + snat_main_per_thread_data_t *tsm; + + int i; + + /* Find SNAT address */ + for (i=0; i < vec_len (sm->addresses); i++) + { + if (sm->addresses[i].addr.as_u32 == addr.as_u32) + { + a = sm->addresses + i; + break; + } + } + if (!a) + return VNET_API_ERROR_NO_SUCH_ENTRY; + + /* Check if address is used in some static mapping */ + if (is_snat_address_used_in_static_mapping(sm, addr)) + { + clib_warning ("address used in static mapping"); + return VNET_API_ERROR_UNSPECIFIED; + } + + /* Delete sessions using address */ + if (a->busy_ports) + { + vec_foreach (tsm, sm->per_thread_data) + { + pool_foreach (ses, tsm->sessions, ({ + if (ses->out2in.addr.as_u32 == addr.as_u32) + { + vec_add1 (ses_to_be_removed, ses - tsm->sessions); + kv.key = ses->in2out.as_u64; + clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0); + kv.key = ses->out2in.as_u64; + clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0); + clib_dlist_remove (tsm->list_pool, ses->per_user_index); + user_key.addr = ses->in2out.addr; + user_key.fib_index = ses->in2out.fib_index; + kv.key = user_key.as_u64; + if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value)) + { + u = pool_elt_at_index (tsm->users, value.value); + u->nsessions--; + } + } + })); + + vec_foreach (ses_index, ses_to_be_removed) + pool_put_index (tsm->sessions, ses_index[0]); + + vec_free (ses_to_be_removed); + } + } + + vec_del1 (sm->addresses, i); + + return 0; +} + +static void increment_v4_address (ip4_address_t * a) +{ + u32 v; + + v = clib_net_to_host_u32(a->as_u32) + 1; + a->as_u32 = clib_host_to_net_u32(v); +} + +/** + * @brief Add static mapping. + * + * Create static mapping between local addr+port and external addr+port. + * + * @param l_addr Local IPv4 address. + * @param e_addr External IPv4 address. + * @param l_port Local port number. + * @param e_port External port number. + * @param vrf_id VRF ID. + * @param addr_only If 0 address port and pair mapping, otherwise address only. + * @param is_add If 0 delete static mapping, otherwise add. + * + * @returns + */ +int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr, + u16 l_port, u16 e_port, u32 vrf_id, int addr_only, + int is_add) +{ + snat_main_t * sm = &snat_main; + snat_static_mapping_t *m; + snat_static_mapping_key_t m_key; + clib_bihash_kv_8_8_t kv, value; + snat_address_t *a = 0; + u32 fib_index = ~0; + uword * p; + int i; + + /* If outside FIB index is not resolved yet */ + if (sm->outside_fib_index == ~0) + { + p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id); + if (!p) + return VNET_API_ERROR_NO_SUCH_FIB; + sm->outside_fib_index = p[0]; + } + + m_key.addr = e_addr; + m_key.port = addr_only ? 0 : e_port; + m_key.fib_index = sm->outside_fib_index; + kv.key = m_key.as_u64; + if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value)) + m = 0; + else + m = pool_elt_at_index (sm->static_mappings, value.value); + + if (is_add) + { + if (m) + return VNET_API_ERROR_VALUE_EXIST; + + /* Convert VRF id to FIB index */ + if (vrf_id != ~0) + { + p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id); + if (!p) + return VNET_API_ERROR_NO_SUCH_FIB; + fib_index = p[0]; + } + /* If not specified use inside VRF id from SNAT plugin startup config */ + else + { + if (sm->inside_fib_index == ~0) + { + p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->inside_vrf_id); + if (!p) + return VNET_API_ERROR_NO_SUCH_FIB; + fib_index = p[0]; + sm->inside_fib_index = fib_index; + } + else + fib_index = sm->inside_fib_index; + + vrf_id = sm->inside_vrf_id; + } + + /* Find external address in allocated addresses and reserve port for + address and port pair mapping when dynamic translations enabled */ + if (!addr_only && !(sm->static_mapping_only)) + { + for (i = 0; i < vec_len (sm->addresses); i++) + { + if (sm->addresses[i].addr.as_u32 == e_addr.as_u32) + { + a = sm->addresses + i; + /* External port must be unused */ + if (clib_bitmap_get_no_check (a->busy_port_bitmap, e_port)) + return VNET_API_ERROR_INVALID_VALUE; + clib_bitmap_set_no_check (a->busy_port_bitmap, e_port, 1); + if (e_port > 1024) + a->busy_ports++; + + break; + } + } + /* External address must be allocated */ + if (!a) + return VNET_API_ERROR_NO_SUCH_ENTRY; + } + + pool_get (sm->static_mappings, m); + memset (m, 0, sizeof (*m)); + m->local_addr = l_addr; + m->external_addr = e_addr; + m->addr_only = addr_only; + m->vrf_id = vrf_id; + m->fib_index = fib_index; + if (!addr_only) + { + m->local_port = l_port; + m->external_port = e_port; + } + + m_key.addr = m->local_addr; + m_key.port = m->local_port; + m_key.fib_index = m->fib_index; + kv.key = m_key.as_u64; + kv.value = m - sm->static_mappings; + clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1); + + m_key.addr = m->external_addr; + m_key.port = m->external_port; + m_key.fib_index = sm->outside_fib_index; + kv.key = m_key.as_u64; + kv.value = m - sm->static_mappings; + clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1); + + /* Assign worker */ + if (sm->workers) + { + snat_user_key_t w_key0; + snat_static_mapping_key_t w_key1; + + w_key0.addr = m->local_addr; + w_key0.fib_index = m->fib_index; + kv.key = w_key0.as_u64; + + if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value)) + { + kv.value = sm->first_worker_index + + sm->workers[sm->next_worker++ % vec_len (sm->workers)]; + + clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv, 1); + } + else + { + kv.value = value.value; + } + + w_key1.addr = m->external_addr; + w_key1.port = clib_host_to_net_u16 (m->external_port); + w_key1.fib_index = sm->outside_fib_index; + kv.key = w_key1.as_u64; + clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv, 1); + } + } + else + { + if (!m) + return VNET_API_ERROR_NO_SUCH_ENTRY; + + /* Free external address port */ + if (!addr_only && !(sm->static_mapping_only)) + { + for (i = 0; i < vec_len (sm->addresses); i++) + { + if (sm->addresses[i].addr.as_u32 == e_addr.as_u32) + { + a = sm->addresses + i; + clib_bitmap_set_no_check (a->busy_port_bitmap, e_port, 0); + a->busy_ports--; + + break; + } + } + } + + m_key.addr = m->local_addr; + m_key.port = m->local_port; + m_key.fib_index = m->fib_index; + kv.key = m_key.as_u64; + clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0); + + m_key.addr = m->external_addr; + m_key.port = m->external_port; + m_key.fib_index = sm->outside_fib_index; + kv.key = m_key.as_u64; + clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0); + + /* Delete session(s) for static mapping if exist */ + if (!(sm->static_mapping_only) || + (sm->static_mapping_only && sm->static_mapping_connection_tracking)) + { + snat_user_key_t u_key; + snat_user_t *u; + dlist_elt_t * head, * elt; + u32 elt_index, head_index, del_elt_index; + u32 ses_index; + u64 user_index; + snat_session_t * s; + snat_main_per_thread_data_t *tsm; + + u_key.addr = m->local_addr; + u_key.fib_index = m->fib_index; + kv.key = u_key.as_u64; + if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value)) + { + user_index = value.value; + if (!clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value)) + tsm = vec_elt_at_index (sm->per_thread_data, value.value); + else + tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers); + u = pool_elt_at_index (tsm->users, user_index); + if (u->nstaticsessions) + { + head_index = u->sessions_per_user_list_head_index; + head = pool_elt_at_index (tsm->list_pool, head_index); + elt_index = head->next; + elt = pool_elt_at_index (tsm->list_pool, elt_index); + ses_index = elt->value; + while (ses_index != ~0) + { + s = pool_elt_at_index (tsm->sessions, ses_index); + del_elt_index = elt_index; + elt_index = elt->next; + elt = pool_elt_at_index (tsm->list_pool, elt_index); + ses_index = elt->value; + + if (!addr_only) + { + if ((s->out2in.addr.as_u32 != e_addr.as_u32) && + (clib_net_to_host_u16 (s->out2in.port) != e_port)) + continue; + } + + value.key = s->in2out.as_u64; + clib_bihash_add_del_8_8 (&sm->in2out, &value, 0); + value.key = s->out2in.as_u64; + clib_bihash_add_del_8_8 (&sm->out2in, &value, 0); + pool_put (tsm->sessions, s); + + clib_dlist_remove (tsm->list_pool, del_elt_index); + pool_put_index (tsm->list_pool, del_elt_index); + u->nstaticsessions--; + + if (!addr_only) + break; + } + if (addr_only) + { + pool_put (tsm->users, u); + clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 0); + } + } + } + } + + /* Delete static mapping from pool */ + pool_put (sm->static_mappings, m); + } + + return 0; +} + +static int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del) +{ + snat_main_t *sm = &snat_main; + snat_interface_t *i; + const char * feature_name; + + if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)) + feature_name = is_inside ? "snat-in2out-fast" : "snat-out2in-fast"; + else + { + if (sm->num_workers > 1) + feature_name = is_inside ? "snat-in2out-worker-handoff" : "snat-out2in-worker-handoff"; + else + feature_name = is_inside ? "snat-in2out" : "snat-out2in"; + } + + vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, + !is_del, 0, 0); + + if (sm->fq_in2out_index == ~0) + sm->fq_in2out_index = vlib_frame_queue_main_init (snat_in2out_node.index, 0); + + if (sm->fq_out2in_index == ~0) + sm->fq_out2in_index = vlib_frame_queue_main_init (snat_out2in_node.index, 0); + + pool_foreach (i, sm->interfaces, + ({ + if (i->sw_if_index == sw_if_index) + { + if (is_del) + pool_put (sm->interfaces, i); + else + return VNET_API_ERROR_VALUE_EXIST; + + return 0; + } + })); + + if (is_del) + return VNET_API_ERROR_NO_SUCH_ENTRY; + + pool_get (sm->interfaces, i); + i->sw_if_index = sw_if_index; + i->is_inside = is_inside; + + return 0; +} + +static int snat_set_workers (uword * bitmap) +{ + snat_main_t *sm = &snat_main; + int i; + + if (sm->num_workers < 2) + return VNET_API_ERROR_FEATURE_DISABLED; + + if (clib_bitmap_last_set (bitmap) >= sm->num_workers) + return VNET_API_ERROR_INVALID_WORKER; + + vec_free (sm->workers); + clib_bitmap_foreach (i, bitmap, + ({ + vec_add1(sm->workers, i); + })); + + return 0; +} + +static void +vl_api_snat_add_address_range_t_handler +(vl_api_snat_add_address_range_t * mp) +{ + snat_main_t * sm = &snat_main; + vl_api_snat_add_address_range_reply_t * rmp; + ip4_address_t this_addr; + u32 start_host_order, end_host_order; + int i, count; + int rv = 0; + u32 * tmp; + + if (mp->is_ip4 != 1) + { + rv = VNET_API_ERROR_UNIMPLEMENTED; + goto send_reply; + } + + if (sm->static_mapping_only) + { + rv = VNET_API_ERROR_FEATURE_DISABLED; + goto send_reply; + } + + tmp = (u32 *) mp->first_ip_address; + start_host_order = clib_host_to_net_u32 (tmp[0]); + tmp = (u32 *) mp->last_ip_address; + end_host_order = clib_host_to_net_u32 (tmp[0]); + + count = (end_host_order - start_host_order) + 1; + + if (count > 1024) + clib_warning ("%U - %U, %d addresses...", + format_ip4_address, mp->first_ip_address, + format_ip4_address, mp->last_ip_address, + count); + + memcpy (&this_addr.as_u8, mp->first_ip_address, 4); + + for (i = 0; i < count; i++) + { + if (mp->is_add) + snat_add_address (sm, &this_addr); + else + rv = snat_del_address (sm, this_addr); + + if (rv) + goto send_reply; + + increment_v4_address (&this_addr); + } + + send_reply: + REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY); +} + +static void *vl_api_snat_add_address_range_t_print +(vl_api_snat_add_address_range_t *mp, void * handle) +{ + u8 * s; + + s = format (0, "SCRIPT: snat_add_address_range "); + s = format (s, "%U ", format_ip4_address, mp->first_ip_address); + if (memcmp (mp->first_ip_address, mp->last_ip_address, 4)) + { + s = format (s, " - %U ", format_ip4_address, mp->last_ip_address); + } + FINISH; +} + +static void +send_snat_address_details +(snat_address_t * a, unix_shared_memory_queue_t * q, u32 context) +{ + vl_api_snat_address_details_t *rmp; + snat_main_t * sm = &snat_main; + + rmp = vl_msg_api_alloc (sizeof (*rmp)); + memset (rmp, 0, sizeof (*rmp)); + rmp->_vl_msg_id = ntohs (VL_API_SNAT_ADDRESS_DETAILS+sm->msg_id_base); + rmp->is_ip4 = 1; + clib_memcpy (rmp->ip_address, &(a->addr), 4); + rmp->context = context; + + vl_msg_api_send_shmem (q, (u8 *) & rmp); +} + +static void +vl_api_snat_address_dump_t_handler +(vl_api_snat_address_dump_t * mp) +{ + unix_shared_memory_queue_t *q; + snat_main_t * sm = &snat_main; + snat_address_t * a; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + return; + + vec_foreach (a, sm->addresses) + send_snat_address_details (a, q, mp->context); +} + +static void *vl_api_snat_address_dump_t_print +(vl_api_snat_address_dump_t *mp, void * handle) +{ + u8 *s; + + s = format (0, "SCRIPT: snat_address_dump "); + + FINISH; +} + +static void +vl_api_snat_interface_add_del_feature_t_handler +(vl_api_snat_interface_add_del_feature_t * mp) +{ + snat_main_t * sm = &snat_main; + vl_api_snat_interface_add_del_feature_reply_t * rmp; + u8 is_del = mp->is_add == 0; + u32 sw_if_index = ntohl(mp->sw_if_index); + int rv = 0; + + VALIDATE_SW_IF_INDEX(mp); + + rv = snat_interface_add_del (sw_if_index, mp->is_inside, is_del); + + BAD_SW_IF_INDEX_LABEL; + + REPLY_MACRO(VL_API_SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY); +} + +static void *vl_api_snat_interface_add_del_feature_t_print +(vl_api_snat_interface_add_del_feature_t * mp, void *handle) +{ + u8 * s; + + s = format (0, "SCRIPT: snat_interface_add_del_feature "); + s = format (s, "sw_if_index %d %s %s", + clib_host_to_net_u32(mp->sw_if_index), + mp->is_inside ? "in":"out", + mp->is_add ? "" : "del"); + + FINISH; +} + +static void +send_snat_interface_details +(snat_interface_t * i, unix_shared_memory_queue_t * q, u32 context) +{ + vl_api_snat_interface_details_t *rmp; + snat_main_t * sm = &snat_main; + + rmp = vl_msg_api_alloc (sizeof (*rmp)); + memset (rmp, 0, sizeof (*rmp)); + rmp->_vl_msg_id = ntohs (VL_API_SNAT_INTERFACE_DETAILS+sm->msg_id_base); + rmp->sw_if_index = ntohl (i->sw_if_index); + rmp->is_inside = i->is_inside; + rmp->context = context; + + vl_msg_api_send_shmem (q, (u8 *) & rmp); +} + +static void +vl_api_snat_interface_dump_t_handler +(vl_api_snat_interface_dump_t * mp) +{ + unix_shared_memory_queue_t *q; + snat_main_t * sm = &snat_main; + snat_interface_t * i; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + return; + + pool_foreach (i, sm->interfaces, + ({ + send_snat_interface_details(i, q, mp->context); + })); +} + +static void *vl_api_snat_interface_dump_t_print +(vl_api_snat_interface_dump_t *mp, void * handle) +{ + u8 *s; + + s = format (0, "SCRIPT: snat_interface_dump "); + + FINISH; +}static void + +vl_api_snat_add_static_mapping_t_handler +(vl_api_snat_add_static_mapping_t * mp) +{ + snat_main_t * sm = &snat_main; + vl_api_snat_add_static_mapping_reply_t * rmp; + ip4_address_t local_addr, external_addr; + u16 local_port = 0, external_port = 0; + u32 vrf_id; + int rv = 0; + + if (mp->is_ip4 != 1) + { + rv = VNET_API_ERROR_UNIMPLEMENTED; + goto send_reply; + } + + memcpy (&local_addr.as_u8, mp->local_ip_address, 4); + memcpy (&external_addr.as_u8, mp->external_ip_address, 4); + if (mp->addr_only == 0) + { + local_port = clib_net_to_host_u16 (mp->local_port); + external_port = clib_net_to_host_u16 (mp->external_port); + } + vrf_id = clib_net_to_host_u32 (mp->vrf_id); + + rv = snat_add_static_mapping(local_addr, external_addr, local_port, + external_port, vrf_id, mp->addr_only, + mp->is_add); + + send_reply: + REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY); +} + +static void *vl_api_snat_add_static_mapping_t_print +(vl_api_snat_add_static_mapping_t *mp, void * handle) +{ + u8 * s; + + s = format (0, "SCRIPT: snat_add_static_mapping "); + s = format (s, "local_addr %U external_addr %U ", + format_ip4_address, mp->local_ip_address, + format_ip4_address, mp->external_ip_address); + + if (mp->addr_only == 0) + s = format (s, "local_port %d external_port %d ", + clib_net_to_host_u16 (mp->local_port), + clib_net_to_host_u16 (mp->external_port)); + + if (mp->vrf_id != ~0) + s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id)); + + FINISH; +} + +static void +send_snat_static_mapping_details +(snat_static_mapping_t * m, unix_shared_memory_queue_t * q, u32 context) +{ + vl_api_snat_static_mapping_details_t *rmp; + snat_main_t * sm = &snat_main; + + rmp = vl_msg_api_alloc (sizeof (*rmp)); + memset (rmp, 0, sizeof (*rmp)); + rmp->_vl_msg_id = ntohs (VL_API_SNAT_STATIC_MAPPING_DETAILS+sm->msg_id_base); + rmp->is_ip4 = 1; + rmp->addr_only = m->addr_only; + clib_memcpy (rmp->local_ip_address, &(m->local_addr), 4); + clib_memcpy (rmp->external_ip_address, &(m->external_addr), 4); + rmp->local_port = htons (m->local_port); + rmp->external_port = htons (m->external_port); + rmp->vrf_id = htonl (m->vrf_id); + rmp->context = context; + + vl_msg_api_send_shmem (q, (u8 *) & rmp); +} + +static void +vl_api_snat_static_mapping_dump_t_handler +(vl_api_snat_static_mapping_dump_t * mp) +{ + unix_shared_memory_queue_t *q; + snat_main_t * sm = &snat_main; + snat_static_mapping_t * m; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + return; + + pool_foreach (m, sm->static_mappings, + ({ + send_snat_static_mapping_details (m, q, mp->context); + })); +} + +static void *vl_api_snat_static_mapping_dump_t_print +(vl_api_snat_static_mapping_dump_t *mp, void * handle) +{ + u8 *s; + + s = format (0, "SCRIPT: snat_static_mapping_dump "); + + FINISH; +} + +static void +vl_api_snat_control_ping_t_handler +(vl_api_snat_control_ping_t * mp) +{ + vl_api_snat_control_ping_reply_t *rmp; + snat_main_t * sm = &snat_main; + int rv = 0; + + REPLY_MACRO2(VL_API_SNAT_CONTROL_PING_REPLY, + ({ + rmp->vpe_pid = ntohl (getpid()); + })); +} + +static void *vl_api_snat_control_ping_t_print +(vl_api_snat_control_ping_t *mp, void * handle) +{ + u8 *s; + + s = format (0, "SCRIPT: snat_control_ping "); + + FINISH; +} + +static void +vl_api_snat_show_config_t_handler +(vl_api_snat_show_config_t * mp) +{ + vl_api_snat_show_config_reply_t *rmp; + snat_main_t * sm = &snat_main; + int rv = 0; + + REPLY_MACRO2(VL_API_SNAT_SHOW_CONFIG_REPLY, + ({ + rmp->translation_buckets = htonl (sm->translation_buckets); + rmp->translation_memory_size = htonl (sm->translation_memory_size); + rmp->user_buckets = htonl (sm->user_buckets); + rmp->user_memory_size = htonl (sm->user_memory_size); + rmp->max_translations_per_user = htonl (sm->max_translations_per_user); + rmp->outside_vrf_id = htonl (sm->outside_vrf_id); + rmp->inside_vrf_id = htonl (sm->inside_vrf_id); + rmp->static_mapping_only = sm->static_mapping_only; + rmp->static_mapping_connection_tracking = + sm->static_mapping_connection_tracking; + })); +} + +static void *vl_api_snat_show_config_t_print +(vl_api_snat_show_config_t *mp, void * handle) +{ + u8 *s; + + s = format (0, "SCRIPT: snat_show_config "); + + FINISH; +} + +static void +vl_api_snat_set_workers_t_handler +(vl_api_snat_set_workers_t * mp) +{ + snat_main_t * sm = &snat_main; + vl_api_snat_set_workers_reply_t * rmp; + int rv = 0; + uword *bitmap = 0; + u64 mask = clib_net_to_host_u64 (mp->worker_mask); + + if (sm->num_workers < 2) + { + rv = VNET_API_ERROR_FEATURE_DISABLED; + goto send_reply; + } + + bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask)); + rv = snat_set_workers(bitmap); + clib_bitmap_free (bitmap); + + send_reply: + REPLY_MACRO (VL_API_SNAT_SET_WORKERS_REPLY); +} + +static void *vl_api_snat_set_workers_t_print +(vl_api_snat_set_workers_t *mp, void * handle) +{ + u8 * s; + uword *bitmap = 0; + u8 first = 1; + int i; + u64 mask = clib_net_to_host_u64 (mp->worker_mask); + + s = format (0, "SCRIPT: snat_set_workers "); + bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask)); + clib_bitmap_foreach (i, bitmap, + ({ + if (first) + s = format (s, "%d", i); + else + s = format (s, ",%d", i); + first = 0; + })); + clib_bitmap_free (bitmap); + FINISH; +} + +static void +send_snat_worker_details +(u32 worker_index, unix_shared_memory_queue_t * q, u32 context) +{ + vl_api_snat_worker_details_t *rmp; + snat_main_t * sm = &snat_main; + vlib_worker_thread_t *w = + vlib_worker_threads + worker_index + sm->first_worker_index; + + rmp = vl_msg_api_alloc (sizeof (*rmp)); + memset (rmp, 0, sizeof (*rmp)); + rmp->_vl_msg_id = ntohs (VL_API_SNAT_WORKER_DETAILS+sm->msg_id_base); + rmp->context = context; + rmp->worker_index = htonl (worker_index); + rmp->lcore_id = htonl (w->lcore_id); + strncpy ((char *) rmp->name, (char *) w->name, ARRAY_LEN (rmp->name) - 1); + + vl_msg_api_send_shmem (q, (u8 *) & rmp); +} + +static void +vl_api_snat_worker_dump_t_handler +(vl_api_snat_worker_dump_t * mp) +{ + unix_shared_memory_queue_t *q; + snat_main_t * sm = &snat_main; + u32 * worker_index; + + q = vl_api_client_index_to_input_queue (mp->client_index); + if (q == 0) + return; + + vec_foreach (worker_index, sm->workers) + { + send_snat_worker_details(*worker_index, q, mp->context); + } +} + +static void *vl_api_snat_worker_dump_t_print +(vl_api_snat_worker_dump_t *mp, void * handle) +{ + u8 *s; + + s = format (0, "SCRIPT: snat_worker_dump "); + + FINISH; +} + +/* List of message types that this plugin understands */ +#define foreach_snat_plugin_api_msg \ +_(SNAT_ADD_ADDRESS_RANGE, snat_add_address_range) \ +_(SNAT_INTERFACE_ADD_DEL_FEATURE, snat_interface_add_del_feature) \ +_(SNAT_ADD_STATIC_MAPPING, snat_add_static_mapping) \ +_(SNAT_CONTROL_PING, snat_control_ping) \ +_(SNAT_STATIC_MAPPING_DUMP, snat_static_mapping_dump) \ +_(SNAT_SHOW_CONFIG, snat_show_config) \ +_(SNAT_ADDRESS_DUMP, snat_address_dump) \ +_(SNAT_INTERFACE_DUMP, snat_interface_dump) \ +_(SNAT_SET_WORKERS, snat_set_workers) \ +_(SNAT_WORKER_DUMP, snat_worker_dump) + +/* Set up the API message handling tables */ +static clib_error_t * +snat_plugin_api_hookup (vlib_main_t *vm) +{ + snat_main_t * sm __attribute__ ((unused)) = &snat_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_snat_plugin_api_msg; +#undef _ + + return 0; +} + +#define vl_msg_name_crc_list +#include +#undef vl_msg_name_crc_list + +static void +setup_message_id_table (snat_main_t * sm, api_main_t * am) +{ +#define _(id,n,crc) \ + vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base); + foreach_vl_msg_name_crc_snat; +#undef _ +} + +static void plugin_custom_dump_configure (snat_main_t * sm) +{ +#define _(n,f) sm->api_main->msg_print_handlers \ + [VL_API_##n + sm->msg_id_base] \ + = (void *) vl_api_##f##_t_print; + foreach_snat_plugin_api_msg; +#undef _ +} + +static clib_error_t * snat_init (vlib_main_t * vm) +{ + snat_main_t * sm = &snat_main; + clib_error_t * error = 0; + ip4_main_t * im = &ip4_main; + ip_lookup_main_t * lm = &im->lookup_main; + u8 * name; + uword *p; + vlib_thread_registration_t *tr; + vlib_thread_main_t *tm = vlib_get_thread_main (); + uword *bitmap = 0; + u32 i; + + name = format (0, "snat_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + sm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + + sm->vlib_main = vm; + sm->vnet_main = vnet_get_main(); + sm->ip4_main = im; + sm->ip4_lookup_main = lm; + sm->api_main = &api_main; + sm->first_worker_index = 0; + sm->next_worker = 0; + sm->num_workers = 0; + sm->workers = 0; + sm->fq_in2out_index = ~0; + sm->fq_out2in_index = ~0; + + p = hash_get_mem (tm->thread_registrations_by_name, "workers"); + if (p) + { + tr = (vlib_thread_registration_t *) p[0]; + if (tr) + { + sm->num_workers = tr->count; + sm->first_worker_index = tr->first_index; + } + } + + /* Use all available workers by default */ + if (sm->num_workers > 1) + { + for (i=0; i < sm->num_workers; i++) + bitmap = clib_bitmap_set (bitmap, i, 1); + snat_set_workers(bitmap); + clib_bitmap_free (bitmap); + } + + error = snat_plugin_api_hookup (vm); + + /* Add our API messages to the global name_crc hash table */ + setup_message_id_table (sm, &api_main); + + plugin_custom_dump_configure (sm); + vec_free(name); + + return error; +} + +VLIB_INIT_FUNCTION (snat_init); + +void snat_free_outside_address_and_port (snat_main_t * sm, + snat_session_key_t * k, + u32 address_index) +{ + snat_address_t *a; + u16 port_host_byte_order = clib_net_to_host_u16 (k->port); + + ASSERT (address_index < vec_len (sm->addresses)); + + a = sm->addresses + address_index; + + ASSERT (clib_bitmap_get_no_check (a->busy_port_bitmap, + port_host_byte_order) == 1); + + clib_bitmap_set_no_check (a->busy_port_bitmap, port_host_byte_order, 0); + a->busy_ports--; +} + +/** + * @brief Match SNAT static mapping. + * + * @param sm SNAT main. + * @param match Address and port to match. + * @param mapping External or local address and port of the matched mapping. + * @param by_external If 0 match by local address otherwise match by external + * address. + * + * @returns 0 if match found otherwise 1. + */ +int snat_static_mapping_match (snat_main_t * sm, + snat_session_key_t match, + snat_session_key_t * mapping, + u8 by_external) +{ + clib_bihash_kv_8_8_t kv, value; + snat_static_mapping_t *m; + snat_static_mapping_key_t m_key; + clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local; + + if (by_external) + mapping_hash = &sm->static_mapping_by_external; + + m_key.addr = match.addr; + m_key.port = clib_net_to_host_u16 (match.port); + m_key.fib_index = match.fib_index; + + kv.key = m_key.as_u64; + + if (clib_bihash_search_8_8 (mapping_hash, &kv, &value)) + { + /* Try address only mapping */ + m_key.port = 0; + kv.key = m_key.as_u64; + if (clib_bihash_search_8_8 (mapping_hash, &kv, &value)) + return 1; + } + + m = pool_elt_at_index (sm->static_mappings, value.value); + + if (by_external) + { + mapping->addr = m->local_addr; + /* Address only mapping doesn't change port */ + mapping->port = m->addr_only ? match.port + : clib_host_to_net_u16 (m->local_port); + mapping->fib_index = m->fib_index; + } + else + { + mapping->addr = m->external_addr; + /* Address only mapping doesn't change port */ + mapping->port = m->addr_only ? match.port + : clib_host_to_net_u16 (m->external_port); + mapping->fib_index = sm->outside_fib_index; + } + + return 0; +} + +int snat_alloc_outside_address_and_port (snat_main_t * sm, + snat_session_key_t * k, + u32 * address_indexp) +{ + int i; + snat_address_t *a; + u32 portnum; + + for (i = 0; i < vec_len (sm->addresses); i++) + { + if (sm->addresses[i].busy_ports < (65535-1024)) + { + a = sm->addresses + i; + + while (1) + { + portnum = random_u32 (&sm->random_seed); + portnum &= 0xFFFF; + if (portnum < 1024) + continue; + if (clib_bitmap_get_no_check (a->busy_port_bitmap, portnum)) + continue; + clib_bitmap_set_no_check (a->busy_port_bitmap, portnum, 1); + a->busy_ports++; + /* Caller sets protocol and fib index */ + k->addr = a->addr; + k->port = clib_host_to_net_u16(portnum); + *address_indexp = i; + return 0; + } + } + } + /* Totally out of translations to use... */ + return 1; +} + + +static clib_error_t * +add_address_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + snat_main_t * sm = &snat_main; + ip4_address_t start_addr, end_addr, this_addr; + u32 start_host_order, end_host_order; + int i, count; + int is_add = 1; + int rv = 0; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "%U - %U", + unformat_ip4_address, &start_addr, + unformat_ip4_address, &end_addr)) + ; + else if (unformat (line_input, "%U", unformat_ip4_address, &start_addr)) + end_addr = start_addr; + else if (unformat (line_input, "del")) + is_add = 0; + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + } + unformat_free (line_input); + + if (sm->static_mapping_only) + return clib_error_return (0, "static mapping only mode"); + + start_host_order = clib_host_to_net_u32 (start_addr.as_u32); + end_host_order = clib_host_to_net_u32 (end_addr.as_u32); + + if (end_host_order < start_host_order) + return clib_error_return (0, "end address less than start address"); + + count = (end_host_order - start_host_order) + 1; + + if (count > 1024) + clib_warning ("%U - %U, %d addresses...", + format_ip4_address, &start_addr, + format_ip4_address, &end_addr, + count); + + this_addr = start_addr; + + for (i = 0; i < count; i++) + { + if (is_add) + snat_add_address (sm, &this_addr); + else + rv = snat_del_address (sm, this_addr); + + switch (rv) + { + case VNET_API_ERROR_NO_SUCH_ENTRY: + return clib_error_return (0, "S-NAT address not exist."); + break; + case VNET_API_ERROR_UNSPECIFIED: + return clib_error_return (0, "S-NAT address used in static mapping."); + break; + default: + break; + } + + increment_v4_address (&this_addr); + } + + return 0; +} + +VLIB_CLI_COMMAND (add_address_command, static) = { + .path = "snat add address", + .short_help = "snat add addresses [- ] [del]", + .function = add_address_command_fn, +}; + +static clib_error_t * +snat_feature_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + vnet_main_t * vnm = vnet_get_main(); + clib_error_t * error = 0; + u32 sw_if_index; + u32 * inside_sw_if_indices = 0; + u32 * outside_sw_if_indices = 0; + int is_del = 0; + int i; + + sw_if_index = ~0; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "in %U", unformat_vnet_sw_interface, + vnm, &sw_if_index)) + vec_add1 (inside_sw_if_indices, sw_if_index); + else if (unformat (line_input, "out %U", unformat_vnet_sw_interface, + vnm, &sw_if_index)) + vec_add1 (outside_sw_if_indices, sw_if_index); + else if (unformat (line_input, "del")) + is_del = 1; + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + } + unformat_free (line_input); + + if (vec_len (inside_sw_if_indices)) + { + for (i = 0; i < vec_len(inside_sw_if_indices); i++) + { + sw_if_index = inside_sw_if_indices[i]; + snat_interface_add_del (sw_if_index, 1, is_del); + } + } + + if (vec_len (outside_sw_if_indices)) + { + for (i = 0; i < vec_len(outside_sw_if_indices); i++) + { + sw_if_index = outside_sw_if_indices[i]; + snat_interface_add_del (sw_if_index, 0, is_del); + } + } + + vec_free (inside_sw_if_indices); + vec_free (outside_sw_if_indices); + + return error; +} + +VLIB_CLI_COMMAND (set_interface_snat_command, static) = { + .path = "set interface snat", + .function = snat_feature_command_fn, + .short_help = "set interface snat in out [del]", +}; + +static clib_error_t * +add_static_mapping_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t * error = 0; + ip4_address_t l_addr, e_addr; + u32 l_port = 0, e_port = 0, vrf_id = ~0; + int is_add = 1; + int addr_only = 1; + int rv; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "local %U %u", unformat_ip4_address, &l_addr, + &l_port)) + addr_only = 0; + else if (unformat (line_input, "local %U", unformat_ip4_address, &l_addr)) + ; + else if (unformat (line_input, "external %U %u", unformat_ip4_address, + &e_addr, &e_port)) + addr_only = 0; + else if (unformat (line_input, "external %U", unformat_ip4_address, + &e_addr)) + ; + else if (unformat (line_input, "vrf %u", &vrf_id)) + ; + else if (unformat (line_input, "del")) + is_add = 0; + else + return clib_error_return (0, "unknown input: '%U'", + format_unformat_error, line_input); + } + unformat_free (line_input); + + rv = snat_add_static_mapping(l_addr, e_addr, (u16) l_port, (u16) e_port, + vrf_id, addr_only, is_add); + + switch (rv) + { + case VNET_API_ERROR_INVALID_VALUE: + return clib_error_return (0, "External port already in use."); + break; + case VNET_API_ERROR_NO_SUCH_ENTRY: + if (is_add) + return clib_error_return (0, "External addres must be allocated."); + else + return clib_error_return (0, "Mapping not exist."); + break; + case VNET_API_ERROR_NO_SUCH_FIB: + return clib_error_return (0, "No such VRF id."); + case VNET_API_ERROR_VALUE_EXIST: + return clib_error_return (0, "Mapping already exist."); + default: + break; + } + + return error; +} + +/*? + * @cliexpar + * @cliexstart{snat add static mapping} + * Static mapping allows hosts on the external network to initiate connection + * to to the local network host. + * To create static mapping between local host address 10.0.0.3 port 6303 and + * external address 4.4.4.4 port 3606 use: + * vpp# snat add static mapping local 10.0.0.3 6303 external 4.4.4.4 3606 + * If not runnig "static mapping only" S-NAT plugin mode use before: + * vpp# snat add address 4.4.4.4 + * To create static mapping between local and external address use: + * vpp# snat add static mapping local 10.0.0.3 external 4.4.4.4 + * @cliexend +?*/ +VLIB_CLI_COMMAND (add_static_mapping_command, static) = { + .path = "snat add static mapping", + .function = add_static_mapping_command_fn, + .short_help = + "snat add static mapping local [] external [] [vrf ] [del]", +}; + +static clib_error_t * +set_workers_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + uword *bitmap = 0; + int rv = 0; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "%U", unformat_bitmap_list, &bitmap)) + ; + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + } + unformat_free (line_input); + + if (bitmap == 0) + return clib_error_return (0, "List of workers must be specified."); + + rv = snat_set_workers(bitmap); + + clib_bitmap_free (bitmap); + + switch (rv) + { + case VNET_API_ERROR_INVALID_WORKER: + return clib_error_return (0, "Invalid worker(s)."); + break; + case VNET_API_ERROR_FEATURE_DISABLED: + return clib_error_return (0, + "Supported only if 2 or more workes available."); + break; + default: + break; + } + + return 0; +} + +/*? + * @cliexpar + * @cliexstart{set snat workers} + * Set SNAT workers if 2 or more workers available, use: + * vpp# set snat workers 0-2,5 + * @cliexend +?*/ +VLIB_CLI_COMMAND (set_workers_command, static) = { + .path = "set snat workers", + .function = set_workers_command_fn, + .short_help = + "set snat workers ", +}; + +static clib_error_t * +snat_config (vlib_main_t * vm, unformat_input_t * input) +{ + snat_main_t * sm = &snat_main; + u32 translation_buckets = 1024; + u32 translation_memory_size = 128<<20; + u32 user_buckets = 128; + u32 user_memory_size = 64<<20; + u32 max_translations_per_user = 100; + u32 outside_vrf_id = 0; + u32 inside_vrf_id = 0; + u32 static_mapping_buckets = 1024; + u32 static_mapping_memory_size = 64<<20; + u8 static_mapping_only = 0; + u8 static_mapping_connection_tracking = 0; + vlib_thread_main_t *tm = vlib_get_thread_main (); + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "translation hash buckets %d", &translation_buckets)) + ; + else if (unformat (input, "translation hash memory %d", + &translation_memory_size)); + else if (unformat (input, "user hash buckets %d", &user_buckets)) + ; + else if (unformat (input, "user hash memory %d", + &user_memory_size)) + ; + else if (unformat (input, "max translations per user %d", + &max_translations_per_user)) + ; + else if (unformat (input, "outside VRF id %d", + &outside_vrf_id)) + ; + else if (unformat (input, "inside VRF id %d", + &inside_vrf_id)) + ; + else if (unformat (input, "static mapping only")) + { + static_mapping_only = 1; + if (unformat (input, "connection tracking")) + static_mapping_connection_tracking = 1; + } + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + } + + /* for show commands, etc. */ + sm->translation_buckets = translation_buckets; + sm->translation_memory_size = translation_memory_size; + sm->user_buckets = user_buckets; + sm->user_memory_size = user_memory_size; + sm->max_translations_per_user = max_translations_per_user; + sm->outside_vrf_id = outside_vrf_id; + sm->outside_fib_index = ~0; + sm->inside_vrf_id = inside_vrf_id; + sm->inside_fib_index = ~0; + sm->static_mapping_only = static_mapping_only; + sm->static_mapping_connection_tracking = static_mapping_connection_tracking; + + if (!static_mapping_only || + (static_mapping_only && static_mapping_connection_tracking)) + { + clib_bihash_init_8_8 (&sm->worker_by_in, "worker-by-in", user_buckets, + user_memory_size); + + clib_bihash_init_8_8 (&sm->worker_by_out, "worker-by-out", user_buckets, + user_memory_size); + + vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1); + + clib_bihash_init_8_8 (&sm->in2out, "in2out", translation_buckets, + translation_memory_size); + + clib_bihash_init_8_8 (&sm->out2in, "out2in", translation_buckets, + translation_memory_size); + + clib_bihash_init_8_8 (&sm->user_hash, "users", user_buckets, + user_memory_size); + } + clib_bihash_init_8_8 (&sm->static_mapping_by_local, + "static_mapping_by_local", static_mapping_buckets, + static_mapping_memory_size); + + clib_bihash_init_8_8 (&sm->static_mapping_by_external, + "static_mapping_by_external", static_mapping_buckets, + static_mapping_memory_size); + return 0; +} + +VLIB_CONFIG_FUNCTION (snat_config, "snat"); + +u8 * format_snat_key (u8 * s, va_list * args) +{ + snat_session_key_t * key = va_arg (*args, snat_session_key_t *); + char * protocol_string = "unknown"; + static char *protocol_strings[] = { + "UDP", + "TCP", + "ICMP", + }; + + if (key->protocol < ARRAY_LEN(protocol_strings)) + protocol_string = protocol_strings[key->protocol]; + + s = format (s, "%U proto %s port %d fib %d", + format_ip4_address, &key->addr, protocol_string, + clib_net_to_host_u16 (key->port), key->fib_index); + return s; +} + +u8 * format_snat_session (u8 * s, va_list * args) +{ + snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *); + snat_session_t * sess = va_arg (*args, snat_session_t *); + + s = format (s, " i2o %U\n", format_snat_key, &sess->in2out); + s = format (s, " o2i %U\n", format_snat_key, &sess->out2in); + s = format (s, " last heard %.2f\n", sess->last_heard); + s = format (s, " total pkts %d, total bytes %lld\n", + sess->total_pkts, sess->total_bytes); + if (snat_is_session_static (sess)) + s = format (s, " static translation\n"); + else + s = format (s, " dynamic translation\n"); + + return s; +} + +u8 * format_snat_user (u8 * s, va_list * args) +{ + snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *); + snat_user_t * u = va_arg (*args, snat_user_t *); + int verbose = va_arg (*args, int); + dlist_elt_t * head, * elt; + u32 elt_index, head_index; + u32 session_index; + snat_session_t * sess; + + s = format (s, "%U: %d dynamic translations, %d static translations\n", + format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions); + + if (verbose == 0) + return s; + + if (u->nsessions || u->nstaticsessions) + { + head_index = u->sessions_per_user_list_head_index; + head = pool_elt_at_index (sm->list_pool, head_index); + + elt_index = head->next; + elt = pool_elt_at_index (sm->list_pool, elt_index); + session_index = elt->value; + + while (session_index != ~0) + { + sess = pool_elt_at_index (sm->sessions, session_index); + + s = format (s, " %U\n", format_snat_session, sm, sess); + + elt_index = elt->next; + elt = pool_elt_at_index (sm->list_pool, elt_index); + session_index = elt->value; + } + } + + return s; +} + +u8 * format_snat_static_mapping (u8 * s, va_list * args) +{ + snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *); + + if (m->addr_only) + s = format (s, "local %U external %U vrf %d", + format_ip4_address, &m->local_addr, + format_ip4_address, &m->external_addr, + m->vrf_id); + else + s = format (s, "local %U:%d external %U:%d vrf %d", + format_ip4_address, &m->local_addr, m->local_port, + format_ip4_address, &m->external_addr, m->external_port, + m->vrf_id); + + return s; +} + +static clib_error_t * +show_snat_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + int verbose = 0; + snat_main_t * sm = &snat_main; + snat_user_t * u; + snat_static_mapping_t *m; + snat_interface_t *i; + snat_address_t * ap; + vnet_main_t *vnm = vnet_get_main(); + snat_main_per_thread_data_t *tsm; + u32 users_num = 0, sessions_num = 0, *worker; + uword j = 0; + + if (unformat (input, "detail")) + verbose = 1; + else if (unformat (input, "verbose")) + verbose = 2; + + if (sm->static_mapping_only) + { + if (sm->static_mapping_connection_tracking) + vlib_cli_output (vm, "SNAT mode: static mapping only connection " + "tracking"); + else + vlib_cli_output (vm, "SNAT mode: static mapping only"); + } + else + { + vlib_cli_output (vm, "SNAT mode: dynamic translations enabled"); + } + + if (verbose > 0) + { + pool_foreach (i, sm->interfaces, + ({ + vlib_cli_output (vm, "%U %s", format_vnet_sw_interface_name, vnm, + vnet_get_sw_interface (vnm, i->sw_if_index), + i->is_inside ? "in" : "out"); + })); + + vec_foreach (ap, sm->addresses) + { + u8 * s = format (0, ""); + vlib_cli_output (vm, "%U", format_ip4_address, &ap->addr); + clib_bitmap_foreach (j, ap->busy_port_bitmap, + ({ + s = format (s, " %d", j); + })); + vlib_cli_output (vm, " %d busy ports:%v", ap->busy_ports, s); + } + } + + if (sm->num_workers > 1) + { + vlib_cli_output (vm, "%d workers", vec_len (sm->workers)); + if (verbose > 0) + { + vec_foreach (worker, sm->workers) + { + vlib_worker_thread_t *w = + vlib_worker_threads + *worker + sm->first_worker_index; + vlib_cli_output (vm, " %v", w->name); + } + } + } + + if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)) + { + vlib_cli_output (vm, "%d static mappings", + pool_elts (sm->static_mappings)); + + if (verbose > 0) + { + pool_foreach (m, sm->static_mappings, + ({ + vlib_cli_output (vm, "%U", format_snat_static_mapping, m); + })); + } + } + else + { + vec_foreach (tsm, sm->per_thread_data) + { + users_num += pool_elts (tsm->users); + sessions_num += pool_elts (tsm->sessions); + } + + vlib_cli_output (vm, "%d users, %d outside addresses, %d active sessions," + " %d static mappings", + users_num, + vec_len (sm->addresses), + sessions_num, + pool_elts (sm->static_mappings)); + + if (verbose > 0) + { + vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->in2out, + verbose - 1); + vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->out2in, + verbose - 1); + vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_in, + verbose - 1); + vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_out, + verbose - 1); + vec_foreach_index (j, sm->per_thread_data) + { + tsm = vec_elt_at_index (sm->per_thread_data, j); + + if (pool_elts (tsm->users) == 0) + continue; + + vlib_worker_thread_t *w = vlib_worker_threads + j; + vlib_cli_output (vm, "Thread %d (%v at lcore %u):", j, w->name, + w->lcore_id); + vlib_cli_output (vm, " %d list pool elements", + pool_elts (tsm->list_pool)); + + pool_foreach (u, tsm->users, + ({ + vlib_cli_output (vm, " %U", format_snat_user, tsm, u, + verbose - 1); + })); + } + + if (pool_elts (sm->static_mappings)) + { + vlib_cli_output (vm, "static mappings:"); + pool_foreach (m, sm->static_mappings, + ({ + vlib_cli_output (vm, "%U", format_snat_static_mapping, m); + })); + } + } + } + + return 0; +} + +VLIB_CLI_COMMAND (show_snat_command, static) = { + .path = "show snat", + .short_help = "show snat", + .function = show_snat_command_fn, +}; diff --git a/src/plugins/snat/snat.h b/src/plugins/snat/snat.h new file mode 100644 index 00000000..cb31dc51 --- /dev/null +++ b/src/plugins/snat/snat.h @@ -0,0 +1,259 @@ + +/* + * snat.h - simple nat definitions + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_snat_h__ +#define __included_snat_h__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Key */ +typedef struct { + union + { + struct + { + ip4_address_t addr; + u16 port; + u16 protocol:3, + fib_index:13; + }; + u64 as_u64; + }; +} snat_session_key_t; + +typedef struct { + union + { + struct + { + ip4_address_t addr; + u32 fib_index; + }; + u64 as_u64; + }; +} snat_user_key_t; + +typedef struct { + union + { + struct + { + ip4_address_t addr; + u16 port; + u16 fib_index; + }; + u64 as_u64; + }; +} snat_static_mapping_key_t; + + +typedef enum { + SNAT_PROTOCOL_UDP = 0, + SNAT_PROTOCOL_TCP, + SNAT_PROTOCOL_ICMP, +} snat_protocol_t; + + +#define SNAT_SESSION_FLAG_STATIC_MAPPING 1 + +typedef CLIB_PACKED(struct { + snat_session_key_t out2in; /* 0-15 */ + + snat_session_key_t in2out; /* 16-31 */ + + u32 flags; /* 32-35 */ + + /* per-user translations */ + u32 per_user_index; /* 36-39 */ + + u32 per_user_list_head_index; /* 40-43 */ + + /* Last heard timer */ + f64 last_heard; /* 44-51 */ + + u64 total_bytes; /* 52-59 */ + + u32 total_pkts; /* 60-63 */ + + /* Outside address */ + u32 outside_address_index; /* 64-67 */ + +}) snat_session_t; + + +typedef struct { + ip4_address_t addr; + u32 sessions_per_user_list_head_index; + u32 nsessions; + u32 nstaticsessions; +} snat_user_t; + +typedef struct { + ip4_address_t addr; + u32 busy_ports; + uword * busy_port_bitmap; +} snat_address_t; + +typedef struct { + ip4_address_t local_addr; + ip4_address_t external_addr; + u16 local_port; + u16 external_port; + u8 addr_only; + u32 vrf_id; + u32 fib_index; +} snat_static_mapping_t; + +typedef struct { + u32 sw_if_index; + u8 is_inside; +} snat_interface_t; + +typedef struct { + /* User pool */ + snat_user_t * users; + + /* Session pool */ + snat_session_t * sessions; + + /* Pool of doubly-linked list elements */ + dlist_elt_t * list_pool; +} snat_main_per_thread_data_t; + +typedef struct { + /* Main lookup tables */ + clib_bihash_8_8_t out2in; + clib_bihash_8_8_t in2out; + + /* Find-a-user => src address lookup */ + clib_bihash_8_8_t user_hash; + + /* Non-translated packets worker lookup => src address + VRF */ + clib_bihash_8_8_t worker_by_in; + + /* Translated packets worker lookup => IP address + port number */ + clib_bihash_8_8_t worker_by_out; + + u32 num_workers; + u32 first_worker_index; + u32 next_worker; + u32 * workers; + + /* Per thread data */ + snat_main_per_thread_data_t * per_thread_data; + + /* Find a static mapping by local */ + clib_bihash_8_8_t static_mapping_by_local; + + /* Find a static mapping by external */ + clib_bihash_8_8_t static_mapping_by_external; + + /* Static mapping pool */ + snat_static_mapping_t * static_mappings; + + /* Interface pool */ + snat_interface_t * interfaces; + + /* Vector of outside addresses */ + snat_address_t * addresses; + + /* Randomize port allocation order */ + u32 random_seed; + + /* Worker handoff index */ + u32 fq_in2out_index; + u32 fq_out2in_index; + + /* Config parameters */ + u8 static_mapping_only; + u8 static_mapping_connection_tracking; + u32 translation_buckets; + u32 translation_memory_size; + u32 user_buckets; + u32 user_memory_size; + u32 max_translations_per_user; + u32 outside_vrf_id; + u32 outside_fib_index; + u32 inside_vrf_id; + u32 inside_fib_index; + + /* API message ID base */ + u16 msg_id_base; + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; + ip4_main_t * ip4_main; + ip_lookup_main_t * ip4_lookup_main; + ethernet_main_t * ethernet_main; + api_main_t * api_main; +} snat_main_t; + +extern snat_main_t snat_main; +extern vlib_node_registration_t snat_in2out_node; +extern vlib_node_registration_t snat_out2in_node; +extern vlib_node_registration_t snat_in2out_fast_node; +extern vlib_node_registration_t snat_out2in_fast_node; +extern vlib_node_registration_t snat_in2out_worker_handoff_node; +extern vlib_node_registration_t snat_out2in_worker_handoff_node; + +void snat_free_outside_address_and_port (snat_main_t * sm, + snat_session_key_t * k, + u32 address_index); + +int snat_alloc_outside_address_and_port (snat_main_t * sm, + snat_session_key_t * k, + u32 * address_indexp); + +int snat_static_mapping_match (snat_main_t * sm, + snat_session_key_t match, + snat_session_key_t * mapping, + u8 by_external); + +format_function_t format_snat_user; + +typedef struct { + u32 cached_sw_if_index; + u32 cached_ip4_address; +} snat_runtime_t; + +/** \brief Check if SNAT session is created from static mapping. + @param s SNAT session + @return 1 if SNAT session is created from static mapping otherwise 0 +*/ +#define snat_is_session_static(s) s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING + +/* + * Why is this here? Because we don't need to touch this layer to + * simply reply to an icmp. We need to change id to a unique + * value to NAT an echo request/reply. + */ + +typedef struct { + u16 identifier; + u16 sequence; +} icmp_echo_header_t; + +#endif /* __included_snat_h__ */ diff --git a/src/plugins/snat/snat_all_api_h.h b/src/plugins/snat/snat_all_api_h.h new file mode 100644 index 00000000..49017700 --- /dev/null +++ b/src/plugins/snat/snat_all_api_h.h @@ -0,0 +1,19 @@ + +/* + * snat_all_api_h.h - skeleton vpp engine plug-in api #include file + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include diff --git a/src/plugins/snat/snat_msg_enum.h b/src/plugins/snat/snat_msg_enum.h new file mode 100644 index 00000000..2c76fd51 --- /dev/null +++ b/src/plugins/snat/snat_msg_enum.h @@ -0,0 +1,31 @@ + +/* + * snat_msg_enum.h - skeleton vpp engine plug-in message enumeration + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_snat_msg_enum_h +#define included_snat_msg_enum_h + +#include + +#define vl_msg_id(n,h) n, +typedef enum { +#include + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_snat_msg_enum_h */ diff --git a/src/plugins/snat/snat_test.c b/src/plugins/snat/snat_test.c new file mode 100644 index 00000000..2a003ba6 --- /dev/null +++ b/src/plugins/snat/snat_test.c @@ -0,0 +1,602 @@ + +/* + * snat.c - skeleton vpp-api-test plug-in + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +uword unformat_sw_if_index (unformat_input_t * input, va_list * args); + +/* Declare message IDs */ +#include + +/* define message structures */ +#define vl_typedefs +#include +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +typedef struct { + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} snat_test_main_t; + +snat_test_main_t snat_test_main; + +#define foreach_standard_reply_retval_handler \ +_(snat_add_address_range_reply) \ +_(snat_interface_add_del_feature_reply) \ +_(snat_add_static_mapping_reply) \ +_(snat_set_workers_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = snat_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(SNAT_ADD_ADDRESS_RANGE_REPLY, snat_add_address_range_reply) \ +_(SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY, \ + snat_interface_add_del_feature_reply) \ +_(SNAT_ADD_STATIC_MAPPING_REPLY, snat_add_static_mapping_reply) \ +_(SNAT_CONTROL_PING_REPLY, snat_control_ping_reply) \ +_(SNAT_STATIC_MAPPING_DETAILS, snat_static_mapping_details) \ +_(SNAT_SHOW_CONFIG_REPLY, snat_show_config_reply) \ +_(SNAT_ADDRESS_DETAILS, snat_address_details) \ +_(SNAT_INTERFACE_DETAILS, snat_interface_details) \ +_(SNAT_SET_WORKERS_REPLY, snat_set_workers_reply) \ +_(SNAT_WORKER_DETAILS, snat_worker_details) + +/* M: construct, but don't yet send a message */ +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int api_snat_add_address_range (vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + ip4_address_t start_addr, end_addr; + u32 start_host_order, end_host_order; + vl_api_snat_add_address_range_t * mp; + u8 is_add = 1; + int count; + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "%U - %U", + unformat_ip4_address, &start_addr, + unformat_ip4_address, &end_addr)) + ; + else if (unformat (i, "%U", unformat_ip4_address, &start_addr)) + end_addr = start_addr; + else if (unformat (i, "del")) + is_add = 0; + else + { + clib_warning("unknown input '%U'", format_unformat_error, i); + return -99; + } + } + + start_host_order = clib_host_to_net_u32 (start_addr.as_u32); + end_host_order = clib_host_to_net_u32 (end_addr.as_u32); + + if (end_host_order < start_host_order) + { + errmsg ("end address less than start address\n"); + return -99; + } + + count = (end_host_order - start_host_order) + 1; + + if (count > 1024) + { + errmsg ("%U - %U, %d addresses...\n", + format_ip4_address, &start_addr, + format_ip4_address, &end_addr, + count); + } + + M(SNAT_ADD_ADDRESS_RANGE, snat_add_address_range); + + memcpy (mp->first_ip_address, &start_addr, 4); + memcpy (mp->last_ip_address, &end_addr, 4); + mp->is_ip4 = 1; + mp->is_add = is_add; + + S; W; + + /* NOTREACHED */ + return 0; +} + +static int api_snat_interface_add_del_feature (vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_snat_interface_add_del_feature_t * mp; + u32 sw_if_index; + u8 sw_if_index_set = 0; + u8 is_inside = 1; + u8 is_add = 1; + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + sw_if_index_set = 1; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + sw_if_index_set = 1; + else if (unformat (i, "out")) + is_inside = 0; + else if (unformat (i, "in")) + is_inside = 1; + else if (unformat (i, "del")) + is_add = 0; + else + { + clib_warning("unknown input '%U'", format_unformat_error, i); + return -99; + } + } + + if (sw_if_index_set == 0) + { + errmsg ("interface / sw_if_index required\n"); + return -99; + } + + M(SNAT_INTERFACE_ADD_DEL_FEATURE, snat_interface_add_del_feature); + mp->sw_if_index = ntohl(sw_if_index); + mp->is_add = is_add; + mp->is_inside = is_inside; + + S; W; + /* NOTREACHED */ + return 0; +} + +static int api_snat_add_static_mapping(vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_snat_add_static_mapping_t * mp; + u8 addr_set_n = 0; + u8 is_add = 1; + u8 addr_only = 1; + ip4_address_t local_addr, external_addr; + u32 local_port = 0, external_port = 0, vrf_id = ~0; + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "local_addr %U", unformat_ip4_address, &local_addr)) + addr_set_n++; + else if (unformat (i, "external_addr %U", unformat_ip4_address, + &external_addr)) + addr_set_n++; + else if (unformat (i, "local_port %u", &local_port)) + addr_only = 0; + else if (unformat (i, "external_port %u", &external_port)) + addr_only = 0; + else if (unformat (i, "vrf %u", &vrf_id)) + ; + else if (unformat (i, "del")) + is_add = 0; + else + { + clib_warning("unknown input '%U'", format_unformat_error, i); + return -99; + } + } + + if (addr_set_n != 2) + { + errmsg ("local_addr and remote_addr required\n"); + return -99; + } + + M(SNAT_ADD_STATIC_MAPPING, snat_add_static_mapping); + mp->is_add = is_add; + mp->is_ip4 = 1; + mp->addr_only = addr_only; + mp->local_port = ntohs ((u16) local_port); + mp->external_port = ntohs ((u16) external_port); + mp->vrf_id = ntohl (vrf_id); + memcpy (mp->local_ip_address, &local_addr, 4); + memcpy (mp->external_ip_address, &external_addr, 4); + + S; W; + /* NOTREACHED */ + return 0; +} + +static void vl_api_snat_control_ping_reply_t_handler + (vl_api_snat_control_ping_reply_t * mp) +{ + vat_main_t *vam = &vat_main; + i32 retval = ntohl (mp->retval); + if (vam->async_mode) + { + vam->async_errors += (retval < 0); + } + else + { + vam->retval = retval; + vam->result_ready = 1; + } +} + +static void vl_api_snat_static_mapping_details_t_handler + (vl_api_snat_static_mapping_details_t *mp) +{ + snat_test_main_t * sm = &snat_test_main; + vat_main_t *vam = sm->vat_main; + + if (mp->addr_only) + fformat (vam->ofp, "%15U%6s%15U%6s%11d\n", + format_ip4_address, &mp->local_ip_address, "", + format_ip4_address, &mp->external_ip_address, "", + ntohl (mp->vrf_id)); + else + fformat (vam->ofp, "%15U%6d%15U%6d%11d\n", + format_ip4_address, &mp->local_ip_address, + ntohs (mp->local_port), + format_ip4_address, &mp->external_ip_address, + ntohs (mp->external_port), + ntohl (mp->vrf_id)); + +} + +static int api_snat_static_mapping_dump(vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + f64 timeout; + vl_api_snat_static_mapping_dump_t * mp; + + if (vam->json_output) + { + clib_warning ("JSON output not supported for snat_static_mapping_dump"); + return -99; + } + + fformat (vam->ofp, "%21s%21s\n", "local", "external"); + fformat (vam->ofp, "%15s%6s%15s%6s%11s\n", "address", "port", "address", + "port", "vrf"); + + M(SNAT_STATIC_MAPPING_DUMP, snat_static_mapping_dump); + S; + /* Use a control ping for synchronization */ + { + vl_api_snat_control_ping_t *mp; + M (SNAT_CONTROL_PING, snat_control_ping); + S; + } + W; + /* NOTREACHED */ + return 0; +} + +static void vl_api_snat_show_config_reply_t_handler + (vl_api_snat_show_config_reply_t *mp) +{ + snat_test_main_t * sm = &snat_test_main; + vat_main_t *vam = sm->vat_main; + i32 retval = ntohl (mp->retval); + + if (retval >= 0) + { + fformat (vam->ofp, "translation hash buckets %d\n", + ntohl (mp->translation_buckets)); + fformat (vam->ofp, "translation hash memory %d\n", + ntohl (mp->translation_memory_size)); + fformat (vam->ofp, "user hash buckets %d\n", ntohl (mp->user_buckets)); + fformat (vam->ofp, "user hash memory %d\n", ntohl (mp->user_memory_size)); + fformat (vam->ofp, "max translations per user %d\n", + ntohl (mp->max_translations_per_user)); + fformat (vam->ofp, "outside VRF id %d\n", ntohl (mp->outside_vrf_id)); + fformat (vam->ofp, "inside VRF id %d\n", ntohl (mp->inside_vrf_id)); + if (mp->static_mapping_only) + { + fformat (vam->ofp, "static mapping only"); + if (mp->static_mapping_connection_tracking) + fformat (vam->ofp, " connection tracking"); + fformat (vam->ofp, "\n"); + } + } + vam->retval = retval; + vam->result_ready = 1; +} + +static int api_snat_show_config(vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + f64 timeout; + vl_api_snat_show_config_t * mp; + + if (vam->json_output) + { + clib_warning ("JSON output not supported for snat_show_config"); + return -99; + } + + M(SNAT_SHOW_CONFIG, snat_show_config); + S; W; + /* NOTREACHED */ + return 0; +} + +static void vl_api_snat_address_details_t_handler + (vl_api_snat_address_details_t *mp) +{ + snat_test_main_t * sm = &snat_test_main; + vat_main_t *vam = sm->vat_main; + + fformat (vam->ofp, "%U\n", format_ip4_address, &mp->ip_address); +} + +static int api_snat_address_dump(vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + f64 timeout; + vl_api_snat_address_dump_t * mp; + + if (vam->json_output) + { + clib_warning ("JSON output not supported for snat_address_dump"); + return -99; + } + + M(SNAT_ADDRESS_DUMP, snat_address_dump); + S; + /* Use a control ping for synchronization */ + { + vl_api_snat_control_ping_t *mp; + M (SNAT_CONTROL_PING, snat_control_ping); + S; + } + W; + /* NOTREACHED */ + return 0; +} + +static void vl_api_snat_interface_details_t_handler + (vl_api_snat_interface_details_t *mp) +{ + snat_test_main_t * sm = &snat_test_main; + vat_main_t *vam = sm->vat_main; + + fformat (vam->ofp, "sw_if_index %d %s\n", ntohl (mp->sw_if_index), + mp->is_inside ? "in" : "out"); +} + +static int api_snat_interface_dump(vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + f64 timeout; + vl_api_snat_interface_dump_t * mp; + + if (vam->json_output) + { + clib_warning ("JSON output not supported for snat_address_dump"); + return -99; + } + + M(SNAT_INTERFACE_DUMP, snat_interface_dump); + S; + /* Use a control ping for synchronization */ + { + vl_api_snat_control_ping_t *mp; + M (SNAT_CONTROL_PING, snat_control_ping); + S; + } + W; + /* NOTREACHED */ + return 0; +} + +static int api_snat_set_workers (vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + unformat_input_t * i = vam->input; + f64 timeout; + vl_api_snat_set_workers_t * mp; + uword *bitmap; + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "%U", unformat_bitmap_list, &bitmap)) + ; + else + { + clib_warning("unknown input '%U'", format_unformat_error, i); + return -99; + } + } + + M(SNAT_SET_WORKERS, snat_set_workers); + mp->worker_mask = clib_host_to_net_u64 (bitmap[0]); + + S; W; + + /* NOTREACHED */ + return 0; +} + +static void vl_api_snat_worker_details_t_handler + (vl_api_snat_worker_details_t *mp) +{ + snat_test_main_t * sm = &snat_test_main; + vat_main_t *vam = sm->vat_main; + + fformat (vam->ofp, "worker_index %d (%s at lcore %u)\n", + ntohl (mp->worker_index), mp->name, ntohl (mp->lcore_id)); +} + +static int api_snat_worker_dump(vat_main_t * vam) +{ + snat_test_main_t * sm = &snat_test_main; + f64 timeout; + vl_api_snat_worker_dump_t * mp; + + if (vam->json_output) + { + clib_warning ("JSON output not supported for snat_address_dump"); + return -99; + } + + M(SNAT_WORKER_DUMP, snat_worker_dump); + S; + /* Use a control ping for synchronization */ + { + vl_api_snat_control_ping_t *mp; + M (SNAT_CONTROL_PING, snat_control_ping); + S; + } + W; + /* NOTREACHED */ + return 0; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(snat_add_address_range, " [- | sw_if_index [in] [out] [del]") \ +_(snat_add_static_mapping, "local_addr external_addr " \ + "[local_port ] [external_port ] [vrf ] [del]") \ +_(snat_set_workers, "") \ +_(snat_static_mapping_dump, "") \ +_(snat_show_config, "") \ +_(snat_address_dump, "") \ +_(snat_interface_dump, "") \ +_(snat_worker_dump, "") + +void vat_api_hookup (vat_main_t *vam) +{ + snat_test_main_t * sm __attribute__((unused)) = &snat_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * vat_plugin_register (vat_main_t *vam) +{ + snat_test_main_t * sm = &snat_test_main; + u8 * name; + + sm->vat_main = vam; + + /* Ask the vpp engine for the first assigned message-id */ + name = format (0, "snat_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (sm->msg_id_base != (u16) ~0) + vat_api_hookup (vam); + + vec_free(name); + + return 0; +} diff --git a/src/suffix-rules.mk b/src/suffix-rules.mk index e3eeb922..f97b2276 100644 --- a/src/suffix-rules.mk +++ b/src/suffix-rules.mk @@ -18,10 +18,10 @@ @echo " APIGEN " $@ ; \ mkdir -p `dirname $@` ; \ $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --output $@ --show-name $@ + | vppapigen --input - --output $@ --show-name $@ > /dev/null %.api.json: %.api - @echo " JSON APIGEN " $@ ; \ + @echo " JSON API" $@ ; \ mkdir -p `dirname $@` ; \ $(CC) $(CPPFLAGS) -E -P -C -x c $^ \ - | vppapigen --input - --json $@ + | vppapigen --input - --json $@ > /dev/null diff --git a/src/tools/g2/configure.ac b/src/tools/g2/configure.ac deleted file mode 100644 index c8af7747..00000000 --- a/src/tools/g2/configure.ac +++ /dev/null @@ -1,12 +0,0 @@ -AC_INIT(g2, 3.0) -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) - -AC_CHECK_LIB([vppinfra], [clib_mem_get_page_size],, - AC_MSG_ERROR([Please install the vpp-lib package])) -AC_CHECK_HEADER([vppinfra/clib.h],, - AC_MSG_ERROR([Please install the vpp-dev package])) - -PKG_CHECK_MODULES(g2, gtk+-2.0) - -AC_OUTPUT([Makefile]) diff --git a/src/tools/perftool/configure.ac b/src/tools/perftool/configure.ac deleted file mode 100644 index f4a98697..00000000 --- a/src/tools/perftool/configure.ac +++ /dev/null @@ -1,12 +0,0 @@ -AC_INIT(perftool, 2.0) -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) - -AC_CHECK_LIB([vppinfra], [clib_mem_get_page_size],, - AC_MSG_ERROR([Please install the vpp-lib package])) -AC_CHECK_HEADER([vppinfra/clib.h],, - AC_MSG_ERROR([Please install the vpp-dev package])) - -AM_PROG_LIBTOOL - -AC_OUTPUT([Makefile]) diff --git a/src/tools/vppapigen/configure.ac b/src/tools/vppapigen/configure.ac deleted file mode 100644 index 16ad59d2..00000000 --- a/src/tools/vppapigen/configure.ac +++ /dev/null @@ -1,14 +0,0 @@ -# -*- Autoconf -*- -# Copyright (c) 2008 by cisco Systems, Inc. -# All rights reserved. -# Process this file with autoconf to produce a configure script. - -AC_INIT(vppapigen, 1.0) -AM_INIT_AUTOMAKE -AM_SILENT_RULES([yes]) - -# Checks for programs. -AC_PROG_CC -AC_PROG_YACC - -AC_OUTPUT([Makefile]) diff --git a/src/vpp-api/java/Makefile.am b/src/vpp-api/java/Makefile.am new file mode 100644 index 00000000..c7051ec1 --- /dev/null +++ b/src/vpp-api/java/Makefile.am @@ -0,0 +1,199 @@ +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AUTOMAKE_OPTIONS = foreign +ACLOCAL_AMFLAGS = -I m4 + +AM_CFLAGS = -Wall -I${top_srcdir} -I${top_builddir} \ + -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux \ + -I@top_srcdir@/plugins -I@top_builddir@/plugins + +AM_LDFLAGS = -module -shared -avoid-version -rpath /none -no-undefined + +BUILT_SOURCES = +bin_PROGRAMS = +noinst_LTLIBRARIES = +JAR_FILES = +CLEANDIRS = + +# +# jvpp-common +# + +noinst_LTLIBRARIES += libjvpp_common.la +libjvpp_common_la_SOURCES = jvpp-common/jvpp_common.c +libjvpp_common_la_LDFLAGS = + +# +# jvpp-registry (connection management + plugin registry) +# + +noinst_LTLIBRARIES += libjvpp_registry.la + +libjvpp_registry_la_SOURCES = jvpp-registry/jvpp_registry.c +libjvpp_registry_la_CPPFLAGS = -Ijvpp-registry +libjvpp_registry_la_LIBAD = libjvpp_common.la + +packagedir_jvpp_registry = io/fd/vpp/jvpp +jvpp_registry_src_files := \ + $(wildcard @srcdir@/jvpp-registry/$(packagedir_jvpp_registry)/*.java) \ + $(wildcard @srcdir@/jvpp-registry/$(packagedir_jvpp_registry)/**/*.java) + +BUILT_SOURCES += jvpp-registry/io_fd_vpp_jvpp_VppJNIConnection.h +CLEANDIRS += jvpp-registry/target +JAR_FILES += jvpp-registry-$(PACKAGE_VERSION).jar + +jvpp_registry_ok = jvpp-registry/io_fd_vpp_jvpp_VppJNIConnection.h + +jvpp-registry/io_fd_vpp_jvpp_VppJNIConnection.h: $(jvpp_registry_src_files) + @echo " JAPIGEN $@" + @rm -rf jvpp-registry/target + @mkdir -p jvpp-registry/target + @$(JAVAC) -d jvpp-registry/target $^ + @$(JAVAH) -force -classpath jvpp-registry/target -d jvpp-registry io.fd.vpp.jvpp.VppJNIConnection + @$(JAVAH) -force -classpath jvpp-registry/target -d jvpp-registry io.fd.vpp.jvpp.JVppRegistryImpl + @touch jvpp-registry.ok + +define japigen + @echo " JAPIGEN $@" + @rm -rf jvpp-$(1)/target + @ @srcdir@/jvpp/gen/jvpp_gen.py --plugin_name $(1) --root_dir jvpp-$(1) \ + -i $(jvpp_$(1)_json_files) > /dev/null + @find jvpp-$(1)/target -name \*.java > jvpp-$(1).files + @$(JAVAC) -classpath jvpp-registry/target \ + -d jvpp-$(1)/target @jvpp-$(1).files + @$(JAVAH) -force \ + -classpath jvpp-registry/target:jvpp-$(1)/target \ + -d jvpp-$(1) io.fd.vpp.jvpp.$(1).$(2) +endef + +# +# jvpp-core (Java wrapper for vpe.api) +# +noinst_LTLIBRARIES += libjvpp_core.la +libjvpp_core_la_SOURCES = jvpp-core/jvpp_core.c jvpp-core/jvpp_core_gen.h +libjvpp_core_la_CPPFLAGS = -Ijvpp-registry -Ijvpp-core +BUILT_SOURCES += jvpp-core/io_fd_vpp_jvpp_core_JVppCoreImpl.h + +JAR_FILES += jvpp-core-$(PACKAGE_VERSION).jar +CLEANDIRS += jvpp-core/target +jvpp_core_json_files = $(shell find @top_builddir@/vnet/ -type f -name '*.api.json') + +jvpp-core/io_fd_vpp_jvpp_core_JVppCoreImpl.h: $(jvpp_registry_ok) $(jvpp_core_json_files) + $(call japigen,core,JVppCoreImpl) + +# +# ACL Plugin +# +if ENABLE_ACL_PLUGIN +noinst_LTLIBRARIES += libjvpp_acl.la +libjvpp_acl_la_SOURCES = jvpp-acl/jvpp_acl.c +libjvpp_acl_la_CPPFLAGS = -Ijvpp-acl + +BUILT_SOURCES += jvpp-acl/io_fd_vpp_jvpp_acl_JVppAclImpl.h + +JAR_FILES += jvpp-acl-$(PACKAGE_VERSION).jar +CLEANDIRS += jvpp-acl/target + +jvpp_acl_json_files = @top_builddir@/plugins/acl/acl.api.json + +jvpp-acl/io_fd_vpp_jvpp_acl_JVppAclImpl.h: $(jvpp_registry_ok) $(jvpp_acl_json_files) + $(call japigen,acl,JVppAclImpl) +endif + +# +# SNAT Plugin +# +if ENABLE_SNAT_PLUGIN +noinst_LTLIBRARIES += libjvpp_snat.la +libjvpp_snat_la_SOURCES = jvpp-snat/jvpp_snat.c +libjvpp_snat_la_CPPFLAGS = -Ijvpp-snat + +BUILT_SOURCES += jvpp-snat/io_fd_vpp_jvpp_snat_JVppSnatImpl.h + +JAR_FILES += jvpp-snat-$(PACKAGE_VERSION).jar +CLEANDIRS += jvpp-snat/target + +jvpp_snat_json_files = @top_builddir@/plugins/snat/snat.api.json + +jvpp-snat/io_fd_vpp_jvpp_snat_JVppSnatImpl.h: $(jvpp_registry_ok) $(jvpp_snat_json_files) + $(call japigen,snat,JVppSnatImpl) +endif + +# +# iOAM Trace Plugin +# +if ENABLE_IOAM_PLUGIN +noinst_LTLIBRARIES += libjvpp_ioamtrace.la +libjvpp_ioamtrace_la_SOURCES = jvpp-ioamtrace/jvpp_ioam_trace.c + +BUILT_SOURCES += jvpp-ioamtrace/io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl.h +JAR_FILES += jvpp-ioamtrace-$(PACKAGE_VERSION).jar +CLEANDIRS += jvpp-ioamtrace/target + +jvpp_ioamtrace_json_files = @top_builddir@/plugins/ioam/lib-trace/trace.api.json + +jvpp-ioamtrace/io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl.h: $(jvpp_registry_ok) $(jvpp_ioamtrace_json_files) + $(call japigen,ioamtrace,JVppIoamtraceImpl) + +# +# iOAM POT Plugin +# +noinst_LTLIBRARIES += libjvpp_ioampot.la +libjvpp_ioampot_la_SOURCES = jvpp-ioampot/jvpp_ioam_pot.c + +BUILT_SOURCES += jvpp-ioampot/io_fd_vpp_jvpp_ioampot_JVppIoampotImpl.h +JAR_FILES += jvpp-ioampot-$(PACKAGE_VERSION).jar +CLEANDIRS += jvpp-ioampot/target + +jvpp_ioampot_json_files = @top_builddir@/plugins/ioam/lib-pot/pot.api.json + +jvpp-ioampot/io_fd_vpp_jvpp_ioampot_JVppIoampotImpl.h: $(jvpp_registry_ok) $(jvpp_ioampot_json_files) + $(call japigen,ioampot,JVppIoampotImpl) + +# +# iOAM Export Plugin +# +noinst_LTLIBRARIES += libjvpp_ioamexport.la +libjvpp_ioamexport_la_SOURCES = jvpp-ioamexport/jvpp_ioam_export.c + +BUILT_SOURCES += jvpp-ioamexport/io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl.h +JAR_FILES += jvpp-ioamexport-$(PACKAGE_VERSION).jar +CLEANDIRS += jvpp-ioamexport/target + +jvpp_ioamexport_json_files = @top_builddir@/plugins/ioam/export/ioam_export.api.json + +jvpp-ioamexport/io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl.h: $(jvpp_registry_ok) $(jvpp_ioamexport_json_files) + $(call japigen,ioamexport,JVppIoamexportImpl) +endif + +# +# JAR creation +# +jvpp-%-$(PACKAGE_VERSION).jar: libjvpp_%.la + @echo " JAR $@" + @cp .libs/libjvpp_$*.so jvpp-$*/target + @$(JAR) cf $(JARFLAGS) $@ -C jvpp-$*/target . + +jardir = $(prefix)/share/java +jar_DATA = $(JAR_FILES) + +all-local: $(JAR_FILES) + +# +# Cleanup +# +CLEANFILES = jvpp-registry.ok $(JAR_FILES) $(BUILT_SOURCES) + +clean-local: + rm -rf $(CLEANDIRS) diff --git a/src/vpp-api/java/Readme.txt b/src/vpp-api/java/Readme.txt new file mode 100644 index 00000000..689b9b37 --- /dev/null +++ b/src/vpp-api/java/Readme.txt @@ -0,0 +1,236 @@ += JVpp + +JVpp is JNI based Java API for VPP. + +== Features +It is: + +* Asynchronous +* Fully generated +* Lightweight + +== Architecture + +=== Plugin support + + /-------------\ /--------------\ /---------------\ + | JvppPlugin1 +<-------+ JVppRegistry +--------->+ VppConnection | + \-------------/ inits \--+-----------/ uses \---------------/ + | + /-------------\ | + | JvppPlugin2 +<----------+ inits + \-------------/ | + | + ... | + | + /----------\ | + | JVppCore +<-------------+ + \----------/ + + +VppRegistry opens connection to vpp (VppConnection) and manages jvpp plugins. +Each plugin needs to be registered in the VppRegistry. Registration involves +plugin initialization (providing JNI implementation with JVppCallback reference, +vpp client identifier and vpp shared memory queue address). + +API user sends message by calling a method of appropriate plugin interface. +The call is delegated to JNI implementation provided by the particular plugin. +When JNI code receives reply, it invokes callback method of JVppCallback +that corresponds to the received message reply. + +=== JVppCore as an example of JVpp plugin architecture + + JVpp Java + + /--------------\ /----------\ /------------\ /------\ + | JVppRegistry | | JVppCore | | Callbacks | | DTOs | + \----+---------/ \----+-----/ \------+-----/ \------/ + ^ ^ ^ + | implements | implements | implements + /----+--------------\ /---+----------\ /-----+---------\ + | JVppRegistryImpl* +-------->+ JVppCoreImpl | | JVppCallback | + \-------+-----------/ inits \---+----------/ \-------+-------/ + | | ^ + | | uses | calls back + | | | +----------|--------------------------|-----------------------|--------------------- + | | | + C JNI | +-------------------+ | /-----------------\ + v | | +-->+ jvpp_core_gen.h | + /--------+--------\ | | | \-----------------/ + | jpp_registry.c* +---+ /--------+----+----\ | | | + \-----------------/ | | << shared lib >> | /-+--+---+------\ + + ->+ jvpp_common* <--------+ jvpp_core.c* | + uses \------------------/ uses \---------------/ + + +* Components marked with an asterisk contain manually crafted code, which in addition +to generated classes form jvpp. Exception applies to Callbacks and DTOs, since there are +manually crafted marker interfaces in callback and dto package (dto/JVppRequest, dto/JVppReply, +dto/JVppDump, dto/JVppReplyDump, callback/JVppCallback) + +Note: jvpp_core.c calls back the JVppCallback instance with every response. An instance of the +JVppCallback is provided to jvpp_core.c by JVppRegistryImpl on JVppCoreImpl initialization. + +Part of the JVpp is also Future facade. It is asynchronous API returning Future objects +on top of low level JVpp. It wraps dump reply messages in one DTO using control_ping message +(provided by JVppRegistry). + + +Future facade + + /----------------\ /---------------\ + | FutureJVppCore | +-->+ JVppRegistry* | + \-----+----------/ | \---------------/ + ^ | + | implements | uses + | | + /--------+-------------\ | /------------------------------\ + | FutureJVppCoreFacade +---+--->+ FutureJVppCoreFacadeCallback | + \---------+------------/ uses \-------+----------------------/ + | | +---------------|-----------------------------|------------------------------- + | uses | implements +JVpp Java | | + | | + /----------\ | | + | JVppCore +<-+ | + \----+-----/ | + ^ | + | implements v + /----+---------\ /--------+---------------\ + | JVppCoreImpl | | JVppCoreGlobalCallback | + \--------------/ \------------------------/ + + + +Another useful utility of the JVpp is Callback facade. It is asynchronous API +capable of calling specific callback instance (provided when performing a call) +per call. + + +Callback facade + + /------------------\ /---------------\ + | CallbackJVppCore | +-->+ JVppRegistry* | + \-----+------------/ | \---------------/ + ^ | + | implements | uses + | | + /--------+---------------\ | /--------------------------\ + | CallbackJVppCoreFacade +---+--->+ CallbackJVppCoreCallback | + \---------+--------------/ uses \-----+--------------------/ + | | +---------------|-----------------------------|------------------------------- + | uses | implements +JVpp Java | | + | | + /----------\ | | + | JVppCore +<-+ | + \----+-----/ | + ^ | + | implements v + /----+---------\ /----------+-------------\ + | JVppCoreImpl | | JVppCoreGlobalCallback | + \--------------/ \------------------------/ + + +== Package structure + +* *io.fd.vpp.jvpp* - top level package for generated JVpp interface+ implementation and hand-crafted +VppConnection interface + implementation - packaged as jvpp-registry-version.jar + +* *io.fd.vpp.jvpp.[plugin]* - top level package for generated JVpp interface + implementation ++ plugin's API tests - packaged as jvpp-[plugin]-version.jar + +** *dto* - package for DTOs generated from VPP API structures + base/marker hand-crafted interfaces +(in case of jvpp-registry) +** *callback* - package for low-level JVpp callbacks and a global callback interface implementing each of +the low-level JVppcallbacks +** *future* - package for future based facade on top of JVpp and callbacks +** *callfacade* - package for callback based facade on top of JVpp and callbacks. Allowing +users to provide callback per request +** *test* - package for JVpp standalone tests. Can also serve as samples for JVpp. + +C code is structured into modules: + +* *jvpp_common* - shared library that provides jvpp_main_t reference used by jvpp_registry and plugins. + +* *jvpp_registry* - native library used by JVppRegistryImpl, responsible for: + +** VPP connection open/close +** Rx thread to java thread attach +** control ping message handling + +* *jvpp_core* - native library used by jvpp core plugin: +** *jvpp_core.c* - contains hand crafted code for core plugin initialization +** *jvpp_core_gen.h* - contains generated JNI compatible handlers for all requests and replies defined in vpe.api + +== Code generators +All of the required code except the base/marker interfaces is generated using +simple python2 code generators. The generators use __defs_vpp_papi.py__ input +file produced by __vppapigen__ from vpe.api file. + +=== JNI compatible C code +Produces __jvpp_[plugin]_gen.h__ file containing JNI compatible handlers for each VPP +request and reply. + +[NOTE] +==== +Source: jvpp_c_gen.py +==== + +=== Request/Reply DTOs +For all the structures in __defs_vpp_papi.py__ a POJO DTO is produced. Logically, +there are 4 types of DTOs: + +* Request - requests that can be sent to VPP and only a single response is expected +* DumpRequest - requests that can be sent to VPP and a stream of responses is expected +* Reply - reply to a simple request or a single response from dump triggered response stream +* ReplyDump - collection of replies from a single dump request +* Notifications/Events - Not implemented yet + +[NOTE] +==== +Source: dto_gen.py +==== + +=== JVpp +Produces __JVpp.java__ and __JVppImpl.java__. This is the layer right above JNI compatible C +code. + +[NOTE] +==== +Source: jvpp_impl_gen.py +==== + +=== Callbacks +Produces callback interface for each VPP reply + a global callback interface called +__JVpp[plugin]GlobalCallback.java__ aggregating all of the callback interfaces. The JNI +compatible C code expects only a single instance of this global callback and calls +it with every reply. + +[NOTE] +==== +Source: callback_gen.py +==== + +=== Future facade +Produces an asynchronous facade on top of JVpp and callbacks, which returns a Future that provides +matching reply once VPP invocation finishes. Sources produced: +__FutureJVpp[plugin].java, FutureJVpp[plugin]Facade.java and FutureJVpp[plugin]Callback.java__ + +[NOTE] +==== +Source: jvpp_future_facade_gen.py +==== + +=== Callback facade +Similar to future facade, only this facade takes callback objects as part of the invocation +and the callback is called with result once VPP invocation finishes. Sources produced: +__CallbackJVpp[plugin].java, CallbackJVpp[plugin]Facade.java and CallbackJVpp[plugin]Callback.java__ + +[NOTE] +==== +Source: jvpp_callback_facade_gen.py +==== diff --git a/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java new file mode 100644 index 00000000..979edbc4 --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclExpectedDumpData.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.acl.test; + + +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_ADDRESS_2_AS_ARRAY; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_ADDRESS_AS_ARRAY; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_DST_ICMP_TYPE_END; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_DST_ICMP_TYPE_START; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_MAC; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_MAC_MASK; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_PREFIX; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_PREFIX_2; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_SRC_ICMP_TYPE_END; +import static io.fd.vpp.jvpp.acl.test.AclTestData.FIRST_RULE_SRC_ICMP_TYPE_START; +import static io.fd.vpp.jvpp.acl.test.AclTestData.ICMP_PROTOCOL; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_ADDRESS_2_AS_ARRAY; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_ADDRESS_AS_ARRAY; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_DST_PORT_RANGE_END; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_DST_PORT_RANGE_START; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_MAC; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_MAC_MASK; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_PREFIX; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_PREFIX_2; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_SRC_PORT_RANGE_END; +import static io.fd.vpp.jvpp.acl.test.AclTestData.SECOND_RULE_SRC_PORT_RANGE_START; +import static io.fd.vpp.jvpp.acl.test.AclTestData.UDP_PROTOCOL; + +import io.fd.vpp.jvpp.acl.dto.AclDetails; +import io.fd.vpp.jvpp.acl.dto.AclInterfaceListDetails; +import io.fd.vpp.jvpp.acl.dto.MacipAclDetails; +import io.fd.vpp.jvpp.acl.types.AclRule; +import io.fd.vpp.jvpp.acl.types.MacipAclRule; +import java.util.Arrays; + +class AclExpectedDumpData { + + static void verifyMacIpDump(final MacipAclDetails macipAclDetails) { + // asserting data create by previous call + assertEquals(0, macipAclDetails.aclIndex); + assertEquals(2, macipAclDetails.count); + + final MacipAclRule currentIpv4Rule = macipAclDetails.r[0]; + final MacipAclRule currentIpv6Rule = macipAclDetails.r[1]; + + // Comparing one property at the time to better pointer if something is wrong + //Ipv4 rule + assertEquals(0, currentIpv4Rule.isIpv6); + assertEquals(1, currentIpv4Rule.isPermit); + + // cutting expected ipv4 to 4 bytes,vpp sends it as 16 always + assertArrays(FIRST_RULE_ADDRESS_AS_ARRAY, Arrays.copyOfRange(currentIpv4Rule.srcIpAddr, 0, 4)); + assertEquals(FIRST_RULE_PREFIX, currentIpv4Rule.srcIpPrefixLen); + assertArrays(FIRST_RULE_MAC, currentIpv4Rule.srcMac); + assertArrays(FIRST_RULE_MAC_MASK, currentIpv4Rule.srcMacMask); + + //Ipv6 rule + assertEquals(1, currentIpv6Rule.isIpv6); + assertEquals(0, currentIpv6Rule.isPermit); + assertArrays(SECOND_RULE_ADDRESS_AS_ARRAY, currentIpv6Rule.srcIpAddr); + assertEquals(SECOND_RULE_PREFIX, currentIpv6Rule.srcIpPrefixLen); + assertArrays(SECOND_RULE_MAC, currentIpv6Rule.srcMac); + assertArrays(SECOND_RULE_MAC_MASK, currentIpv6Rule.srcMacMask); + } + + static void verifyAclDump(final AclDetails aclDetails) { + assertEquals(0, aclDetails.aclIndex); + assertEquals(2, aclDetails.count); + + final AclRule currentIpv4Rule = aclDetails.r[0]; + final AclRule currentIpv6Rule = aclDetails.r[1]; + + // Comparing one property at the time to better pointer if something is wrong + //Ipv4 rule + assertEquals(0, currentIpv4Rule.isIpv6); + assertEquals(1, currentIpv4Rule.isPermit); + + // cutting expected ipv4 to 4 bytes,vpp sends it as 16 always + assertArrays(FIRST_RULE_ADDRESS_AS_ARRAY, Arrays.copyOfRange(currentIpv4Rule.srcIpAddr, 0, 4)); + assertEquals(FIRST_RULE_PREFIX, currentIpv4Rule.srcIpPrefixLen); + assertArrays(FIRST_RULE_ADDRESS_2_AS_ARRAY, Arrays.copyOfRange(currentIpv4Rule.dstIpAddr, 0, 4)); + assertEquals(FIRST_RULE_PREFIX_2, currentIpv4Rule.dstIpPrefixLen); + + assertEquals(ICMP_PROTOCOL, currentIpv4Rule.proto); + assertEquals(FIRST_RULE_SRC_ICMP_TYPE_START, currentIpv4Rule.srcportOrIcmptypeFirst); + assertEquals(FIRST_RULE_SRC_ICMP_TYPE_END, currentIpv4Rule.srcportOrIcmptypeLast); + assertEquals(FIRST_RULE_DST_ICMP_TYPE_START, currentIpv4Rule.dstportOrIcmpcodeFirst); + assertEquals(FIRST_RULE_DST_ICMP_TYPE_END, currentIpv4Rule.dstportOrIcmpcodeLast); + + assertArrays(SECOND_RULE_ADDRESS_AS_ARRAY, currentIpv6Rule.srcIpAddr); + assertEquals(SECOND_RULE_PREFIX, currentIpv6Rule.srcIpPrefixLen); + assertArrays(SECOND_RULE_ADDRESS_2_AS_ARRAY, currentIpv6Rule.dstIpAddr); + assertEquals(SECOND_RULE_PREFIX_2, currentIpv6Rule.dstIpPrefixLen); + + assertEquals(UDP_PROTOCOL, currentIpv6Rule.proto); + assertEquals(SECOND_RULE_SRC_PORT_RANGE_START, currentIpv6Rule.srcportOrIcmptypeFirst); + assertEquals(SECOND_RULE_SRC_PORT_RANGE_END, currentIpv6Rule.srcportOrIcmptypeLast); + assertEquals(SECOND_RULE_DST_PORT_RANGE_START, currentIpv6Rule.dstportOrIcmpcodeFirst); + assertEquals(SECOND_RULE_DST_PORT_RANGE_END, currentIpv6Rule.dstportOrIcmpcodeLast); + } + + static void verifyAclInterfaceList(final AclInterfaceListDetails aclInterfaceListDetails) { + assertEquals(1, aclInterfaceListDetails.count); + assertEquals(1, aclInterfaceListDetails.acls[0]); + assertEquals(0, aclInterfaceListDetails.nInput); + assertEquals(0, aclInterfaceListDetails.swIfIndex); + } + + private static void assertArrays(final byte[] expected, final byte[] actual) { + if (!Arrays.equals(expected, actual)) { + throw new IllegalArgumentException( + String.format("Expected[%s]/Actual[%s]", Arrays.toString(expected), Arrays.toString(actual))); + } + } + + private static void assertEquals(final int expected, final int actual) { + if (expected != actual) { + throw new IllegalArgumentException(String.format("Expected[%s]/Actual[%s]", expected, actual)); + } + } +} diff --git a/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestData.java b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestData.java new file mode 100644 index 00000000..5d228eea --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestData.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.acl.test; + + +import io.fd.vpp.jvpp.acl.types.AclRule; +import io.fd.vpp.jvpp.acl.types.MacipAclRule; + +class AclTestData { + + static final byte[] FIRST_RULE_ADDRESS_AS_ARRAY = {-64, -88, 2, 1}; + static final byte[] FIRST_RULE_ADDRESS_2_AS_ARRAY = {-64, -88, 2, 3}; + static final byte[] SECOND_RULE_ADDRESS_AS_ARRAY = + {32, 1, 13, -72, 10, 11, 18, -16, 0, 0, 0, 0, 0, 0, 0, 1}; + static final byte[] SECOND_RULE_ADDRESS_2_AS_ARRAY = + {32, 1, 13, -72, 10, 11, 18, -16, 0, 0, 0, 0, 0, 0, 0, 1}; + static final byte[] FIRST_RULE_MAC = {11, 11, 11, 11, 11, 11}; + static final byte[] FIRST_RULE_MAC_MASK = {0, 0, 0, 0, 0, 0}; + static final byte[] SECOND_RULE_MAC = {11, 12, 11, 11, 12, 11}; + static final byte[] SECOND_RULE_MAC_MASK = {(byte) 170, 0, 0, 0, 0, 0}; + static final int FIRST_RULE_PREFIX = 32; + static final int FIRST_RULE_PREFIX_2 = 24; + static final int SECOND_RULE_PREFIX = 64; + static final int SECOND_RULE_PREFIX_2 = 62; + static final int FIRST_RULE_DST_ICMP_TYPE_START = 0; + static final int FIRST_RULE_DST_ICMP_TYPE_END = 8; + static final int FIRST_RULE_SRC_ICMP_TYPE_START = 1; + static final int FIRST_RULE_SRC_ICMP_TYPE_END = 7; + static final int ICMP_PROTOCOL = 1; + static final int SECOND_RULE_DST_PORT_RANGE_START = 2000; + static final int SECOND_RULE_DST_PORT_RANGE_END = 6000; + static final int SECOND_RULE_SRC_PORT_RANGE_START = 400; + static final int SECOND_RULE_SRC_PORT_RANGE_END = 2047; + static final int UDP_PROTOCOL = 17; + + + static MacipAclRule[] createMacipRules() { + MacipAclRule ruleOne = new MacipAclRule(); + ruleOne.isIpv6 = 0; + ruleOne.isPermit = 1; + ruleOne.srcIpAddr = FIRST_RULE_ADDRESS_AS_ARRAY; + ruleOne.srcIpPrefixLen = FIRST_RULE_PREFIX; + ruleOne.srcMac = FIRST_RULE_MAC; + ruleOne.srcMacMask = FIRST_RULE_MAC_MASK;// no mask + + MacipAclRule ruleTwo = new MacipAclRule(); + ruleTwo.isIpv6 = 1; + ruleTwo.isPermit = 0; + ruleTwo.srcIpAddr = SECOND_RULE_ADDRESS_AS_ARRAY; + ruleTwo.srcIpPrefixLen = SECOND_RULE_PREFIX; + ruleTwo.srcMac = SECOND_RULE_MAC; + ruleTwo.srcMacMask = SECOND_RULE_MAC_MASK; + + return new MacipAclRule[]{ruleOne, ruleTwo}; + } + + static AclRule[] createAclRules() { + AclRule ruleOne = new AclRule(); + + ruleOne.isIpv6 = 0; + ruleOne.isPermit = 1; + ruleOne.srcIpAddr = FIRST_RULE_ADDRESS_AS_ARRAY; + ruleOne.srcIpPrefixLen = FIRST_RULE_PREFIX; + ruleOne.dstIpAddr = FIRST_RULE_ADDRESS_2_AS_ARRAY; + ruleOne.dstIpPrefixLen = FIRST_RULE_PREFIX_2; + ruleOne.dstportOrIcmpcodeFirst = FIRST_RULE_DST_ICMP_TYPE_START; + ruleOne.dstportOrIcmpcodeLast = FIRST_RULE_DST_ICMP_TYPE_END; + ruleOne.srcportOrIcmptypeFirst = FIRST_RULE_SRC_ICMP_TYPE_START; + ruleOne.srcportOrIcmptypeLast = FIRST_RULE_SRC_ICMP_TYPE_END; + ruleOne.proto = ICMP_PROTOCOL; //ICMP + + AclRule ruleTwo = new AclRule(); + ruleTwo.isIpv6 = 1; + ruleTwo.isPermit = 0; + ruleTwo.srcIpAddr = SECOND_RULE_ADDRESS_AS_ARRAY; + ruleTwo.srcIpPrefixLen = SECOND_RULE_PREFIX; + ruleTwo.dstIpAddr = SECOND_RULE_ADDRESS_2_AS_ARRAY; + ruleTwo.dstIpPrefixLen = SECOND_RULE_PREFIX_2; + ruleTwo.dstportOrIcmpcodeFirst = SECOND_RULE_DST_PORT_RANGE_START; + ruleTwo.dstportOrIcmpcodeLast = SECOND_RULE_DST_PORT_RANGE_END; + ruleTwo.srcportOrIcmptypeFirst = SECOND_RULE_SRC_PORT_RANGE_START; + ruleTwo.srcportOrIcmptypeLast = SECOND_RULE_SRC_PORT_RANGE_END; + ruleTwo.proto = UDP_PROTOCOL; //UDP + + return new AclRule[]{ruleOne, ruleTwo}; + } +} diff --git a/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestRequests.java b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestRequests.java new file mode 100644 index 00000000..b580ee8c --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/AclTestRequests.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.acl.test; + +import static io.fd.vpp.jvpp.acl.test.AclTestData.createAclRules; +import static io.fd.vpp.jvpp.acl.test.AclTestData.createMacipRules; + +import io.fd.vpp.jvpp.VppInvocationException; +import io.fd.vpp.jvpp.acl.dto.AclAddReplace; +import io.fd.vpp.jvpp.acl.dto.AclAddReplaceReply; +import io.fd.vpp.jvpp.acl.dto.AclDel; +import io.fd.vpp.jvpp.acl.dto.AclDelReply; +import io.fd.vpp.jvpp.acl.dto.AclDetailsReplyDump; +import io.fd.vpp.jvpp.acl.dto.AclDump; +import io.fd.vpp.jvpp.acl.dto.AclInterfaceListDetailsReplyDump; +import io.fd.vpp.jvpp.acl.dto.AclInterfaceListDump; +import io.fd.vpp.jvpp.acl.dto.AclInterfaceSetAclList; +import io.fd.vpp.jvpp.acl.dto.AclInterfaceSetAclListReply; +import io.fd.vpp.jvpp.acl.dto.MacipAclAdd; +import io.fd.vpp.jvpp.acl.dto.MacipAclAddReply; +import io.fd.vpp.jvpp.acl.dto.MacipAclDel; +import io.fd.vpp.jvpp.acl.dto.MacipAclDelReply; +import io.fd.vpp.jvpp.acl.dto.MacipAclDetailsReplyDump; +import io.fd.vpp.jvpp.acl.dto.MacipAclDump; +import io.fd.vpp.jvpp.acl.future.FutureJVppAclFacade; +import java.util.concurrent.ExecutionException; + +class AclTestRequests { + + static MacipAclDetailsReplyDump sendMacIpDumpRequest(final FutureJVppAclFacade jvpp) + throws ExecutionException, InterruptedException { + System.out.println("Sending MacipAclDump request..."); + MacipAclDetailsReplyDump dump = jvpp.macipAclDump(new MacipAclDump()).toCompletableFuture().get(); + System.out.println("MacipAclDump returned"); + return dump; + } + + static void sendMacIpAddRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { + final MacipAclAdd request = createMacIpAddRequest(); + System.out.printf("Sending MacipAclAdd request %s%n", request.toString()); + final MacipAclAddReply reply = jvpp.macipAclAdd(createMacIpAddRequest()).toCompletableFuture().get(); + System.out.printf("MacipAclAdd send result = %s%n", reply); + } + + static void sendMacIpDelRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { + final MacipAclDel request = new MacipAclDel(); + request.aclIndex = 0; + System.out.printf("Sending MacipAclDel request %s%n", request.toString()); + final MacipAclDelReply reply = jvpp.macipAclDel(request).toCompletableFuture().get(); + System.out.printf("MacipAclDel send result = %s%n", reply); + } + + static void sendAclAddRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { + final AclAddReplace request = createAclAddRequest(); + System.out.printf("Sending AclAddReplace request %s%n", request.toString()); + final AclAddReplaceReply reply = jvpp.aclAddReplace(request).toCompletableFuture().get(); + System.out.printf("AclAddReplace send result = %s%n", reply); + } + + static AclDetailsReplyDump sendAclDumpRequest(final FutureJVppAclFacade jvpp) + throws InterruptedException, VppInvocationException, ExecutionException { + System.out.println("Sending AclDump request..."); + final AclDetailsReplyDump dump = jvpp.aclDump(new AclDump()).toCompletableFuture().get(); + System.out.printf("AclDump send result = %s%n", dump); + return dump; + } + + static void sendAclDelRequest(final FutureJVppAclFacade jvpp) throws InterruptedException, ExecutionException { + final AclDel request = new AclDel(); + request.aclIndex = 0; + System.out.printf("Sending AclDel request %s%n", request.toString()); + final AclDelReply reply = jvpp.aclDel(request).toCompletableFuture().get(); + System.out.printf("AclDel send result = %s%n", reply); + } + + static AclInterfaceListDetailsReplyDump sendAclInterfaceListDumpRequest(final FutureJVppAclFacade jvpp) + throws InterruptedException, ExecutionException { + final AclInterfaceListDump request = new AclInterfaceListDump(); + request.swIfIndex = 0; + System.out.printf("Sending AclInterfaceListDump request %s%n", request.toString()); + final AclInterfaceListDetailsReplyDump dump = jvpp.aclInterfaceListDump(request).toCompletableFuture().get(); + System.out.printf("AclInterfaceListDump send result = %s%n", dump); + return dump; + } + + static void sendAclInterfaceSetAclList(final FutureJVppAclFacade jvpp) + throws InterruptedException, ExecutionException { + final AclInterfaceSetAclList request = new AclInterfaceSetAclList(); + request.count = 1; + request.acls = new int[]{1}; + request.swIfIndex = 0; + request.nInput = 0; + System.out.printf("Sending AclInterfaceSetAclList request %s%n", request.toString()); + final AclInterfaceSetAclListReply reply = jvpp.aclInterfaceSetAclList(request).toCompletableFuture().get(); + System.out.printf("AclInterfaceSetAclList send result = %s%n", reply); + } + + static void sendAclInterfaceDeleteList(final FutureJVppAclFacade jvpp) + throws InterruptedException, ExecutionException { + // uses same api but sets list to empty + final AclInterfaceSetAclList request = new AclInterfaceSetAclList(); + request.count = 0; + request.acls = new int[]{}; + request.swIfIndex = 0; + request.nInput = 0; + System.out.printf("Sending AclInterfaceSetAclList(Delete) request %s%n", request.toString()); + final AclInterfaceSetAclListReply reply = jvpp.aclInterfaceSetAclList(request).toCompletableFuture().get(); + System.out.printf("AclInterfaceSetAclList(Delete) send result = %s%n", reply); + } + + private static MacipAclAdd createMacIpAddRequest() { + MacipAclAdd request = new MacipAclAdd(); + + request.count = 2; + request.r = createMacipRules(); + return request; + } + + private static AclAddReplace createAclAddRequest() { + AclAddReplace request = new AclAddReplace(); + + request.aclIndex = -1;// to define new one + request.count = 2; + request.r = createAclRules(); + return request; + } +} diff --git a/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/FutureApiTest.java b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/FutureApiTest.java new file mode 100644 index 00000000..94490193 --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/FutureApiTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.acl.test; + +import static io.fd.vpp.jvpp.acl.test.AclExpectedDumpData.verifyAclDump; +import static io.fd.vpp.jvpp.acl.test.AclExpectedDumpData.verifyAclInterfaceList; +import static io.fd.vpp.jvpp.acl.test.AclExpectedDumpData.verifyMacIpDump; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclAddRequest; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclDelRequest; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclDumpRequest; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclInterfaceDeleteList; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclInterfaceListDumpRequest; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendAclInterfaceSetAclList; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendMacIpAddRequest; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendMacIpDelRequest; +import static io.fd.vpp.jvpp.acl.test.AclTestRequests.sendMacIpDumpRequest; + +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.acl.JVppAclImpl; +import io.fd.vpp.jvpp.acl.future.FutureJVppAclFacade; + +public class FutureApiTest { + + public static void main(String[] args) throws Exception { + testCallbackApi(); + } + + private static void testCallbackApi() throws Exception { + System.out.println("Testing Java callback API for acl plugin"); + try (final JVppRegistry registry = new JVppRegistryImpl("macipAclAddTest"); + final FutureJVppAclFacade jvpp = new FutureJVppAclFacade(registry, new JVppAclImpl())) { + + // adds,dump and verifies Mac-Ip acl + sendMacIpAddRequest(jvpp); + verifyMacIpDump(sendMacIpDumpRequest(jvpp).macipAclDetails.get(0)); + + // adds,dumps and verifies Acl acl + sendAclAddRequest(jvpp); + verifyAclDump(sendAclDumpRequest(jvpp).aclDetails.get(0)); + + // adds,dumps and verifies Interface for acl + sendAclInterfaceSetAclList(jvpp); + verifyAclInterfaceList(sendAclInterfaceListDumpRequest(jvpp).aclInterfaceListDetails.get(0)); + + // deletes all created data + sendAclInterfaceDeleteList(jvpp); + sendAclDelRequest(jvpp); + sendMacIpDelRequest(jvpp); + + System.out.println("Disconnecting..."); + } + } +} diff --git a/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/Readme.txt b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/Readme.txt new file mode 100644 index 00000000..f68e7aba --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/io/fd/vpp/jvpp/acl/test/Readme.txt @@ -0,0 +1 @@ +sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/plugins/acl-plugin/jvpp-acl-1.0.jar io.fd.vpp.jvpp.acl.test.FutureApiTest diff --git a/src/vpp-api/java/jvpp-acl/jvpp_acl.c b/src/vpp-api/java/jvpp-acl/jvpp_acl.c new file mode 100644 index 00000000..d56abe3d --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/jvpp_acl.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#define vl_typedefs /* define message structures */ +#include +#undef vl_typedefs + +#define vl_endianfun +#include +#undef vl_endianfun + +#define vl_print(handle, ...) +#define vl_printfun +#include +#undef vl_printfun + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include +#undef vl_api_version + +#include +#include +#include + +#if VPPJNI_DEBUG == 1 + #define DEBUG_LOG(...) clib_warning(__VA_ARGS__) +#else + #define DEBUG_LOG(...) +#endif + +#include + +#include "jvpp-acl/io_fd_vpp_jvpp_acl_JVppAclImpl.h" +#include "jvpp_acl.h" +#include "jvpp-acl/jvpp_acl_gen.h" + +/* + * Class: io_fd_vpp_jvpp_acl_JVppaclImpl + * Method: init0 + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_acl_JVppAclImpl_init0 + (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) { + acl_main_t * plugin_main = &acl_main; + u8 * name; + clib_warning ("Java_io_fd_vpp_jvpp_acl_JVppAclImpl_init0"); + + plugin_main->my_client_index = my_client_index; + plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address; + + name = format (0, "acl_%08x%c", api_version, 0); + plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + if (plugin_main->msg_id_base == (u16) ~0) { + jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException"); + (*env)->ThrowNew(env, exClass, "acl plugin is not loaded in VPP"); + } else { + plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback); + plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback)); + + #define _(N,n) \ + vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_api_reply_handler; + #undef _ + } +} + +JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_acl_JVppAclImpl_close0 +(JNIEnv *env, jclass clazz) { + acl_main_t * plugin_main = &acl_main; + + // cleanup: + (*env)->DeleteGlobalRef(env, plugin_main->callbackClass); + (*env)->DeleteGlobalRef(env, plugin_main->callbackObject); + + plugin_main->callbackClass = NULL; + plugin_main->callbackObject = NULL; +} + +/* Attach thread to JVM and cache class references when initiating JVPP ACL */ +jint JNI_OnLoad(JavaVM *vm, void *reserved) { + JNIEnv* env; + + if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { + return JNI_EVERSION; + } + + if (cache_class_references(env) != 0) { + clib_warning ("Failed to cache class references\n"); + return JNI_ERR; + } + + return JNI_VERSION_1_8; +} + +/* Clean up cached references when disposing JVPP ACL */ +void JNI_OnUnload(JavaVM *vm, void *reserved) { + JNIEnv* env; + if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) { + return; + } + delete_class_references(env); +} diff --git a/src/vpp-api/java/jvpp-acl/jvpp_acl.h b/src/vpp-api/java/jvpp-acl/jvpp_acl.h new file mode 100644 index 00000000..2b73d672 --- /dev/null +++ b/src/vpp-api/java/jvpp-acl/jvpp_acl.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_jvpp_acl_h__ +#define __included_jvpp_acl_h__ + +#include +#include +#include +#include +#include +#include + +/* Global state for JVPP-acl */ +typedef struct { + /* Base message index for the acl plugin */ + u16 msg_id_base; + + /* Pointer to shared memory queue */ + unix_shared_memory_queue_t * vl_input_queue; + + /* VPP api client index */ + u32 my_client_index; + + /* Callback object and class references enabling asynchronous Java calls */ + jobject callbackObject; + jclass callbackClass; + +} acl_main_t; + +acl_main_t acl_main __attribute__((aligned (64))); + + +#endif /* __included_jvpp_acl_h__ */ diff --git a/src/vpp-api/java/jvpp-common/jvpp_common.c b/src/vpp-api/java/jvpp-common/jvpp_common.c new file mode 100644 index 00000000..a161c09c --- /dev/null +++ b/src/vpp-api/java/jvpp-common/jvpp_common.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define _GNU_SOURCE /* for strcasestr(3) */ + +#include "jvpp_common.h" + +#ifndef JVPP_DEBUG +#define JVPP_DEBUG 0 +#endif + +#if JVPP_DEBUG == 1 +#define DEBUG_LOG(...) clib_warning(__VA_ARGS__) +#else +#define DEBUG_LOG(...) +#endif + +/* shared jvpp main structure */ +jvpp_main_t jvpp_main __attribute__((aligned (64))); + +void call_on_error(const char* callName, int contextId, int retval, + jclass callbackClass, jobject callbackObject, + jclass callbackExceptionClass) { + DEBUG_LOG("\nCallOnError : callback=%s, retval=%d, context=%d\n", callName, + clib_net_to_host_u32(retval), clib_net_to_host_u32(context)); + JNIEnv *env = jvpp_main.jenv; + if (!callbackClass) { + DEBUG_LOG("CallOnError : jm->callbackClass is null!\n"); + return; + } + jmethodID excConstructor = (*env)->GetMethodID(env, callbackExceptionClass, + "", "(Ljava/lang/String;II)V"); + if (!excConstructor) { + DEBUG_LOG("CallOnError : excConstructor is null!\n"); + return; + } + jmethodID callbackExcMethod = (*env)->GetMethodID(env, callbackClass, + "onError", "(Lio/fd/vpp/jvpp/VppCallbackException;)V"); + if (!callbackExcMethod) { + DEBUG_LOG("CallOnError : callbackExcMethod is null!\n"); + return; + } + + jobject excObject = (*env)->NewObject(env, callbackExceptionClass, + excConstructor, (*env)->NewStringUTF(env, callName), + clib_net_to_host_u32(contextId), clib_net_to_host_u32(retval)); + if (!excObject) { + DEBUG_LOG("CallOnError : excObject is null!\n"); + return; + } + + (*env)->CallVoidMethod(env, callbackObject, callbackExcMethod, excObject); + DEBUG_LOG("CallOnError : Response sent\n"); +} diff --git a/src/vpp-api/java/jvpp-common/jvpp_common.h b/src/vpp-api/java/jvpp-common/jvpp_common.h new file mode 100644 index 00000000..bbb203ed --- /dev/null +++ b/src/vpp-api/java/jvpp-common/jvpp_common.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_jvpp_common_h__ +#define __included_jvpp_common_h__ +// +#include +#include +#include +#include + +typedef struct { + /* Unique identifier used for matching replays with requests */ + volatile u32 context_id; + + /* Spinlock */ + volatile u32 lock; + u32 tag; + + /* JNI Native Method Interface pointer for message handlers */ + JNIEnv *jenv; + + /* JNI Invoke Interface pointer for attachment of rx thread to java thread */ + JavaVM *jvm; + + /* Convenience */ + unix_shared_memory_queue_t * vl_input_queue; + u32 my_client_index; +} jvpp_main_t; + +extern jvpp_main_t jvpp_main __attribute__((aligned (64))); + +static_always_inline u32 vppjni_get_context_id(jvpp_main_t * jm) { + return __sync_add_and_fetch(&jm->context_id, 1); +} + +static_always_inline void vppjni_lock(jvpp_main_t * jm, u32 tag) { + while (__sync_lock_test_and_set(&jm->lock, 1)) + ; + jm->tag = tag; +} + +static_always_inline void vppjni_unlock(jvpp_main_t * jm) { + jm->tag = 0; + CLIB_MEMORY_BARRIER(); + jm->lock = 0; +} + +/** + * Calls onError callback on callbackObject reference. Passes instance of callbackExceptionClass as parameter. + */ +void call_on_error(const char* callName, int contextId, int retval, + jclass callbackClass, jobject callbackObject, + jclass callbackExceptionClass); + +#endif /* __included_jvpp_common_h__ */ diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java new file mode 100644 index 00000000..986993b8 --- /dev/null +++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.core.test; + +import io.fd.vpp.jvpp.JVpp; +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.VppCallbackException; +import io.fd.vpp.jvpp.core.JVppCoreImpl; +import io.fd.vpp.jvpp.core.callback.GetNodeIndexCallback; +import io.fd.vpp.jvpp.core.callback.ShowVersionCallback; +import io.fd.vpp.jvpp.core.callback.SwInterfaceCallback; +import io.fd.vpp.jvpp.core.dto.GetNodeIndex; +import io.fd.vpp.jvpp.core.dto.GetNodeIndexReply; +import io.fd.vpp.jvpp.core.dto.ShowVersion; +import io.fd.vpp.jvpp.core.dto.ShowVersionReply; +import io.fd.vpp.jvpp.core.dto.SwInterfaceDetails; +import io.fd.vpp.jvpp.core.dto.SwInterfaceDump; + +public class CallbackApiTest { + + public static void main(String[] args) throws Exception { + testCallbackApi(); + } + + private static void testCallbackApi() throws Exception { + System.out.println("Testing Java callback API with JVppRegistry"); + try (final JVppRegistry registry = new JVppRegistryImpl("CallbackApiTest"); + final JVpp jvpp = new JVppCoreImpl()) { + registry.register(jvpp, new TestCallback()); + + System.out.println("Sending ShowVersion request..."); + final int result = jvpp.send(new ShowVersion()); + System.out.printf("ShowVersion send result = %d%n", result); + + System.out.println("Sending GetNodeIndex request..."); + GetNodeIndex getNodeIndexRequest = new GetNodeIndex(); + getNodeIndexRequest.nodeName = "non-existing-node".getBytes(); + jvpp.send(getNodeIndexRequest); + + System.out.println("Sending SwInterfaceDump request..."); + SwInterfaceDump swInterfaceDumpRequest = new SwInterfaceDump(); + swInterfaceDumpRequest.nameFilterValid = 0; + swInterfaceDumpRequest.nameFilter = "".getBytes(); + jvpp.send(swInterfaceDumpRequest); + + Thread.sleep(1000); + System.out.println("Disconnecting..."); + } + Thread.sleep(1000); + } + + static class TestCallback implements GetNodeIndexCallback, ShowVersionCallback, SwInterfaceCallback { + + @Override + public void onGetNodeIndexReply(final GetNodeIndexReply msg) { + System.out.printf("Received GetNodeIndexReply: %s%n", msg); + } + + @Override + public void onShowVersionReply(final ShowVersionReply msg) { + System.out.printf("Received ShowVersionReply: context=%d, program=%s, version=%s, " + + "buildDate=%s, buildDirectory=%s%n", + msg.context, new String(msg.program), new String(msg.version), + new String(msg.buildDate), new String(msg.buildDirectory)); + } + + @Override + public void onSwInterfaceDetails(final SwInterfaceDetails msg) { + System.out.printf("Received SwInterfaceDetails: interfaceName=%s, l2AddressLength=%d, adminUpDown=%d, " + + "linkUpDown=%d, linkSpeed=%d, linkMtu=%d%n", + new String(msg.interfaceName), msg.l2AddressLength, msg.adminUpDown, + msg.linkUpDown, msg.linkSpeed, (int) msg.linkMtu); + } + + @Override + public void onError(VppCallbackException ex) { + System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(), + ex.getCtxId(), ex.getErrorCode()); + } + } +} diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java new file mode 100644 index 00000000..d84cb034 --- /dev/null +++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.core.test; + +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.VppCallbackException; +import io.fd.vpp.jvpp.core.JVppCore; +import io.fd.vpp.jvpp.core.JVppCoreImpl; +import io.fd.vpp.jvpp.core.callback.WantInterfaceEventsCallback; +import io.fd.vpp.jvpp.core.callfacade.CallbackJVppCoreFacade; +import io.fd.vpp.jvpp.core.dto.WantInterfaceEventsReply; + +public class CallbackJVppFacadeNotificationTest { + + private static void testCallbackFacade() throws Exception { + System.out.println("Testing CallbackJVppFacade for notifications"); + + try (final JVppRegistry registry = new JVppRegistryImpl("CallbackFacadeTest"); + final JVppCore jvpp = new JVppCoreImpl()) { + final CallbackJVppCoreFacade jvppCallbackFacade = new CallbackJVppCoreFacade(registry, jvpp); + System.out.println("Successfully connected to VPP"); + + final AutoCloseable notificationListenerReg = + jvppCallbackFacade.getNotificationRegistry().registerSwInterfaceSetFlagsNotificationCallback( + NotificationUtils::printNotification + ); + + jvppCallbackFacade.wantInterfaceEvents(NotificationUtils.getEnableInterfaceNotificationsReq(), + new WantInterfaceEventsCallback() { + @Override + public void onWantInterfaceEventsReply(final WantInterfaceEventsReply reply) { + System.out.println("Interface events started"); + } + + @Override + public void onError(final VppCallbackException ex) { + System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", + ex.getMethodName(), ex.getCtxId(), ex.getErrorCode()); + } + }); + + System.out.println("Changing interface configuration"); + NotificationUtils.getChangeInterfaceState().send(jvpp); + + Thread.sleep(1000); + + jvppCallbackFacade.wantInterfaceEvents(NotificationUtils.getDisableInterfaceNotificationsReq(), + new WantInterfaceEventsCallback() { + @Override + public void onWantInterfaceEventsReply(final WantInterfaceEventsReply reply) { + System.out.println("Interface events stopped"); + } + + @Override + public void onError(final VppCallbackException ex) { + System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", + ex.getMethodName(), ex.getCtxId(), ex.getErrorCode()); + } + }); + + notificationListenerReg.close(); + + Thread.sleep(2000); + System.out.println("Disconnecting..."); + } + Thread.sleep(1000); + } + + public static void main(String[] args) throws Exception { + testCallbackFacade(); + } +} diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java new file mode 100644 index 00000000..9f7cb8de --- /dev/null +++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.core.test; + +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.VppCallbackException; +import io.fd.vpp.jvpp.core.JVppCoreImpl; +import io.fd.vpp.jvpp.core.callback.GetNodeIndexCallback; +import io.fd.vpp.jvpp.core.callback.ShowVersionCallback; +import io.fd.vpp.jvpp.core.callfacade.CallbackJVppCoreFacade; +import io.fd.vpp.jvpp.core.dto.GetNodeIndex; +import io.fd.vpp.jvpp.core.dto.GetNodeIndexReply; +import io.fd.vpp.jvpp.core.dto.ShowVersionReply; + +/** + * CallbackJVppFacade together with CallbackJVppFacadeCallback allow for setting different callback for each request. + * This is more convenient than the approach shown in CallbackApiTest. + */ +public class CallbackJVppFacadeTest { + + private static ShowVersionCallback showVersionCallback1 = new ShowVersionCallback() { + @Override + public void onShowVersionReply(final ShowVersionReply msg) { + System.out.printf("ShowVersionCallback1 received ShowVersionReply: context=%d, program=%s," + + "version=%s, buildDate=%s, buildDirectory=%s%n", msg.context, new String(msg.program), + new String(msg.version), new String(msg.buildDate), new String(msg.buildDirectory)); + } + + @Override + public void onError(VppCallbackException ex) { + System.out.printf("Received onError exception in showVersionCallback1: call=%s, reply=%d, context=%d%n", + ex.getMethodName(), ex.getErrorCode(), ex.getCtxId()); + } + }; + + private static ShowVersionCallback showVersionCallback2 = new ShowVersionCallback() { + @Override + public void onShowVersionReply(final ShowVersionReply msg) { + System.out.printf("ShowVersionCallback2 received ShowVersionReply: context=%d, program=%s," + + "version=%s, buildDate=%s, buildDirectory=%s%n", msg.context, new String(msg.program), + new String(msg.version), new String(msg.buildDate), new String(msg.buildDirectory)); + } + + @Override + public void onError(VppCallbackException ex) { + System.out.printf("Received onError exception in showVersionCallback2: call=%s, reply=%d, context=%d%n", + ex.getMethodName(), ex.getErrorCode(), ex.getCtxId()); + } + + }; + + private static GetNodeIndexCallback getNodeIndexCallback = new GetNodeIndexCallback() { + @Override + public void onGetNodeIndexReply(final GetNodeIndexReply msg) { + System.out.printf("Received GetNodeIndexReply: %s%n", msg); + } + + @Override + public void onError(VppCallbackException ex) { + System.out.printf("Received onError exception in getNodeIndexCallback: call=%s, reply=%d, context=%d%n", + ex.getMethodName(), ex.getErrorCode(), ex.getCtxId()); + } + }; + + private static void testCallbackFacade() throws Exception { + System.out.println("Testing CallbackJVppFacade"); + + try (final JVppRegistry registry = new JVppRegistryImpl("CallbackFacadeTest"); + final CallbackJVppCoreFacade callbackFacade = new CallbackJVppCoreFacade(registry, new JVppCoreImpl())) { + System.out.println("Successfully connected to VPP"); + + callbackFacade.showVersion(showVersionCallback1); + callbackFacade.showVersion(showVersionCallback2); + + GetNodeIndex getNodeIndexRequest = new GetNodeIndex(); + getNodeIndexRequest.nodeName = "dummyNode".getBytes(); + callbackFacade.getNodeIndex(getNodeIndexRequest, getNodeIndexCallback); + + Thread.sleep(2000); + System.out.println("Disconnecting..."); + } + Thread.sleep(1000); + } + + public static void main(String[] args) throws Exception { + testCallbackFacade(); + } +} diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java new file mode 100644 index 00000000..a9f71f11 --- /dev/null +++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.core.test; + +import static io.fd.vpp.jvpp.core.test.NotificationUtils.getChangeInterfaceState; +import static io.fd.vpp.jvpp.core.test.NotificationUtils.getDisableInterfaceNotificationsReq; +import static io.fd.vpp.jvpp.core.test.NotificationUtils.getEnableInterfaceNotificationsReq; +import static io.fd.vpp.jvpp.core.test.NotificationUtils.printNotification; + +import io.fd.vpp.jvpp.JVpp; +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.VppCallbackException; +import io.fd.vpp.jvpp.core.JVppCoreImpl; +import io.fd.vpp.jvpp.core.callback.SwInterfaceSetFlagsCallback; +import io.fd.vpp.jvpp.core.callback.SwInterfaceSetFlagsNotificationCallback; +import io.fd.vpp.jvpp.core.callback.WantInterfaceEventsCallback; +import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlagsNotification; +import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlagsReply; +import io.fd.vpp.jvpp.core.dto.WantInterfaceEventsReply; + +public class CallbackNotificationApiTest { + + private static void testCallbackApi() throws Exception { + System.out.println("Testing Java callback API for notifications"); + try (final JVppRegistry registry = new JVppRegistryImpl("CallbackNotificationTest"); + final JVpp jvpp = new JVppCoreImpl()) { + registry.register(jvpp, new TestCallback()); + System.out.println("Successfully connected to VPP"); + + getEnableInterfaceNotificationsReq().send(jvpp); + System.out.println("Interface notifications started"); + // TODO test ifc dump which also triggers interface flags send + + System.out.println("Changing interface configuration"); + getChangeInterfaceState().send(jvpp); + + // Notifications are received + Thread.sleep(500); + + getDisableInterfaceNotificationsReq().send(jvpp); + System.out.println("Interface events stopped"); + + Thread.sleep(2000); + System.out.println("Disconnecting..."); + } + Thread.sleep(1000); + } + + public static void main(String[] args) throws Exception { + testCallbackApi(); + } + + private static class TestCallback implements SwInterfaceSetFlagsNotificationCallback, + WantInterfaceEventsCallback, SwInterfaceSetFlagsCallback { + + @Override + public void onSwInterfaceSetFlagsNotification( + final SwInterfaceSetFlagsNotification msg) { + printNotification(msg); + } + + @Override + public void onWantInterfaceEventsReply(final WantInterfaceEventsReply wantInterfaceEventsReply) { + System.out.println("Interface notification stream updated"); + } + + @Override + public void onSwInterfaceSetFlagsReply(final SwInterfaceSetFlagsReply swInterfaceSetFlagsReply) { + System.out.println("Interface flags set successfully"); + } + + @Override + public void onError(VppCallbackException ex) { + System.out.printf("Received onError exception in getNodeIndexCallback: call=%s, reply=%d, context=%d%n", + ex.getMethodName(), ex.getErrorCode(), ex.getCtxId()); + + } + } +} diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java new file mode 100644 index 00000000..e97f4e3a --- /dev/null +++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.core.test; + +import io.fd.vpp.jvpp.JVpp; +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.VppCallbackException; +import io.fd.vpp.jvpp.callback.ControlPingCallback; +import io.fd.vpp.jvpp.core.JVppCoreImpl; +import io.fd.vpp.jvpp.dto.ControlPing; +import io.fd.vpp.jvpp.dto.ControlPingReply; + +public class ControlPingTest { + + private static void testControlPing() throws Exception { + System.out.println("Testing ControlPing using Java callback API"); + try (JVppRegistry registry = new JVppRegistryImpl("ControlPingTest"); + JVpp jvpp = new JVppCoreImpl()) { + + registry.register(jvpp, new ControlPingCallback() { + @Override + public void onControlPingReply(final ControlPingReply reply) { + System.out.printf("Received ControlPingReply: %s%n", reply); + } + + @Override + public void onError(VppCallbackException ex) { + System.out.printf("Received onError exception: call=%s, reply=%d, context=%d ", ex.getMethodName(), + ex.getErrorCode(), ex.getCtxId()); + } + + }); + System.out.println("Successfully connected to VPP"); + Thread.sleep(1000); + + System.out.println("Sending control ping using JVppRegistry"); + registry.controlPing(jvpp.getClass()); + + Thread.sleep(2000); + + System.out.println("Sending control ping using JVpp plugin"); + jvpp.send(new ControlPing()); + + Thread.sleep(2000); + System.out.println("Disconnecting..."); + } + Thread.sleep(1000); + } + + public static void main(String[] args) throws Exception { + testControlPing(); + } +} diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java new file mode 100644 index 00000000..a96258f4 --- /dev/null +++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.vpp.jvpp.core.test; + +import static java.util.Objects.requireNonNull; + +import io.fd.vpp.jvpp.JVppRegistry; +import io.fd.vpp.jvpp.JVppRegistryImpl; +import io.fd.vpp.jvpp.core.JVppCoreImpl; +import io.fd.vpp.jvpp.core.dto.CreateSubif; +import io.fd.vpp.jvpp.core.dto.CreateSubifReply; +import io.fd.vpp.jvpp.core.dto.SwInterfaceDetailsReplyDump; +import io.fd.vpp.jvpp.core.dto.SwInterfaceDump; +import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade; + +/** + *

Tests sub-interface creation.
Equivalent to:
+ * + * 

{@code
+ * vppctl create sub GigabitEthernet0/9/0 1 dot1q 100 inner-dot1q any
+ * }
+ *
+ * + * To verify invoke:
+ * 
{@code
+ * vpp_api_test json
+ * vat# sw_interface_dump
+ * }
+ */
+public class CreateSubInterfaceTest {
+
+    private static SwInterfaceDump createSwInterfaceDumpRequest(final String ifaceName) {
+        SwInterfaceDump request = new SwInterfaceDump();
+        request.nameFilter = ifaceName.getBytes();
+        request.nameFilterValid = 1;
+        return request;
+    }
+
+    private static void requireSingleIface(final SwInterfaceDetailsReplyDump response, final String ifaceName) {
+        if (response.swInterfaceDetails.size() != 1) {
+            throw new IllegalStateException(
+                String.format("Expected one interface matching filter %s but was %d", ifaceName,
+                    response.swInterfaceDetails.size()));
+        }
+    }
+
+    private static CreateSubif createSubifRequest(final int swIfIndex, final int subId) {
+        CreateSubif request = new CreateSubif();
+        request.swIfIndex = swIfIndex; // super interface id
+        request.subId = subId;
+        request.noTags = 0;
+        request.oneTag = 0;
+        request.twoTags = 1;
+        request.dot1Ad = 0;
+        request.exactMatch = 1;
+        request.defaultSub = 0;
+        request.outerVlanIdAny = 0;
+        request.innerVlanIdAny = 1;
+        request.outerVlanId = 100;
+        request.innerVlanId = 0;
+        return request;
+    }
+
+    private static void print(CreateSubifReply reply) {
+        System.out.printf("CreateSubifReply: %s%n", reply);
+    }
+
+    private static void testCreateSubInterface() throws Exception {
+        System.out.println("Testing sub-interface creation using Java callback API");
+        try (final JVppRegistry registry = new JVppRegistryImpl("CreateSubInterface");
+             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
+            System.out.println("Successfully connected to VPP");
+            Thread.sleep(1000);
+
+            final String ifaceName = "GigabitEthernet0/8/0";
+
+            final SwInterfaceDetailsReplyDump swInterfaceDetails =
+                jvppFacade.swInterfaceDump(createSwInterfaceDumpRequest(ifaceName)).toCompletableFuture().get();
+
+            requireNonNull(swInterfaceDetails, "swInterfaceDump returned null");
+            requireNonNull(swInterfaceDetails.swInterfaceDetails, "swInterfaceDetails is null");
+            requireSingleIface(swInterfaceDetails, ifaceName);
+
+            final int swIfIndex = swInterfaceDetails.swInterfaceDetails.get(0).swIfIndex;
+            final int subId = 1;
+
+            final CreateSubifReply createSubifReply =
+                jvppFacade.createSubif(createSubifRequest(swIfIndex, subId)).toCompletableFuture().get();
+            print(createSubifReply);
+
+            final String subIfaceName = "GigabitEthernet0/8/0." + subId;
+            final SwInterfaceDetailsReplyDump subIface =
+                jvppFacade.swInterfaceDump(createSwInterfaceDumpRequest(subIfaceName)).toCompletableFuture().get();
+            requireNonNull(swInterfaceDetails, "swInterfaceDump returned null");
+            requireNonNull(subIface.swInterfaceDetails, "swInterfaceDump returned null");
+            requireSingleIface(swInterfaceDetails, ifaceName);
+
+            System.out.println("Disconnecting...");
+        }
+        Thread.sleep(1000);
+    }
+
+    public static void main(String[] args) throws Exception {
+        testCreateSubInterface();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java
new file mode 100644
index 00000000..9efeae19
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.core.test;
+
+import static io.fd.vpp.jvpp.core.test.NotificationUtils.getChangeInterfaceState;
+import static io.fd.vpp.jvpp.core.test.NotificationUtils.getDisableInterfaceNotificationsReq;
+import static io.fd.vpp.jvpp.core.test.NotificationUtils.getEnableInterfaceNotificationsReq;
+
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.core.JVppCoreImpl;
+import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
+
+public class FutureApiNotificationTest {
+
+    private static void testFutureApi() throws Exception {
+        System.out.println("Testing Java future API for notifications");
+        try (final JVppRegistry registry = new JVppRegistryImpl("FutureApiNotificationTest");
+             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl());
+             final AutoCloseable notificationListenerReg =
+                 jvppFacade.getNotificationRegistry()
+                     .registerSwInterfaceSetFlagsNotificationCallback(NotificationUtils::printNotification)) {
+            System.out.println("Successfully connected to VPP");
+            jvppFacade.wantInterfaceEvents(getEnableInterfaceNotificationsReq()).toCompletableFuture().get();
+            System.out.println("Interface events started");
+
+            System.out.println("Changing interface configuration");
+            jvppFacade.swInterfaceSetFlags(getChangeInterfaceState()).toCompletableFuture().get();
+
+            Thread.sleep(1000);
+
+            jvppFacade.wantInterfaceEvents(getDisableInterfaceNotificationsReq()).toCompletableFuture().get();
+            System.out.println("Interface events stopped");
+            System.out.println("Disconnecting...");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testFutureApi();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java
new file mode 100644
index 00000000..f478bab4
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.core.test;
+
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.core.JVppCoreImpl;
+import io.fd.vpp.jvpp.core.dto.BridgeDomainDetailsReplyDump;
+import io.fd.vpp.jvpp.core.dto.BridgeDomainDump;
+import io.fd.vpp.jvpp.core.dto.GetNodeIndex;
+import io.fd.vpp.jvpp.core.dto.GetNodeIndexReply;
+import io.fd.vpp.jvpp.core.dto.ShowVersion;
+import io.fd.vpp.jvpp.core.dto.ShowVersionReply;
+import io.fd.vpp.jvpp.core.dto.SwInterfaceDetails;
+import io.fd.vpp.jvpp.core.dto.SwInterfaceDetailsReplyDump;
+import io.fd.vpp.jvpp.core.dto.SwInterfaceDump;
+import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class FutureApiTest {
+
+    private static final Logger LOG = Logger.getLogger(FutureApiTest.class.getName());
+
+    private static void testShowVersion(final FutureJVppCoreFacade jvpp) throws Exception {
+        LOG.info("Sending ShowVersion request...");
+        final Future replyFuture = jvpp.showVersion(new ShowVersion()).toCompletableFuture();
+        final ShowVersionReply reply = replyFuture.get();
+        LOG.info(
+            String.format(
+                "Received ShowVersionReply: context=%d, program=%s, version=%s, buildDate=%s, buildDirectory=%s%n",
+                reply.context, new String(reply.program), new String(reply.version), new String(reply.buildDate),
+                new String(reply.buildDirectory)));
+    }
+
+    private static void testEmptyBridgeDomainDump(final FutureJVppCoreFacade jvpp) throws Exception {
+        LOG.info("Sending ShowVersion request...");
+        final BridgeDomainDump request = new BridgeDomainDump();
+        request.bdId = -1; // dump call
+
+        final CompletableFuture
+            replyFuture = jvpp.bridgeDomainDump(request).toCompletableFuture();
+        final BridgeDomainDetailsReplyDump reply = replyFuture.get();
+
+        if (reply == null || reply.bridgeDomainDetails == null) {
+            LOG.severe("Received null response for empty dump: " + reply);
+        } else {
+            LOG.info(
+                String.format(
+                    "Received empty bridge-domain dump reply with list of bridge-domains: %s, %s",
+                    reply.bridgeDomainDetails, reply.bridgeDomainSwIfDetails));
+        }
+    }
+
+    private static void testGetNodeIndex(final FutureJVppCoreFacade jvpp) {
+        LOG.info("Sending GetNodeIndex request...");
+        final GetNodeIndex request = new GetNodeIndex();
+        request.nodeName = "non-existing-node".getBytes();
+        final Future replyFuture = jvpp.getNodeIndex(request).toCompletableFuture();
+        try {
+            final GetNodeIndexReply reply = replyFuture.get();
+            LOG.info(
+                String.format(
+                    "Received GetNodeIndexReply: context=%d, nodeIndex=%d%n", reply.context, reply.nodeIndex));
+        } catch (Exception e) {
+            LOG.log(Level.SEVERE, "GetNodeIndex request failed", e);
+        }
+    }
+
+    private static void testSwInterfaceDump(final FutureJVppCoreFacade jvpp) throws Exception {
+        LOG.info("Sending SwInterfaceDump request...");
+        final SwInterfaceDump request = new SwInterfaceDump();
+        request.nameFilterValid = 0;
+        request.nameFilter = "".getBytes();
+
+        final Future replyFuture = jvpp.swInterfaceDump(request).toCompletableFuture();
+        final SwInterfaceDetailsReplyDump reply = replyFuture.get();
+        for (SwInterfaceDetails details : reply.swInterfaceDetails) {
+            Objects.requireNonNull(details, "reply.swInterfaceDetails contains null element!");
+            LOG.info(
+                String.format("Received SwInterfaceDetails: interfaceName=%s, l2AddressLength=%d, adminUpDown=%d, "
+                        + "linkUpDown=%d, linkSpeed=%d, linkMtu=%d%n",
+                    new String(details.interfaceName), details.l2AddressLength, details.adminUpDown,
+                    details.linkUpDown, details.linkSpeed, (int) details.linkMtu));
+        }
+    }
+
+    private static void testFutureApi() throws Exception {
+        LOG.info("Testing Java future API");
+        try (final JVppRegistry registry = new JVppRegistryImpl("FutureApiTest");
+             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
+            LOG.info("Successfully connected to VPP");
+
+            testEmptyBridgeDomainDump(jvppFacade);
+            testShowVersion(jvppFacade);
+            testGetNodeIndex(jvppFacade);
+            testSwInterfaceDump(jvppFacade);
+
+            LOG.info("Disconnecting...");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testFutureApi();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java
new file mode 100644
index 00000000..6b3fa993
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.core.test;
+
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.core.JVppCoreImpl;
+import io.fd.vpp.jvpp.core.dto.ClassifyAddDelSession;
+import io.fd.vpp.jvpp.core.dto.ClassifyAddDelSessionReply;
+import io.fd.vpp.jvpp.core.dto.ClassifyAddDelTable;
+import io.fd.vpp.jvpp.core.dto.ClassifyAddDelTableReply;
+import io.fd.vpp.jvpp.core.dto.ClassifySessionDetailsReplyDump;
+import io.fd.vpp.jvpp.core.dto.ClassifySessionDump;
+import io.fd.vpp.jvpp.core.dto.ClassifyTableByInterface;
+import io.fd.vpp.jvpp.core.dto.ClassifyTableByInterfaceReply;
+import io.fd.vpp.jvpp.core.dto.ClassifyTableIds;
+import io.fd.vpp.jvpp.core.dto.ClassifyTableIdsReply;
+import io.fd.vpp.jvpp.core.dto.ClassifyTableInfo;
+import io.fd.vpp.jvpp.core.dto.ClassifyTableInfoReply;
+import io.fd.vpp.jvpp.core.dto.InputAclSetInterface;
+import io.fd.vpp.jvpp.core.dto.InputAclSetInterfaceReply;
+import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
+import javax.xml.bind.DatatypeConverter;
+
+/**
+ * 
Tests L2 ACL creation and read.
 Equivalent to the following vppctl commands:

+ *
+ * 
{@code
+ * vppctl classify table mask l2 src
+ * vppctl classify session acl-hit-next deny opaque-index 0 table-index 0 match l2 src 01:02:03:04:05:06
+ * vppctl set int input acl intfc local0 l2-table 0
+ * vppctl sh class table verbose
+ * }
+ * 

+ */
+public class L2AclTest {
+
+    private static final int LOCAL0_IFACE_ID = 0;
+
+    private static ClassifyAddDelTable createClassifyTable() {
+        ClassifyAddDelTable request = new ClassifyAddDelTable();
+        request.isAdd = 1;
+        request.tableIndex = ~0; // default
+        request.nbuckets = 2;
+        request.memorySize = 2 << 20;
+        request.nextTableIndex = ~0; // default
+        request.missNextIndex = ~0; // default
+        request.skipNVectors = 0;
+        request.matchNVectors = 1;
+        request.mask =
+            new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
+                (byte) 0xff, (byte) 0xff, 0x00, 0x00, 0x00, 0x00};
+        return request;
+    }
+
+    private static ClassifyTableInfo createClassifyTableInfoRequest(final int tableId) {
+        ClassifyTableInfo request = new ClassifyTableInfo();
+        request.tableId = tableId;
+        return request;
+    }
+
+    private static ClassifyAddDelSession createClassifySession(final int tableIndex) {
+        ClassifyAddDelSession request = new ClassifyAddDelSession();
+        request.isAdd = 1;
+        request.tableIndex = tableIndex;
+        request.hitNextIndex = 0; // deny
+        request.opaqueIndex = 0;
+        request.advance = 0; // default
+        // match 01:02:03:04:05:06 mac address
+        request.match =
+            new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04,
+                (byte) 0x05, (byte) 0x06, 0x00, 0x00, 0x00, 0x00};
+        return request;
+    }
+
+    private static ClassifySessionDump createClassifySessionDumpRequest(final int newTableIndex) {
+        ClassifySessionDump request = new ClassifySessionDump();
+        request.tableId = newTableIndex;
+        return request;
+    }
+
+    private static InputAclSetInterface aclSetInterface() {
+        InputAclSetInterface request = new InputAclSetInterface();
+        request.isAdd = 1;
+        request.swIfIndex = LOCAL0_IFACE_ID;
+        request.ip4TableIndex = ~0; // skip
+        request.ip6TableIndex = ~0; // skip
+        request.l2TableIndex = 0;
+        return request;
+    }
+
+    private static ClassifyTableByInterface createClassifyTableByInterfaceRequest() {
+        ClassifyTableByInterface request = new ClassifyTableByInterface();
+        request.swIfIndex = LOCAL0_IFACE_ID;
+        return request;
+    }
+
+    private static void print(ClassifyAddDelTableReply reply) {
+        System.out.printf("ClassifyAddDelTableReply: %s%n", reply);
+    }
+
+    private static void print(ClassifyTableIdsReply reply) {
+        System.out.printf("ClassifyTableIdsReply: %s%n", reply);
+    }
+
+    private static void print(final ClassifyTableInfoReply reply) {
+        System.out.println(reply);
+        if (reply != null) {
+            System.out.println("Mask hex: " + DatatypeConverter.printHexBinary(reply.mask));
+        }
+    }
+
+    private static void print(ClassifyAddDelSessionReply reply) {
+        System.out.printf("ClassifyAddDelSessionReply: context=%s%n", reply);
+    }
+
+    private static void print(final ClassifySessionDetailsReplyDump reply) {
+        System.out.println(reply);
+        reply.classifySessionDetails.forEach(detail -> {
+            System.out.println(detail);
+            System.out.println("Match hex: " + DatatypeConverter.printHexBinary(detail.match));
+        });
+    }
+
+    private static void print(final InputAclSetInterfaceReply reply) {
+        System.out.printf("InputAclSetInterfaceReply: context=%s%n", reply);
+    }
+
+    private static void print(final ClassifyTableByInterfaceReply reply) {
+        System.out.printf("ClassifyAddDelTableReply: %s%n", reply);
+    }
+
+    private static void testL2Acl() throws Exception {
+        System.out.println("Testing L2 ACLs using Java callback API");
+        try (final JVppRegistry registry = new JVppRegistryImpl("L2AclTest");
+             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
+
+            System.out.println("Successfully connected to VPP");
+            Thread.sleep(1000);
+
+            final ClassifyAddDelTableReply classifyAddDelTableReply =
+                jvppFacade.classifyAddDelTable(createClassifyTable()).toCompletableFuture().get();
+            print(classifyAddDelTableReply);
+
+            final ClassifyTableIdsReply classifyTableIdsReply =
+                jvppFacade.classifyTableIds(new ClassifyTableIds()).toCompletableFuture().get();
+            print(classifyTableIdsReply);
+
+            final ClassifyTableInfoReply classifyTableInfoReply =
+                jvppFacade.classifyTableInfo(createClassifyTableInfoRequest(classifyAddDelTableReply.newTableIndex))
+                    .toCompletableFuture().get();
+            print(classifyTableInfoReply);
+
+            final ClassifyAddDelSessionReply classifyAddDelSessionReply =
+                jvppFacade.classifyAddDelSession(createClassifySession(classifyAddDelTableReply.newTableIndex))
+                    .toCompletableFuture().get();
+            print(classifyAddDelSessionReply);
+
+            final ClassifySessionDetailsReplyDump classifySessionDetailsReplyDump =
+                jvppFacade.classifySessionDump(createClassifySessionDumpRequest(classifyAddDelTableReply.newTableIndex))
+                    .toCompletableFuture().get();
+            print(classifySessionDetailsReplyDump);
+
+            final InputAclSetInterfaceReply inputAclSetInterfaceReply =
+                jvppFacade.inputAclSetInterface(aclSetInterface()).toCompletableFuture().get();
+            print(inputAclSetInterfaceReply);
+
+            final ClassifyTableByInterfaceReply classifyTableByInterfaceReply =
+                jvppFacade.classifyTableByInterface(createClassifyTableByInterfaceRequest()).toCompletableFuture()
+                    .get();
+            print(classifyTableByInterfaceReply);
+
+            System.out.println("Disconnecting...");
+        }
+        Thread.sleep(1000);
+    }
+
+    public static void main(String[] args) throws Exception {
+        testL2Acl();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java
new file mode 100644
index 00000000..d7f5039b
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.core.test;
+
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.core.JVppCoreImpl;
+import io.fd.vpp.jvpp.core.dto.LispAddDelAdjacency;
+import io.fd.vpp.jvpp.core.dto.LispAddDelLocalEid;
+import io.fd.vpp.jvpp.core.dto.LispAddDelLocatorSet;
+import io.fd.vpp.jvpp.core.dto.LispAddDelRemoteMapping;
+import io.fd.vpp.jvpp.core.dto.LispAdjacenciesGet;
+import io.fd.vpp.jvpp.core.dto.LispAdjacenciesGetReply;
+import io.fd.vpp.jvpp.core.dto.LispEnableDisable;
+import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ExecutionException;
+import java.util.logging.Logger;
+
+/**
+ * Tests lisp adjacency creation and read (custom vpe.api type support showcase).
+ */
+public class LispAdjacencyTest {
+
+    private static final Logger LOG = Logger.getLogger(LispAdjacencyTest.class.getName());
+
+    private static void enableLisp(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
+        final LispEnableDisable request = new LispEnableDisable();
+        request.isEn = 1;
+        jvpp.lispEnableDisable(request).toCompletableFuture().get();
+        LOG.info("Lisp enabled successfully");
+    }
+
+    private static void addLocatorSet(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
+        final LispAddDelLocatorSet request = new LispAddDelLocatorSet();
+        request.isAdd = 1;
+        request.locatorSetName = "ls1".getBytes(StandardCharsets.UTF_8);
+        jvpp.lispAddDelLocatorSet(request).toCompletableFuture().get();
+        LOG.info("Locator set created successfully:" + request.toString());
+    }
+
+    private static void addLocalEid(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
+        final LispAddDelLocalEid request = new LispAddDelLocalEid();
+        request.isAdd = 1;
+        request.locatorSetName = "ls1".getBytes(StandardCharsets.UTF_8);
+        request.eid = new byte[] {1, 2, 1, 10};
+        request.eidType = 0; // ip4
+        request.vni = 0;
+        request.prefixLen = 32;
+        jvpp.lispAddDelLocalEid(request).toCompletableFuture().get();
+        LOG.info("Local EID created successfully:" + request.toString());
+    }
+
+    private static void addRemoteMapping(final FutureJVppCoreFacade jvpp)
+        throws ExecutionException, InterruptedException {
+        final LispAddDelRemoteMapping request = new LispAddDelRemoteMapping();
+        request.isAdd = 1;
+        request.vni = 0;
+        request.eid = new byte[] {1, 2, 1, 20};
+        request.eidLen = 32;
+        request.rlocNum = 1;
+        request.rlocs = new byte[] {1, 1, 1, 1, 2, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+        jvpp.lispAddDelRemoteMapping(request).toCompletableFuture().get();
+        LOG.info("Remote mapping created successfully:" + request.toString());
+    }
+
+    private static void addAdjacency(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
+        final LispAddDelAdjacency request = new LispAddDelAdjacency();
+        request.isAdd = 1;
+        request.leid = new byte[] {1, 2, 1, 10};
+        request.leidLen = 32;
+        request.reid = new byte[] {1, 2, 1, 20};
+        request.reidLen = 32;
+        request.eidType = 0; // ip4
+        request.vni = 0;
+        jvpp.lispAddDelAdjacency(request).toCompletableFuture().get();
+        LOG.info("Lisp adjacency created successfully:" + request.toString());
+    }
+
+    private static void showAdjacencies(final FutureJVppCoreFacade jvpp)
+        throws ExecutionException, InterruptedException {
+        final LispAdjacenciesGetReply reply =
+            jvpp.lispAdjacenciesGet(new LispAdjacenciesGet()).toCompletableFuture().get();
+        LOG.info("Lisp adjacency received successfully:" + reply.toString());
+    }
+
+    private static void testAdjacency(final FutureJVppCoreFacade jvpp) throws Exception {
+        enableLisp(jvpp);
+        addLocatorSet(jvpp);
+        addLocalEid(jvpp);
+        addRemoteMapping(jvpp);
+        addAdjacency(jvpp);
+        showAdjacencies(jvpp);
+    }
+
+    private static void testFutureApi() throws Exception {
+        LOG.info("Create lisp adjacency test");
+        try (final JVppRegistry registry = new JVppRegistryImpl("LispAdjacencyTest");
+             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
+            LOG.info("Successfully connected to VPP");
+
+            testAdjacency(jvppFacade);
+            LOG.info("Disconnecting...");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testFutureApi();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java
new file mode 100644
index 00000000..f82946c3
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.core.test;
+
+import java.io.PrintStream;
+import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlags;
+import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlagsNotification;
+import io.fd.vpp.jvpp.core.dto.WantInterfaceEvents;
+
+final class NotificationUtils {
+
+    private NotificationUtils() {}
+
+    static PrintStream printNotification(final SwInterfaceSetFlagsNotification msg) {
+        return System.out.printf("Received interface notification: ifc: %s%n", msg);
+    }
+
+    static SwInterfaceSetFlags getChangeInterfaceState() {
+        final SwInterfaceSetFlags swInterfaceSetFlags = new SwInterfaceSetFlags();
+        swInterfaceSetFlags.swIfIndex = 0;
+        swInterfaceSetFlags.adminUpDown = 1;
+        swInterfaceSetFlags.deleted = 0;
+        return swInterfaceSetFlags;
+    }
+
+    static WantInterfaceEvents getEnableInterfaceNotificationsReq() {
+        WantInterfaceEvents wantInterfaceEvents = new WantInterfaceEvents();
+        wantInterfaceEvents.pid = 1;
+        wantInterfaceEvents.enableDisable = 1;
+        return wantInterfaceEvents;
+    }
+
+    static WantInterfaceEvents getDisableInterfaceNotificationsReq() {
+        WantInterfaceEvents wantInterfaceEvents = new WantInterfaceEvents();
+        wantInterfaceEvents.pid = 1;
+        wantInterfaceEvents.enableDisable = 0;
+        return wantInterfaceEvents;
+    }
+}
diff --git a/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt
new file mode 100644
index 00000000..1344dc9e
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt
@@ -0,0 +1,17 @@
+This package contains basic tests for jvpp. To run the tests:
+
+- Make sure VPP is running
+- From VPP's build-root/ folder execute:
+  - sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/vpp-api/java/jvpp-core-17.01.jar io.fd.vpp.jvpp.core.test.[test name]
+
+Available tests:
+CallbackApiTest - Similar to ControlPingTest, invokes more complex calls (e.g. interface dump) using low level JVpp APIs
+CallbackJVppFacadeNotificationTest - Tests interface notifications using Callback based JVpp facade
+CallbackJVppFacadeTest - Execution of more complex calls using Callback based JVpp facade
+CallbackNotificationApiTest - Tests interface notifications using low level JVpp APIs
+ControlPingTest - Simple test executing a single control ping using low level JVpp APIs
+CreateSubInterfaceTest - Tests sub-interface creation
+FutureApiNotificationTest - Tests interface notifications using Future based JVpp facade
+FutureApiTest - Execution of more complex calls using Future based JVpp facade
+L2AclTest - Tests L2 ACL creation
+LispAdjacencyTest - Tests lisp adjacency creation and read (custom vpe.api type support showcase)
diff --git a/src/vpp-api/java/jvpp-core/jvpp_core.c b/src/vpp-api/java/jvpp-core/jvpp_core.c
new file mode 100644
index 00000000..ef4cb8e3
--- /dev/null
+++ b/src/vpp-api/java/jvpp-core/jvpp_core.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+// TODO: generate jvpp_plugin_name.c files (or at least reuse plugin's main structure)
+typedef struct {
+    /* Base message index for the jvpp-core plugin */
+    u16 msg_id_base;
+
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} core_main_t;
+
+core_main_t core_main __attribute__((aligned (64)));
+
+#include "io_fd_vpp_jvpp_core_JVppCoreImpl.h"
+#include "jvpp_core_gen.h"
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_core_JVppCoreImpl_init0
+(JNIEnv * env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+    core_main_t * plugin_main = &core_main;
+    plugin_main->my_client_index = my_client_index;
+    plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+    plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+    plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+    #define _(N,n)                                  \
+        vl_msg_api_set_handlers(VL_API_##N, #n,     \
+                vl_api_##n##_t_handler,             \
+                vl_noop_handler,                    \
+                vl_noop_handler,              \
+                vl_noop_handler,               \
+                sizeof(vl_api_##n##_t), 1);
+        foreach_api_reply_handler;
+    #undef _
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_core_JVppCoreImpl_close0
+(JNIEnv *env, jclass clazz) {
+    core_main_t * plugin_main = &core_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
+
+
+
diff --git a/src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java b/src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java
new file mode 100644
index 00000000..cb85f005
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/IoamExportApiTest.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.ioamexport.test;
+
+import java.net.InetAddress;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.VppCallbackException;
+import io.fd.vpp.jvpp.ioamexport.JVppIoamexportImpl;
+import io.fd.vpp.jvpp.ioamexport.future.FutureJVppIoamexportFacade;
+import io.fd.vpp.jvpp.ioamexport.dto.IoamExportIp6EnableDisable;
+import io.fd.vpp.jvpp.ioamexport.dto.IoamExportIp6EnableDisableReply;
+
+public class IoamExportApiTest {
+
+    public static void main(String[] args) throws Exception {
+        ioamExportTestApi();
+    }
+
+    private static void ioamExportTestApi() throws Exception {
+        System.out.println("Testing Java API for ioam export plugin");
+        try (final JVppRegistry registry = new JVppRegistryImpl("ioamExportApiTest");
+             final JVpp jvpp = new JVppIoamexportImpl()) {
+	    FutureJVppIoamexportFacade ioamexportJvpp = new FutureJVppIoamexportFacade(registry,jvpp);
+            System.out.println("Sending ioam export request...");
+	    IoamExportIp6EnableDisable request = new IoamExportIp6EnableDisable();
+            request.isDisable = 0;
+	    InetAddress collectorAddress = InetAddress.getByName("2001:0DB8:AC10:FE01:0000:0000:0000:0000");
+	    InetAddress srcAddress = InetAddress.getByName("2001:0DB8:AC10:FE01:0000:0000:0000:0001");
+	    request.collectorAddress = collectorAddress.getAddress();
+	    request.srcAddress = srcAddress.getAddress();
+	    IoamExportIp6EnableDisableReply reply = ioamexportJvpp.ioamExportIp6EnableDisable(request).toCompletableFuture().get();
+            System.out.printf("IoamExportIp6EnableDisableReply = "+reply.toString()+"%n");
+
+            Thread.sleep(1000);
+
+            System.out.println("Disconnecting...");
+        }
+    }
+}
diff --git a/src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/Readme.txt b/src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/Readme.txt
new file mode 100644
index 00000000..1b38c285
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamexport/io/fd/vpp/jvpp/ioamexport/test/Readme.txt
@@ -0,0 +1 @@
+sudo java -cp build-vpp_debug-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp_debug-native/plugins/ioam-plugin/jvpp-ioam-export-1.0.jar io.fd.vpp.jvpp.ioamexport.test.IoamExportApiTest
diff --git a/src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.c b/src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.c
new file mode 100644
index 00000000..5cda89d1
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-ioamexport/io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl.h"
+#include "jvpp_ioam_export.h"
+#include "jvpp-ioamexport/jvpp_ioamexport_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  ioamexport_main_t * plugin_main = &ioamexport_main;
+  u8 * name;
+  clib_warning ("Java_io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+  name = format (0, "ioam_export_%08x%c", api_version, 0);
+  plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (plugin_main->msg_id_base == (u16) ~0) {
+    jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException");
+    (*env)->ThrowNew(env, exClass, "ioam_export plugin is not loaded in VPP");
+  } else {
+    plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+    plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+    #define _(N,n)                                  \
+        vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n,     \
+                vl_api_##n##_t_handler,             \
+                vl_noop_handler,                    \
+                vl_api_##n##_t_endian,              \
+                vl_api_##n##_t_print,               \
+                sizeof(vl_api_##n##_t), 1);
+        foreach_api_reply_handler;
+    #undef _
+  }
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamexport_JVppIoamexportImpl_close0
+(JNIEnv *env, jclass clazz) {
+  ioamexport_main_t * plugin_main = &ioamexport_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP iOAM EXPORT */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP iOAM EXPORT */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.h b/src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.h
new file mode 100644
index 00000000..b6c0c16e
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamexport/jvpp_ioam_export.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_ioam_export_h__
+#define __included_jvpp_ioam_export_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-IOAM-EXPORT */
+typedef struct {
+    /* Base message index for the export plugin */
+    u16 msg_id_base;
+
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} ioamexport_main_t;
+
+ioamexport_main_t ioamexport_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_ioam_export_h__ */
diff --git a/src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java b/src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java
new file mode 100644
index 00000000..74eb86a1
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/IoamPotApiTest.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.ioampot.test;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.VppCallbackException;
+import io.fd.vpp.jvpp.ioampot.JVppIoampotImpl;
+import io.fd.vpp.jvpp.ioampot.callback.PotProfileAddCallback;
+import io.fd.vpp.jvpp.ioampot.dto.PotProfileAdd;
+import io.fd.vpp.jvpp.ioampot.dto.PotProfileAddReply;
+
+public class IoamPotApiTest {
+
+    static class IoamPotTestCallback implements PotProfileAddCallback {
+
+        @Override
+        public void onPotProfileAddReply(final PotProfileAddReply reply) {
+            System.out.printf("Received PotProfileAddReply reply: context=%d%n",
+                reply.context);
+        }
+
+        @Override
+        public void onError(VppCallbackException ex) {
+            System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(),
+                ex.getCtxId(), ex.getErrorCode());
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        ioamPotTestApi();
+    }
+
+    private static void ioamPotTestApi() throws Exception {
+        System.out.println("Testing Java API for ioam pot plugin");
+        try (final JVppRegistry registry = new JVppRegistryImpl("ioamPotApiTest");
+             final JVpp jvpp = new JVppIoampotImpl()) {
+            registry.register(jvpp, new IoamPotTestCallback());
+
+            System.out.println("Sending ioam pot profile add request...");
+            PotProfileAdd request = new PotProfileAdd();
+            request.id = 0;
+            request.validator = 4;
+            request.secretKey = 1;
+            request.secretShare = 2;
+            request.prime = 1234;
+            request.maxBits = 53;
+            request.lpc = 1234;
+            request.polynomialPublic = 1234;
+            request.listNameLen = (byte)"test pot profile".getBytes().length;
+            request.listName = "test pot profile".getBytes();
+            final int result = jvpp.send(request);
+            System.out.printf("PotProfileAdd send result = %d%n", result);
+
+            Thread.sleep(1000);
+
+            System.out.println("Disconnecting...");
+        }
+    }
+}
diff --git a/src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/Readme.txt b/src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/Readme.txt
new file mode 100644
index 00000000..2323494d
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioampot/io/fd/vpp/jvpp/ioampot/test/Readme.txt
@@ -0,0 +1 @@
+sudo java -cp build-vpp_debug-native/vpp-api/java/jvpp-registry-16.12.jar:build-vpp_debug-native/plugins/ioam-plugin/jvpp-ioam-pot-1.0.jar io.fd.vpp.jvpp.ioampot.test.IoamPotApiTest
diff --git a/src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.c b/src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.c
new file mode 100644
index 00000000..9291dbba
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-ioampot/io_fd_vpp_jvpp_ioampot_JVppIoampotImpl.h"
+#include "jvpp_ioam_pot.h"
+#include "jvpp-ioampot/jvpp_ioampot_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_ioampot_JVppIoampotImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioampot_JVppIoampotImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  ioampot_main_t * plugin_main = &ioampot_main;
+  u8 * name;
+  clib_warning ("Java_io_fd_vpp_jvpp_ioampot_JVppIoampotImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+  name = format (0, "ioam_pot_%08x%c", api_version, 0);
+  plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (plugin_main->msg_id_base == (u16) ~0) {
+    jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException");
+    (*env)->ThrowNew(env, exClass, "ioam_pot plugin is not loaded in VPP");
+  } else {
+    plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+    plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+    #define _(N,n)                                  \
+        vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n,     \
+                vl_api_##n##_t_handler,             \
+                vl_noop_handler,                    \
+                vl_api_##n##_t_endian,              \
+                vl_api_##n##_t_print,               \
+                sizeof(vl_api_##n##_t), 1);
+        foreach_api_reply_handler;
+    #undef _
+  }
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioampot_JVppIoampotImpl_close0
+(JNIEnv *env, jclass clazz) {
+  ioampot_main_t * plugin_main = &ioampot_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP iOAM POT */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP iOAM POT */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.h b/src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.h
new file mode 100644
index 00000000..00aa51db
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioampot/jvpp_ioam_pot.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_ioam_pot_h__
+#define __included_jvpp_ioam_pot_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-IOAM-POT */
+typedef struct {
+    /* Base message index for the pot plugin */
+    u16 msg_id_base;
+
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} ioampot_main_t;
+
+ioampot_main_t ioampot_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_ioam_pot_h__ */
diff --git a/src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java b/src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java
new file mode 100644
index 00000000..bc8c1c3a
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/IoamTraceApiTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.ioamtrace.test;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.VppCallbackException;
+import io.fd.vpp.jvpp.ioamtrace.future.FutureJVppIoamtraceFacade;
+import io.fd.vpp.jvpp.ioamtrace.JVppIoamtraceImpl;
+import io.fd.vpp.jvpp.ioamtrace.callback.TraceProfileAddCallback;
+import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileAdd;
+import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileAddReply;
+import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileShowConfig;
+import io.fd.vpp.jvpp.ioamtrace.dto.TraceProfileShowConfigReply;
+
+public class IoamTraceApiTest {
+
+    static class IoamTraceTestCallback implements TraceProfileAddCallback {
+
+        @Override
+        public void onTraceProfileAddReply(final TraceProfileAddReply reply) {
+            System.out.printf("Received TraceProfileAddReply reply: context=%d%n",
+                reply.context);
+        }
+
+        @Override
+        public void onError(VppCallbackException ex) {
+            System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(),
+                ex.getCtxId(), ex.getErrorCode());
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        ioamTraceTestApi();
+    }
+
+    private static void ioamTraceTestApi() throws Exception {
+        System.out.println("Testing Java API for ioam trace plugin");
+        try (final JVppRegistry registry = new JVppRegistryImpl("ioamTraceApiTest");
+             final JVpp jvpp = new JVppIoamtraceImpl()) {
+	    FutureJVppIoamtraceFacade ioamtraceJvpp = new FutureJVppIoamtraceFacade(registry,jvpp);
+
+            System.out.println("Sending ioam trace profile add request...");
+            TraceProfileAdd request = new TraceProfileAdd();
+            request.traceType = 0x1f;
+            request.numElts = 4;
+            request.nodeId = 1;
+            request.traceTsp = 2;
+            request.appData = 1234;
+            final int result = jvpp.send(request);
+            System.out.printf("TraceProfileAdd send result = %d%n", result);
+
+            Thread.sleep(1000);
+
+	    TraceProfileShowConfig showRequest = new TraceProfileShowConfig();
+	    TraceProfileShowConfigReply reply = ioamtraceJvpp.traceProfileShowConfig(showRequest).toCompletableFuture().get();
+            System.out.printf("TraceProfileShowConfig result = "+ reply.toString());
+
+            System.out.println("Disconnecting...");
+        }
+    }
+}
diff --git a/src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt b/src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt
new file mode 100644
index 00000000..17e45a81
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamtrace/io/fd/vpp/jvpp/ioamtrace/test/Readme.txt
@@ -0,0 +1 @@
+sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/plugins/ioam-plugin/jvpp-ioam-trace-1.0.jar io.fd.vpp.jvpp.ioamtrace.test.IoamTraceApiTest
diff --git a/src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.c b/src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.c
new file mode 100644
index 00000000..0bf17889
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-ioamtrace/io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl.h"
+#include "jvpp_ioam_trace.h"
+#include "jvpp-ioamtrace/jvpp_ioamtrace_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  ioamtrace_main_t * plugin_main = &ioamtrace_main;
+  u8 * name;
+  clib_warning ("Java_io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+  name = format (0, "ioam_trace_%08x%c", api_version, 0);
+  plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (plugin_main->msg_id_base == (u16) ~0) {
+    jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException");
+    (*env)->ThrowNew(env, exClass, "ioam_trace plugin is not loaded in VPP");
+  } else {
+    plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+    plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+    #define _(N,n)                                  \
+        vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n,     \
+                vl_api_##n##_t_handler,             \
+                vl_noop_handler,                    \
+                vl_api_##n##_t_endian,              \
+                vl_api_##n##_t_print,               \
+                sizeof(vl_api_##n##_t), 1);
+        foreach_api_reply_handler;
+    #undef _
+  }
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_ioamtrace_JVppIoamtraceImpl_close0
+(JNIEnv *env, jclass clazz) {
+  ioamtrace_main_t * plugin_main = &ioamtrace_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP iOAM Trace */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP iOAM Trace */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.h b/src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.h
new file mode 100644
index 00000000..9fc16c15
--- /dev/null
+++ b/src/vpp-api/java/jvpp-ioamtrace/jvpp_ioam_trace.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_ioam_trace_h__
+#define __included_jvpp_ioam_trace_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-IOAM-TRACE */
+typedef struct {
+    /* Base message index for the trace plugin */
+    u16 msg_id_base;
+
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} ioamtrace_main_t;
+
+ioamtrace_main_t ioamtrace_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_ioam_trace_h__ */
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java
new file mode 100644
index 00000000..55f25a7b
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+import io.fd.vpp.jvpp.callback.JVppCallback;
+import io.fd.vpp.jvpp.dto.ControlPing;
+import io.fd.vpp.jvpp.dto.JVppRequest;
+
+/**
+ * Base interface for plugin's Java API.
+ */
+public interface JVpp extends AutoCloseable {
+
+    /**
+     * Sends request to vpp.
+     *
+     * @param request request to be sent
+     * @return unique identifer of message in message queue
+     * @throws VppInvocationException when message could not be sent
+     */
+    int send(final JVppRequest request) throws VppInvocationException;
+
+    /**
+     * Initializes plugin's Java API.
+     *
+     * @param registry     plugin registry
+     * @param callback     called by vpe.api message handlers
+     * @param queueAddress address of vpp shared memory queue
+     * @param clientIndex  vpp client identifier
+     */
+    void init(final JVppRegistry registry, final JVppCallback callback, final long queueAddress,
+              final int clientIndex);
+
+    /**
+     * Sends control_ping message.
+     *
+     * @param controlPing request DTO
+     * @return unique identifer of message in message queue
+     * @throws VppInvocationException when message could not be sent
+     */
+    int controlPing(final ControlPing controlPing) throws VppInvocationException;
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java
new file mode 100644
index 00000000..6535db02
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+import io.fd.vpp.jvpp.callback.JVppCallback;
+
+/**
+ * Manages VPP connection and stores plugin callbacks.
+ */
+public interface JVppRegistry extends AutoCloseable {
+
+    /**
+     * Vpp connection managed by the registry.
+     *
+     * @return representation of vpp connection
+     */
+    VppConnection getConnection();
+
+    /**
+     * Registers callback and initializes Java API for given plugin.
+     *
+     * @param jvpp     plugin name
+     * @param callback callback provided by the plugin
+     * @throws NullPointerException     if name or callback is null
+     * @throws IllegalArgumentException if plugin was already registered
+     */
+    void register(final JVpp jvpp, final JVppCallback callback);
+
+    /**
+     * Unregisters callback for the given plugin.
+     *
+     * @param name plugin name
+     * @throws NullPointerException     if name is null
+     * @throws IllegalArgumentException if plugin was not registered
+     */
+    void unregister(final String name);
+
+    /**
+     * Returns callback registered for the plugin.
+     *
+     * @param name plugin name
+     * @return callback provided by the plugin
+     * @throws NullPointerException     if name is null
+     * @throws IllegalArgumentException if plugin was not registered
+     */
+    JVppCallback get(final String name);
+
+    /**
+     * Sends control ping. Reply handler calls callback registered for give plugin.
+     *
+     * Control ping is used for initial RX thread to Java thread attachment
+     * that takes place in the plugin's JNI lib
+     * and to wrap dump message replies in one list.
+     *
+     * VPP plugins don't have to provide special control ping, therefore
+     * it is necessary to providing control ping support in JVppRegistry.
+
+     * @param clazz identifies plugin that should receive ping callback
+     * @return unique identifier of message in message queue
+     */
+    int controlPing(final Class clazz) throws VppInvocationException;
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java
new file mode 100644
index 00000000..98ef1c15
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+import static java.util.Objects.requireNonNull;
+
+import io.fd.vpp.jvpp.callback.ControlPingCallback;
+import io.fd.vpp.jvpp.callback.JVppCallback;
+import io.fd.vpp.jvpp.dto.ControlPingReply;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Default implementation of JVppRegistry.
+ */
+public final class JVppRegistryImpl implements JVppRegistry, ControlPingCallback {
+
+    private static final Logger LOG = Logger.getLogger(JVppRegistryImpl.class.getName());
+
+    private final VppJNIConnection connection;
+    // Unguarded concurrent map, no race conditions expected on top of that
+    private final Map pluginRegistry;
+    // Guarded by self
+    private final Map pingCalls;
+
+    public JVppRegistryImpl(final String clientName) throws IOException {
+        connection = new VppJNIConnection(clientName);
+        connection.connect();
+        pluginRegistry = new ConcurrentHashMap<>();
+        pingCalls = new HashMap<>();
+    }
+
+    @Override
+    public VppConnection getConnection() {
+        return connection;
+    }
+
+    @Override
+    public void register(final JVpp jvpp, final JVppCallback callback) {
+        requireNonNull(jvpp, "jvpp should not be null");
+        requireNonNull(callback, "Callback should not be null");
+        final String name = jvpp.getClass().getName();
+        if (pluginRegistry.containsKey(name)) {
+            throw new IllegalArgumentException(
+                String.format("Callback for plugin %s was already registered", name));
+        }
+        jvpp.init(this, callback, connection.getConnectionInfo().queueAddress,
+            connection.getConnectionInfo().clientIndex);
+        pluginRegistry.put(name, callback);
+    }
+
+    @Override
+    public void unregister(final String name) {
+        requireNonNull(name, "Plugin name should not be null");
+        final JVppCallback previous = pluginRegistry.remove(name);
+        assertPluginWasRegistered(name, previous);
+    }
+
+    @Override
+    public JVppCallback get(final String name) {
+        requireNonNull(name, "Plugin name should not be null");
+        JVppCallback value = pluginRegistry.get(name);
+        assertPluginWasRegistered(name, value);
+        return value;
+    }
+
+    private native int controlPing0() throws VppInvocationException;
+
+    @Override
+    public int controlPing(final Class clazz) throws VppInvocationException {
+        connection.checkActive();
+        final String name = clazz.getName();
+
+        final ControlPingCallback callback = (ControlPingCallback) pluginRegistry.get(clazz.getName());
+        assertPluginWasRegistered(name, callback);
+
+        synchronized (pingCalls) {
+            int context = controlPing0();
+            if (context < 0) {
+                throw new VppInvocationException("controlPing", context);
+            }
+
+            pingCalls.put(context, callback);
+            return context;
+        }
+    }
+
+    @Override
+    public void onControlPingReply(final ControlPingReply reply) {
+        final ControlPingCallback callback;
+        synchronized (pingCalls) {
+            callback = pingCalls.remove(reply.context);
+            if (callback == null) {
+                LOG.log(Level.WARNING, "No callback was registered for reply context=" + reply.context + " Contexts waiting="
+                    + pingCalls.keySet());
+                return;
+            }
+        }
+        // pass the reply to the callback registered by the ping caller
+        callback.onControlPingReply(reply);
+    }
+
+    @Override
+    public void onError(final VppCallbackException ex) {
+        final int ctxId = ex.getCtxId();
+        final ControlPingCallback callback;
+
+        synchronized (pingCalls) {
+            callback = pingCalls.get(ctxId);
+        }
+        if (callback == null) {
+            LOG.log(Level.WARNING, "No callback was registered for reply id={0} ", ctxId);
+            return;
+        }
+        // pass the error to the callback registered by the ping caller
+        callback.onError(ex);
+    }
+
+    private static void assertPluginWasRegistered(final String name, final JVppCallback value) {
+        if (value == null) {
+            throw new IllegalArgumentException(String.format("Callback for plugin %s is not registered", name));
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        connection.close();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java
new file mode 100644
index 00000000..ce6d1bfc
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Utility class for loading JNI libraries.
+ */
+public final class NativeLibraryLoader {
+
+    private static final Logger LOG = Logger.getLogger(NativeLibraryLoader.class.getName());
+
+    private NativeLibraryLoader() {
+        throw new UnsupportedOperationException("This utility class cannot be instantiated.");
+    }
+
+    /**
+     * Loads JNI library using class loader of the given class.
+     *
+     * @param libName name of the library to be loaded
+     */
+    public static void loadLibrary(final String libName, final Class clazz) throws IOException {
+        java.util.Objects.requireNonNull(libName, "libName should not be null");
+        java.util.Objects.requireNonNull(clazz, "clazz should not be null");
+        try (final InputStream is = clazz.getResourceAsStream('/' + libName)) {
+            if (is == null) {
+                throw new IOException("Failed to open library resource " + libName);
+            }
+            loadStream(libName, is);
+        }
+    }
+
+    private static void loadStream(final String libName, final InputStream is) throws IOException {
+        final Set perms = PosixFilePermissions.fromString("rwxr-x---");
+        final Path p = Files.createTempFile(libName, null, PosixFilePermissions.asFileAttribute(perms));
+        try {
+            Files.copy(is, p, StandardCopyOption.REPLACE_EXISTING);
+            Runtime.getRuntime().load(p.toString());
+        } catch (Exception e) {
+            throw new IOException("Failed to load library " + p, e);
+        } finally {
+            try {
+                Files.deleteIfExists(p);
+            } catch (IOException e) {
+                LOG.log(Level.WARNING, String.format("Failed to delete temporary file %s.", p), e);
+            }
+        }
+    }
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java
new file mode 100644
index 00000000..d71e3055
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+/**
+ * Base exception representing failed operation of JVpp request call
+ */
+public abstract class VppBaseCallException extends Exception {
+    private final String methodName;
+    private final int errorCode;
+
+    /**
+     * Constructs an VppCallbackException with the specified api method name and error code.
+     *
+     * @param methodName name of a method, which invocation or execution failed
+     * @param errorCode  negative error code value associated with this failure
+     * @throws NullPointerException     if apiMethodName is null
+     */
+    public VppBaseCallException(final String methodName, final int errorCode) {
+        super(String.format("vppApi.%s failed with error code: %d", methodName, errorCode));
+        this.methodName = java.util.Objects.requireNonNull(methodName, "apiMethodName is null!");
+        this.errorCode = errorCode;
+        if(errorCode >= 0) {
+            throw new IllegalArgumentException("Error code must be < 0. Was " + errorCode +
+                    " for " + methodName );
+        }
+    }
+
+    /**
+     * Returns  name of a method, which invocation failed.
+     *
+     * @return method name
+     */
+    public String getMethodName() {
+        return methodName;
+    }
+
+    /**
+     * Returns the error code associated with this failure.
+     *
+     * @return a negative integer error code
+     */
+    public int getErrorCode() {
+        return errorCode;
+    }
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java
new file mode 100644
index 00000000..ccfcbd3c
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+/**
+ * Callback Exception representing failed operation of JVpp request call
+ */
+public class VppCallbackException extends VppBaseCallException {
+    private final int ctxId;
+
+    /**
+     * Constructs an VppCallbackException with the specified api method name and error code.
+     *
+     * @param methodName  name of a method, which invocation failed.
+     * @param ctxId      api request context identifier
+     * @param errorCode  negative error code value associated with this failure
+     * @throws NullPointerException     if apiMethodName is null
+     */
+    public VppCallbackException(final String methodName, final int ctxId, final int errorCode ){
+        super(methodName, errorCode);
+        this.ctxId = ctxId;
+    }
+
+    /**
+     * Returns api request context identifier.
+     *
+     * @return value of context identifier
+     */
+    public int getCtxId() {
+        return ctxId;
+    }
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java
new file mode 100644
index 00000000..e6fd3bdb
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+import java.io.IOException;
+
+/**
+ * Representation of a management connection to VPP.
+ */
+public interface VppConnection extends AutoCloseable {
+
+    /**
+     * Opens VppConnection for communication with VPP.
+     *
+     * @throws IOException if connection is not established
+     */
+    void connect() throws IOException;
+
+    /**
+     * Checks if this instance connection is active.
+     *
+     * @throws IllegalStateException if this instance was disconnected.
+     */
+    void checkActive();
+
+    /**
+     * Closes Vpp connection.
+     */
+    @Override
+    void close();
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java
new file mode 100644
index 00000000..a7ccb197
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+/**
+ * Exception thrown when Vpp jAPI method invocation failed.
+ */
+public class VppInvocationException extends VppBaseCallException {
+    /**
+     * Constructs an VppApiInvocationFailedException with the specified api method name and error code.
+     *
+     * @param methodName  name of a method, which invocation failed.
+     * @param errorCode  negative error code value associated with this failure
+     * @throws NullPointerException     if apiMethodName is null
+     */
+    public VppInvocationException(final String methodName, final int errorCode) {
+        super(methodName, errorCode);
+    }
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java
new file mode 100644
index 00000000..320c1283
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp;
+
+import static io.fd.vpp.jvpp.NativeLibraryLoader.loadLibrary;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * JNI based representation of a management connection to VPP.
+ */
+public final class VppJNIConnection implements VppConnection {
+    private static final Logger LOG = Logger.getLogger(VppJNIConnection.class.getName());
+
+    static {
+        final String libName = "libjvpp_registry.so";
+        try {
+            loadLibrary(libName, VppJNIConnection.class);
+        } catch (IOException e) {
+            LOG.log(Level.SEVERE, String.format("Can't find vpp jni library: %s", libName), e);
+            throw new ExceptionInInitializerError(e);
+        }
+    }
+
+    private ConnectionInfo connectionInfo;
+
+    private final String clientName;
+    private volatile boolean disconnected = false;
+
+    /**
+     * Create VPPJNIConnection instance for client connecting to VPP.
+     *
+     * @param clientName client name instance to be used for communication. Single connection per clientName is
+     *                   allowed.
+     */
+    public VppJNIConnection(final String clientName) {
+        this.clientName = Objects.requireNonNull(clientName, "Null clientName");
+    }
+
+    /**
+     * Guarded by VppJNIConnection.class
+     */
+    private static final Map connections = new HashMap<>();
+
+    /**
+     * Initiate VPP connection for current instance
+     *
+     * Multiple instances are allowed since this class is not a singleton (VPP allows multiple management connections).
+     *
+     * However only a single connection per clientName is allowed.
+     *
+     * @throws IOException in case the connection could not be established
+     */
+
+    @Override
+    public void connect() throws IOException {
+        _connect();
+    }
+
+    private void _connect() throws IOException {
+        synchronized (VppJNIConnection.class) {
+            if (connections.containsKey(clientName)) {
+                throw new IOException("Client " + clientName + " already connected");
+            }
+
+            connectionInfo = clientConnect(clientName);
+            if (connectionInfo.status != 0) {
+                throw new IOException("Connection returned error " + connectionInfo.status);
+            }
+            connections.put(clientName, this);
+        }
+    }
+
+    @Override
+    public final void checkActive() {
+        if (disconnected) {
+            throw new IllegalStateException("Disconnected client " + clientName);
+        }
+    }
+
+    @Override
+    public final synchronized void close() {
+        if (!disconnected) {
+            disconnected = true;
+            try {
+                clientDisconnect();
+            } finally {
+                synchronized (VppJNIConnection.class) {
+                    connections.remove(clientName);
+                }
+            }
+        }
+    }
+
+    public ConnectionInfo getConnectionInfo() {
+        return connectionInfo;
+    }
+
+    /**
+     * VPP connection information used by plugins to reuse the connection.
+     */
+    public static final class ConnectionInfo {
+        public final long queueAddress;
+        public final int clientIndex;
+        public final int status; // FIXME throw exception instead
+
+        public ConnectionInfo(long queueAddress, int clientIndex, int status) {
+            this.queueAddress = queueAddress;
+            this.clientIndex = clientIndex;
+            this.status = status;
+        }
+    }
+
+    private static native ConnectionInfo clientConnect(String clientName);
+
+    private static native void clientDisconnect();
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java
new file mode 100644
index 00000000..efddfdbb
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.callback;
+
+import io.fd.vpp.jvpp.dto.ControlPingReply;
+
+/**
+ * Represents callback for control_ping message.
+ */
+public interface ControlPingCallback extends JVppCallback {
+
+    void onControlPingReply(ControlPingReply reply);
+
+}
+
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java
new file mode 100644
index 00000000..ae02063b
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.callback;
+import io.fd.vpp.jvpp.VppCallbackException;
+
+/**
+ * Base JVppCallback interface
+ */
+public interface JVppCallback {
+    /**
+     * onError callback handler used to report failing operation
+     * @param ex VppCallbackException object containing details about failing operation
+     */
+    void onError(VppCallbackException ex);
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java
new file mode 100644
index 00000000..8ab0cb21
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.callback;
+
+/**
+* Notification callback
+*/
+public interface JVppNotificationCallback {
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java
new file mode 100644
index 00000000..984e1674
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.VppInvocationException;
+
+/**
+ * Represents request DTO for control_ping message.
+ */
+public final class ControlPing implements JVppRequest {
+
+    @Override
+    public int send(final JVpp jvpp) throws VppInvocationException {
+        return jvpp.controlPing(this);
+    }
+
+}
+
+
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java
new file mode 100644
index 00000000..61e4d0e4
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+import java.util.Objects;
+
+/**
+ * Represents reply DTO for control_ping message.
+ */
+public final class ControlPingReply implements JVppReply {
+
+    public int context;
+    public int clientIndex;
+    public int vpePid;
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final ControlPingReply that = (ControlPingReply) o;
+        return context == that.context &&
+                clientIndex == that.clientIndex &&
+                vpePid == that.vpePid;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(context, clientIndex, vpePid);
+    }
+
+    @Override
+    public String toString() {
+        return "ControlPingReply{" +
+                "context=" + context +
+                ", clientIndex=" + clientIndex +
+                ", vpePid=" + vpePid +
+                '}';
+    }
+}
+
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java
new file mode 100644
index 00000000..60b98984
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+/**
+* Base interface for all dump requests
+*/
+public interface JVppDump extends JVppRequest {
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java
new file mode 100644
index 00000000..5554f501
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+/**
+* Base interface for all notification DTOs
+*/
+public interface JVppNotification {
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java
new file mode 100644
index 00000000..73f512d4
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+/**
+* Base interface for all reply DTOs
+*/
+public interface JVppReply {
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java
new file mode 100644
index 00000000..15111395
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+/**
+* Base interface for all dump replies
+*/
+public interface JVppReplyDump>
+    extends JVppReply {
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java
new file mode 100644
index 00000000..9b301da2
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.dto;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.VppInvocationException;
+
+/**
+* Base interface for all request DTOs
+*/
+public interface JVppRequest {
+
+    /**
+     * Invoke current operation asynchronously on VPP
+     *
+     * @return context id of this request. Can be used to track incoming response
+     */
+    int send(JVpp jvpp) throws VppInvocationException;
+
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java
new file mode 100644
index 00000000..e7df528a
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.future;
+
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.VppInvocationException;
+import io.fd.vpp.jvpp.dto.JVppDump;
+import io.fd.vpp.jvpp.dto.JVppReply;
+import io.fd.vpp.jvpp.dto.JVppReplyDump;
+import io.fd.vpp.jvpp.dto.JVppRequest;
+
+/**
+ * Future facade on top of JVpp
+ */
+public abstract class AbstractFutureJVppInvoker implements FutureJVppInvoker {
+
+    private final JVpp jvpp;
+    private final JVppRegistry registry;
+
+    /**
+     * Guarded by self
+     */
+    private final Map>> requests;
+
+    protected AbstractFutureJVppInvoker(final JVpp jvpp, final JVppRegistry registry,
+                                        final Map>> requestMap) {
+        this.jvpp =  Objects.requireNonNull(jvpp, "jvpp should not be null");
+        this.registry =  Objects.requireNonNull(registry, "registry should not be null");
+        // Request map represents the shared state between this facade and it's callback
+        // where facade puts futures in and callback completes + removes them
+        this.requests = Objects.requireNonNull(requestMap, "Null requestMap");
+    }
+
+    protected final Map>> getRequests() {
+        synchronized (requests) {
+            return requests;
+        }
+    }
+
+    // TODO use Optional in Future, java8
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public > CompletionStage send(REQ req) {
+        synchronized(requests) {
+            try {
+                final CompletableFuture replyCompletableFuture;
+                final int contextId = jvpp.send(req);
+
+                if(req instanceof JVppDump) {
+                    throw new IllegalArgumentException("Send with empty reply dump has to be used in case of dump calls");
+                }
+                replyCompletableFuture = new CompletableFuture<>();
+                requests.put(contextId, replyCompletableFuture);
+
+                // TODO in case of timeouts/missing replies, requests from the map are not removed
+                // consider adding cancel method, that would remove requests from the map and cancel
+                // associated replyCompletableFuture
+
+                return replyCompletableFuture;
+            } catch (VppInvocationException ex) {
+                final CompletableFuture replyCompletableFuture = new CompletableFuture<>();
+                replyCompletableFuture.completeExceptionally(ex);
+                return replyCompletableFuture;
+            }
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public , DUMP extends JVppReplyDump> CompletionStage send(
+            REQ req, DUMP emptyReplyDump) {
+        synchronized(requests) {
+            try {
+                final CompletableDumpFuture replyCompletableFuture;
+                final int contextId = jvpp.send(req);
+
+                if(!(req instanceof JVppDump)) {
+                    throw new IllegalArgumentException("Send without empty reply dump has to be used in case of regular calls");
+                }
+                replyCompletableFuture = new CompletableDumpFuture<>(contextId, emptyReplyDump);
+
+                requests.put(contextId, replyCompletableFuture);
+                requests.put(registry.controlPing(jvpp.getClass()), replyCompletableFuture);
+
+                // TODO in case of timeouts/missing replies, requests from the map are not removed
+                // consider adding cancel method, that would remove requests from the map and cancel
+                // associated replyCompletableFuture
+
+                return replyCompletableFuture;
+            } catch (VppInvocationException ex) {
+                final CompletableFuture replyCompletableFuture = new CompletableFuture<>();
+                replyCompletableFuture.completeExceptionally(ex);
+                return replyCompletableFuture;
+            }
+        }
+    }
+
+    public static final class CompletableDumpFuture> extends CompletableFuture {
+        private final T replyDump;
+        private final int contextId;
+
+        public CompletableDumpFuture(final int contextId, final T emptyDump) {
+            this.contextId = contextId;
+            this.replyDump = emptyDump;
+        }
+
+        public int getContextId() {
+            return contextId;
+        }
+
+        public T getReplyDump() {
+            return replyDump;
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        jvpp.close();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java
new file mode 100644
index 00000000..7a48e418
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.future;
+
+
+import io.fd.vpp.jvpp.dto.JVppReply;
+import io.fd.vpp.jvpp.dto.JVppReplyDump;
+import io.fd.vpp.jvpp.dto.JVppRequest;
+
+import java.util.concurrent.CompletionStage;
+import io.fd.vpp.jvpp.notification.NotificationRegistryProvider;
+
+/**
+* Future facade on top of JVpp
+*/
+public interface FutureJVppInvoker extends NotificationRegistryProvider, AutoCloseable {
+
+    /**
+     * Invoke asynchronous operation on VPP
+     *
+     * @return CompletionStage with future result of an async VPP call
+     * @throws io.fd.vpp.jvpp.VppInvocationException when send request failed with details
+     */
+    > CompletionStage send(REQ req);
+
+
+    /**
+     * Invoke asynchronous dump operation on VPP
+     *
+     * @return CompletionStage with aggregated future result of an async VPP dump call
+     * @throws io.fd.vpp.jvpp.VppInvocationException when send request failed with details
+     */
+    , DUMP extends JVppReplyDump> CompletionStage send(
+            REQ req, DUMP emptyReplyDump);
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java
new file mode 100644
index 00000000..3c72ff79
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.notification;
+
+/**
+ * Base registry for notification callbacks.
+ */
+public interface NotificationRegistry extends AutoCloseable {
+
+    void close();
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java
new file mode 100644
index 00000000..4a6e06b7
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.notification;
+
+/**
+ * Provides notification registry
+ */
+public interface NotificationRegistryProvider {
+
+    /**
+     * Get current notification registry instance
+     */
+    NotificationRegistry getNotificationRegistry();
+}
diff --git a/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java
new file mode 100644
index 00000000..27b4d29f
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.test;
+
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+
+/**
+ * Run using:
+ * sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-16.09.jar io.fd.vpp.jvpp.test.ConnectionTest
+ */
+public class ConnectionTest {
+
+    private static void testConnect() throws Exception {
+        System.out.println("Testing JNI connection with JVppRegistry");
+        final JVppRegistry registry = new JVppRegistryImpl("ConnectionTest");
+        try {
+            System.out.println("Successfully connected to vpp");
+            Thread.sleep(5000);
+            System.out.println("Disconnecting...");
+            Thread.sleep(1000);
+        } finally {
+            registry.close();
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testConnect();
+    }
+}
diff --git a/src/vpp-api/java/jvpp-registry/jvpp_registry.c b/src/vpp-api/java/jvpp-registry/jvpp_registry.c
new file mode 100644
index 00000000..cbd5e0ab
--- /dev/null
+++ b/src/vpp-api/java/jvpp-registry/jvpp_registry.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#define _GNU_SOURCE /* for strcasestr(3) */
+#include 
+
+#define vl_api_version(n,v) static u32 vpe_api_version = (v);
+#include 
+#undef vl_api_version
+
+
+#include 
+#include 
+#include "io_fd_vpp_jvpp_VppJNIConnection.h"
+#include "io_fd_vpp_jvpp_JVppRegistryImpl.h"
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+vlib_main_t vlib_global_main;
+vlib_main_t **vlib_mains;
+
+/*
+ * The Java runtime isn't compile w/ -fstack-protector,
+ * so we have to supply missing external references for the
+ * regular vpp libraries.
+ */
+void __stack_chk_guard(void) __attribute__((weak));
+void __stack_chk_guard(void) {
+}
+
+typedef struct {
+    /* UThread attachment */
+    volatile u32 control_ping_result_ready;
+    volatile i32 control_ping_retval;
+
+    /* Control poing callback */
+    jobject registryObject;
+    jclass registryClass;
+    jclass controlPingReplyClass;
+    jclass callbackExceptionClass;
+
+    /* Thread cleanup */
+    pthread_key_t cleanup_rx_thread_key;
+
+    /* Connected indication */
+    volatile u8 is_connected;
+} jvpp_registry_main_t;
+
+jvpp_registry_main_t jvpp_registry_main __attribute__((aligned (64)));
+
+void vl_client_add_api_signatures(vl_api_memclnt_create_t *mp) {
+    /*
+     * Send the main API signature in slot 0. This bit of code must
+     * match the checks in ../vpe/api/api.c: vl_msg_api_version_check().
+     */
+    mp->api_versions[0] = clib_host_to_net_u32(vpe_api_version);
+}
+
+/* cleanup handler for RX thread */
+static_always_inline void cleanup_rx_thread(void *arg) {
+    jvpp_main_t * jm = &jvpp_main;
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+
+    vppjni_lock(jm, 99);
+
+    int getEnvStat = (*jm->jvm)->GetEnv(jm->jvm, (void **) &(jm->jenv),
+    JNI_VERSION_1_8);
+    if (getEnvStat == JNI_EVERSION) {
+        clib_warning("Unsupported JNI version\n");
+        rm->control_ping_retval = VNET_API_ERROR_UNSUPPORTED_JNI_VERSION;
+        goto out;
+    } else if (getEnvStat != JNI_EDETACHED) {
+        (*jm->jvm)->DetachCurrentThread(jm->jvm);
+    }
+    out: vppjni_unlock(jm);
+}
+
+static void vl_api_control_ping_reply_t_handler(
+        vl_api_control_ping_reply_t * mp) {
+    jvpp_main_t * jm = &jvpp_main;
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+    char was_thread_connected = 0;
+
+    // attach to java thread if not attached
+    int getEnvStat = (*jm->jvm)->GetEnv(jm->jvm, (void **) &(jm->jenv),
+            JNI_VERSION_1_8);
+    if (getEnvStat == JNI_EDETACHED) {
+        if ((*jm->jvm)->AttachCurrentThread(jm->jvm, (void **) &(jm->jenv),
+                NULL) != 0) {
+            clib_warning("Failed to attach thread\n");
+            rm->control_ping_retval =
+                    VNET_API_ERROR_FAILED_TO_ATTACH_TO_JAVA_THREAD;
+            goto out;
+        }
+
+        // workaround as we can't use pthread_cleanup_push
+        pthread_key_create(&rm->cleanup_rx_thread_key, cleanup_rx_thread);
+        // destructor is only called if the value of key is non null
+        pthread_setspecific(rm->cleanup_rx_thread_key, (void *) 1);
+        was_thread_connected = 1;
+    } else if (getEnvStat == JNI_EVERSION) {
+        clib_warning("Unsupported JNI version\n");
+        rm->control_ping_retval = VNET_API_ERROR_UNSUPPORTED_JNI_VERSION;
+        goto out;
+    }
+
+    if (was_thread_connected == 0) {
+        JNIEnv *env = jm->jenv;
+        if (mp->retval < 0) {
+            call_on_error("controlPing", mp->context, mp->retval,
+                    rm->registryClass, rm->registryObject,
+                    rm->callbackExceptionClass);
+        } else {
+            jmethodID constructor = (*env)->GetMethodID(env,
+                    rm->controlPingReplyClass, "", "()V");
+            jmethodID callbackMethod = (*env)->GetMethodID(env,
+                    rm->registryClass, "onControlPingReply",
+                    "(Lio/fd/vpp/jvpp/dto/ControlPingReply;)V");
+
+            jobject dto = (*env)->NewObject(env, rm->controlPingReplyClass,
+                    constructor);
+
+            jfieldID contextFieldId = (*env)->GetFieldID(env,
+                    rm->controlPingReplyClass, "context", "I");
+            (*env)->SetIntField(env, dto, contextFieldId,
+                    clib_net_to_host_u32(mp->context));
+
+            jfieldID clientIndexFieldId = (*env)->GetFieldID(env,
+                    rm->controlPingReplyClass, "clientIndex", "I");
+            (*env)->SetIntField(env, dto, clientIndexFieldId,
+                    clib_net_to_host_u32(mp->client_index));
+
+            jfieldID vpePidFieldId = (*env)->GetFieldID(env,
+                    rm->controlPingReplyClass, "vpePid", "I");
+            (*env)->SetIntField(env, dto, vpePidFieldId,
+                    clib_net_to_host_u32(mp->vpe_pid));
+
+            (*env)->CallVoidMethod(env, rm->registryObject, callbackMethod,
+                    dto);
+            (*env)->DeleteLocalRef(env, dto);
+        }
+    }
+
+    out: rm->control_ping_result_ready = 1;
+}
+
+static int send_initial_control_ping() {
+    f64 timeout;
+    clib_time_t clib_time;
+    vl_api_control_ping_t * mp;
+    jvpp_main_t * jm = &jvpp_main;
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+
+    clib_time_init(&clib_time);
+
+    rm->control_ping_result_ready = 0;
+    mp = vl_msg_api_alloc(sizeof(*mp));
+    memset(mp, 0, sizeof(*mp));
+    mp->_vl_msg_id = ntohs(VL_API_CONTROL_PING);
+    mp->client_index = jm->my_client_index;
+
+    // send message:
+    vl_msg_api_send_shmem(jm->vl_input_queue, (u8 *) &mp);
+
+    // wait for results: Current time + 10 seconds is the timeout
+    timeout = clib_time_now(&clib_time) + 10.0;
+    int rv = VNET_API_ERROR_RESPONSE_NOT_READY;
+    while (clib_time_now(&clib_time) < timeout) {
+        if (rm->control_ping_result_ready == 1) {
+            rv = rm->control_ping_retval;
+            break;
+        }
+    }
+
+    if (rv != 0) {
+        clib_warning("common: first control ping failed: %d", rv);
+    }
+
+    return rv;
+}
+
+static int connect_to_vpe(char *name) {
+    jvpp_main_t * jm = &jvpp_main;
+    api_main_t * am = &api_main;
+
+    if (vl_client_connect_to_vlib("/vpe-api", name, 32) < 0)
+        return -1;
+
+    jm->my_client_index = am->my_client_index;
+
+    jm->vl_input_queue = am->shmem_hdr->vl_input_queue;
+
+    vl_msg_api_set_handlers(VL_API_CONTROL_PING_REPLY, "control_ping_reply",
+            vl_api_control_ping_reply_t_handler, vl_noop_handler,
+            vl_api_control_ping_reply_t_endian,
+            vl_api_control_ping_reply_t_print,
+            sizeof(vl_api_control_ping_reply_t), 1);
+
+    return send_initial_control_ping();
+}
+
+JNIEXPORT jobject JNICALL Java_io_fd_vpp_jvpp_VppJNIConnection_clientConnect(
+        JNIEnv *env, jclass obj, jstring clientName) {
+    int rv;
+    const char *client_name;
+    void vl_msg_reply_handler_hookup(void);
+    jvpp_main_t * jm = &jvpp_main;
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+
+    jclass connectionInfoClass = (*env)->FindClass(env,
+            "io/fd/vpp/jvpp/VppJNIConnection$ConnectionInfo");
+    jmethodID connectionInfoConstructor = (*env)->GetMethodID(env,
+            connectionInfoClass, "", "(JII)V");
+
+    /*
+     * Bail out now if we're not running as root
+     */
+    if (geteuid() != 0) {
+        return (*env)->NewObject(env, connectionInfoClass,
+                connectionInfoConstructor, 0, 0,
+                VNET_API_ERROR_NOT_RUNNING_AS_ROOT);
+    }
+
+    if (rm->is_connected) {
+        return (*env)->NewObject(env, connectionInfoClass,
+                connectionInfoConstructor, 0, 0,
+                VNET_API_ERROR_ALREADY_CONNECTED);
+    }
+
+    client_name = (*env)->GetStringUTFChars(env, clientName, 0);
+    if (!client_name) {
+        return (*env)->NewObject(env, connectionInfoClass,
+                connectionInfoConstructor, 0, 0, VNET_API_ERROR_INVALID_VALUE);
+    }
+
+    rv = connect_to_vpe((char *) client_name);
+
+    if (rv < 0)
+        clib_warning("connection failed, rv %d", rv);
+
+    (*env)->ReleaseStringUTFChars(env, clientName, client_name);
+
+    return (*env)->NewObject(env, connectionInfoClass,
+            connectionInfoConstructor, (jlong) jm->vl_input_queue,
+            (jint) jm->my_client_index, (jint) rv);
+}
+
+JNIEXPORT jint JNICALL Java_io_fd_vpp_jvpp_JVppRegistryImpl_controlPing0(
+        JNIEnv *env, jobject regstryObject) {
+    jvpp_main_t * jm = &jvpp_main;
+    vl_api_control_ping_t * mp;
+    u32 my_context_id = vppjni_get_context_id(&jvpp_main);
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+
+    if (rm->registryObject == 0) {
+        rm->registryObject = (*env)->NewGlobalRef(env, regstryObject);
+    }
+    if (rm->registryClass == 0) {
+        rm->registryClass = (jclass) (*env)->NewGlobalRef(env,
+                (*env)->GetObjectClass(env, regstryObject));
+    }
+
+    mp = vl_msg_api_alloc(sizeof(*mp));
+    memset(mp, 0, sizeof(*mp));
+    mp->_vl_msg_id = ntohs(VL_API_CONTROL_PING);
+    mp->client_index = jm->my_client_index;
+    mp->context = clib_host_to_net_u32(my_context_id);
+
+    // send message:
+    vl_msg_api_send_shmem(jm->vl_input_queue, (u8 *) &mp);
+    return my_context_id;
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_VppJNIConnection_clientDisconnect(
+        JNIEnv *env, jclass clazz) {
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+    rm->is_connected = 0; // TODO make thread safe
+    vl_client_disconnect_from_vlib();
+
+    // cleanup:
+    if (rm->registryObject) {
+        (*env)->DeleteGlobalRef(env, rm->registryObject);
+        rm->registryObject = 0;
+    }
+    if (rm->registryClass) {
+        (*env)->DeleteGlobalRef(env, rm->registryClass);
+        rm->registryClass = 0;
+    }
+}
+
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    jvpp_main_t * jm = &jvpp_main;
+    jvpp_registry_main_t * rm = &jvpp_registry_main;
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    rm->controlPingReplyClass = (jclass) (*env)->NewGlobalRef(env,
+            (*env)->FindClass(env, "io/fd/vpp/jvpp/dto/ControlPingReply"));
+    if ((*env)->ExceptionCheck(env)) {
+        (*env)->ExceptionDescribe(env);
+        clib_warning("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    rm->callbackExceptionClass = (jclass) (*env)->NewGlobalRef(env,
+            (*env)->FindClass(env, "io/fd/vpp/jvpp/VppCallbackException"));
+    if ((*env)->ExceptionCheck(env)) {
+        (*env)->ExceptionDescribe(env);
+        return JNI_ERR;
+    }
+
+    jm->jvm = vm;
+    return JNI_VERSION_1_8;
+}
+
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    jvpp_main_t * jm = &jvpp_main;
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+
+    jm->jenv = NULL;
+    jm->jvm = NULL;
+}
diff --git a/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java b/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java
new file mode 100644
index 00000000..32165d96
--- /dev/null
+++ b/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/CallbackApiTest.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.snat.test;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.VppCallbackException;
+import io.fd.vpp.jvpp.snat.JVppSnatImpl;
+import io.fd.vpp.jvpp.snat.callback.SnatInterfaceAddDelFeatureCallback;
+import io.fd.vpp.jvpp.snat.dto.SnatInterfaceAddDelFeature;
+import io.fd.vpp.jvpp.snat.dto.SnatInterfaceAddDelFeatureReply;
+
+public class CallbackApiTest {
+
+    static class TestCallback implements SnatInterfaceAddDelFeatureCallback {
+
+        @Override
+        public void onSnatInterfaceAddDelFeatureReply(final SnatInterfaceAddDelFeatureReply msg) {
+            System.out.printf("Received SnatInterfaceAddDelFeatureReply: context=%d%n",
+                msg.context);
+        }
+
+        @Override
+        public void onError(VppCallbackException ex) {
+            System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(),
+                ex.getCtxId(), ex.getErrorCode());
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testCallbackApi();
+    }
+
+    private static void testCallbackApi() throws Exception {
+        System.out.println("Testing Java callback API for snat plugin");
+        try (final JVppRegistry registry = new JVppRegistryImpl("SnatCallbackApiTest");
+             final JVpp jvpp = new JVppSnatImpl()) {
+            registry.register(jvpp, new TestCallback());
+
+            System.out.println("Sending SnatInterfaceAddDelFeature request...");
+            SnatInterfaceAddDelFeature request = new SnatInterfaceAddDelFeature();
+            request.isAdd = 1;
+            request.isInside = 1;
+            request.swIfIndex = 1;
+            final int result = jvpp.send(request);
+            System.out.printf("SnatInterfaceAddDelFeature send result = %d%n", result);
+
+            Thread.sleep(1000);
+
+            System.out.println("Disconnecting...");
+        }
+    }
+}
diff --git a/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/Readme.txt b/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/Readme.txt
new file mode 100644
index 00000000..a2b0c41f
--- /dev/null
+++ b/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/test/Readme.txt
@@ -0,0 +1 @@
+sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/plugins/snat-plugin/jvpp-snat-1.0.jar io.fd.vpp.jvpp.snat.test.CallbackApiTest
diff --git a/src/vpp-api/java/jvpp-snat/jvpp_snat.c b/src/vpp-api/java/jvpp-snat/jvpp_snat.c
new file mode 100644
index 00000000..1095b6eb
--- /dev/null
+++ b/src/vpp-api/java/jvpp-snat/jvpp_snat.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-snat/io_fd_vpp_jvpp_snat_JVppSnatImpl.h"
+#include "jvpp_snat.h"
+#include "jvpp-snat/jvpp_snat_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_snat_JVppsnatImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  snat_main_t * plugin_main = &snat_main;
+  u8 * name;
+  clib_warning ("Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+  name = format (0, "snat_%08x%c", api_version, 0);
+  plugin_main->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (plugin_main->msg_id_base == (u16) ~0) {
+    jclass exClass = (*env)->FindClass(env, "java/lang/IllegalStateException");
+    (*env)->ThrowNew(env, exClass, "snat plugin is not loaded in VPP");
+  } else {
+    plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+    plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+    #define _(N,n)                                  \
+        vl_msg_api_set_handlers(VL_API_##N + plugin_main->msg_id_base, #n,     \
+                vl_api_##n##_t_handler,             \
+                vl_noop_handler,                    \
+                vl_api_##n##_t_endian,              \
+                vl_api_##n##_t_print,               \
+                sizeof(vl_api_##n##_t), 1);
+        foreach_api_reply_handler;
+    #undef _
+  }
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_close0
+(JNIEnv *env, jclass clazz) {
+  snat_main_t * plugin_main = &snat_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP SNAT */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP SNAT */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-snat/jvpp_snat.h b/src/vpp-api/java/jvpp-snat/jvpp_snat.h
new file mode 100644
index 00000000..6426bda8
--- /dev/null
+++ b/src/vpp-api/java/jvpp-snat/jvpp_snat.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_snat_h__
+#define __included_jvpp_snat_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-SNAT */
+typedef struct {
+    /* Base message index for the nsh plugin */
+    u16 msg_id_base;
+
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} snat_main_t;
+
+snat_main_t snat_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_snat_h__ */
diff --git a/src/vpp-api/java/jvpp/gen/jvpp_gen.py b/src/vpp-api/java/jvpp/gen/jvpp_gen.py
new file mode 100755
index 00000000..f51b11d0
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvpp_gen.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+# l
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import importlib
+import sys
+import os
+import json
+
+from jvppgen import types_gen
+from jvppgen import callback_gen
+from jvppgen import notification_gen
+from jvppgen import dto_gen
+from jvppgen import jvpp_callback_facade_gen
+from jvppgen import jvpp_future_facade_gen
+from jvppgen import jvpp_impl_gen
+from jvppgen import jvpp_c_gen
+from jvppgen import util
+
+blacklist = [ "memclnt.api", "flowperpkt.api" ]
+
+# Invocation:
+# ~/Projects/vpp/vpp-api/jvpp/gen$ mkdir -p java/io/fd/vpp/jvpp && cd java/io/fd/vpp/jvpp
+# ~/Projects/vpp/vpp-api/jvpp/gen/java/io/fd/vpp/jvpp$ ../../../../jvpp_gen.py -idefs_api_vpp_papi.py
+#
+# Compilation:
+# ~/Projects/vpp/vpp-api/jvpp/gen/java/io/fd/vpp/jvpp$ javac *.java dto/*.java callback/*.java
+#
+# where
+# defs_api_vpp_papi.py - vpe.api in python format (generated by vppapigen)
+
+parser = argparse.ArgumentParser(description='VPP Java API generator')
+parser.add_argument('-i', action="store", dest="inputfiles", nargs='+')
+parser.add_argument('--plugin_name', action="store", dest="plugin_name")
+parser.add_argument('--root_dir', action="store", dest="root_dir")
+args = parser.parse_args()
+
+sys.path.append(".")
+cwd = os.getcwd()
+
+print "Generating Java API for %s" % args.inputfiles
+print "inputfiles %s" % args.inputfiles
+plugin_name = args.plugin_name
+print "plugin_name %s" % plugin_name
+
+cfg = {}
+
+base_package = 'io.fd.vpp.jvpp'
+plugin_package = base_package + '.' + plugin_name
+root_dir = os.path.abspath(args.root_dir)
+print "root_dir %s" % root_dir
+work_dir = root_dir + "/target/" + plugin_package.replace(".","/")
+
+try:
+    os.makedirs(work_dir)
+except OSError:
+    if not os.path.isdir(work_dir):
+        raise
+
+os.chdir(work_dir)
+
+for inputfile in args.inputfiles:
+    if any(substring in inputfile for substring in blacklist):
+        print "WARNING: Imput file %s blacklisted" % inputfile
+        continue
+    _cfg = json.load(open(cwd + "/" + inputfile, 'r'))
+    if 'types' in cfg:
+        cfg['types'].extend(_cfg['types'])
+    else:
+        cfg['types'] = _cfg['types']
+    if 'messages' in cfg:
+        cfg['messages'].extend(_cfg['messages'])
+    else:
+        cfg['messages'] = _cfg['messages']
+
+
+def is_request_field(field_name):
+    return field_name not in {'_vl_msg_id', 'client_index', 'context'}
+
+
+def is_response_field(field_name):
+    return field_name not in {'_vl_msg_id'}
+
+
+def get_args(t, filter):
+    arg_list = []
+    for i in t:
+        if is_crc(i):
+            continue
+        if not filter(i[1]):
+            continue
+        arg_list.append(i[1])
+    return arg_list
+
+
+def get_types(t, filter):
+    types_list = []
+    lengths_list = []
+    crc = None
+    for i in t:
+        if is_crc(i):
+            crc = ('crc', i['crc'][2:])
+            continue
+        if not filter(i[1]):
+            continue
+        if len(i) is 3:  # array type
+            types_list.append(i[0] + '[]')
+            lengths_list.append((i[2], False))
+        elif len(i) is 4:  # variable length array type
+            types_list.append(i[0] + '[]')
+            lengths_list.append((i[3], True))
+        else:  # primitive type
+            types_list.append(i[0])
+            lengths_list.append((0, False))
+    return types_list, lengths_list, crc
+
+
+def is_crc(arg):
+    """ Check whether the argument inside message definition is just crc """
+    return 'crc' in arg
+
+
+def get_definitions(defs):
+    # Pass 1
+    func_list = []
+    func_name = {}
+    for a in defs:
+        java_name = util.underscore_to_camelcase(a[0])
+
+        # For replies include all the arguments except message_id
+        if util.is_reply(java_name):
+            types, lengths, crc = get_types(a[1:], is_response_field)
+            func_name[a[0]] = dict(
+                [('name', a[0]), ('java_name', java_name),
+                 ('args', get_args(a[1:], is_response_field)), ('full_args', get_args(a[1:], lambda x: True)),
+                 ('types', types), ('lengths', lengths), crc])
+        # For requests skip message_id, client_id and context
+        else:
+            types, lengths, crc = get_types(a[1:], is_request_field)
+            func_name[a[0]] = dict(
+                [('name', a[0]), ('java_name', java_name),
+                 ('args', get_args(a[1:], is_request_field)), ('full_args', get_args(a[1:], lambda x: True)),
+                 ('types', types), ('lengths', lengths), crc])
+
+        # Indexed by name
+        func_list.append(func_name[a[0]])
+    return func_list, func_name
+
+
+types_package = 'types'
+dto_package = 'dto'
+callback_package = 'callback'
+notification_package = 'notification'
+future_package = 'future'
+# TODO find better package name
+callback_facade_package = 'callfacade'
+
+types_list, types_name = get_definitions(cfg['types'])
+
+types_gen.generate_types(types_list, plugin_package, types_package, args.inputfiles)
+
+func_list, func_name = get_definitions(cfg['messages'])
+
+dto_gen.generate_dtos(func_list, base_package, plugin_package, plugin_name.title(), dto_package, args.inputfiles)
+jvpp_impl_gen.generate_jvpp(func_list, base_package, plugin_package, plugin_name, dto_package, args.inputfiles)
+callback_gen.generate_callbacks(func_list, base_package, plugin_package, plugin_name.title(), callback_package, dto_package, args.inputfiles)
+notification_gen.generate_notification_registry(func_list, base_package, plugin_package, plugin_name.title(), notification_package, callback_package, dto_package, args.inputfiles)
+jvpp_c_gen.generate_jvpp(func_list, plugin_name, args.inputfiles, root_dir)
+jvpp_future_facade_gen.generate_jvpp(func_list, base_package, plugin_package, plugin_name.title(), dto_package, callback_package, notification_package, future_package, args.inputfiles)
+jvpp_callback_facade_gen.generate_jvpp(func_list, base_package, plugin_package, plugin_name.title(), dto_package, callback_package, notification_package, callback_facade_package, args.inputfiles)
+
+print "Java API for %s generated successfully" % args.inputfiles
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/__init__.py b/src/vpp-api/java/jvpp/gen/jvppgen/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py
new file mode 100644
index 00000000..b3024b9c
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import util
+from string import Template
+
+from util import remove_suffix
+
+callback_suffix = "Callback"
+
+callback_template = Template("""
+package $plugin_package.$callback_package;
+
+/**
+ * 
Represents callback for plugin's api file message.
+ * 
It was generated by callback_gen.py based on $inputfile preparsed data:
+ * 
+$docs
+ * 

+ */
+public interface $cls_name extends $base_package.$callback_package.$callback_type {
+
+    $callback_method
+
+}
+""")
+
+global_callback_template = Template("""
+package $plugin_package.$callback_package;
+
+/**
+ * 
Global aggregated callback interface.
+ * 
It was generated by callback_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface JVpp${plugin_name}GlobalCallback extends $base_package.$callback_package.ControlPingCallback, $callbacks {
+}
+""")
+
+
+def generate_callbacks(func_list, base_package, plugin_package, plugin_name, callback_package, dto_package, inputfile):
+    """ Generates callback interfaces """
+    print "Generating Callback interfaces"
+
+    if not os.path.exists(callback_package):
+        os.mkdir(callback_package)
+
+    callbacks = []
+    for func in func_list:
+
+        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
+
+        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_name_with_suffix):
+            continue
+        if not util.is_reply(camel_case_name_with_suffix) and not util.is_notification(func['name']):
+            continue
+
+        if util.is_reply(camel_case_name_with_suffix):
+            camel_case_name = util.remove_reply_suffix(camel_case_name_with_suffix)
+            callback_type = "JVppCallback"
+        else:
+            camel_case_name_with_suffix = util.add_notification_suffix(camel_case_name_with_suffix)
+            camel_case_name = camel_case_name_with_suffix
+            callback_type = "JVppNotificationCallback"
+
+        callbacks.append("{0}.{1}.{2}".format(plugin_package, callback_package, camel_case_name + callback_suffix))
+        callback_path = os.path.join(callback_package, camel_case_name + callback_suffix + ".java")
+        callback_file = open(callback_path, 'w')
+
+        reply_type = "%s.%s.%s" % (plugin_package, dto_package, camel_case_name_with_suffix)
+        method = "void on{0}({1} reply);".format(camel_case_name_with_suffix, reply_type)
+        callback_file.write(
+            callback_template.substitute(inputfile=inputfile,
+                                         docs=util.api_message_to_javadoc(func),
+                                         cls_name=camel_case_name + callback_suffix,
+                                         callback_method=method,
+                                         base_package=base_package,
+                                         plugin_package=plugin_package,
+                                         callback_package=callback_package,
+                                         callback_type=callback_type))
+        callback_file.flush()
+        callback_file.close()
+
+    callback_file = open(os.path.join(callback_package, "JVpp%sGlobalCallback.java" % plugin_name), 'w')
+    callback_file.write(global_callback_template.substitute(inputfile=inputfile,
+                                                            callbacks=", ".join(callbacks),
+                                                            base_package=base_package,
+                                                            plugin_package=plugin_package,
+                                                            plugin_name=plugin_name,
+                                                            callback_package=callback_package))
+    callback_file.flush()
+    callback_file.close()
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
new file mode 100644
index 00000000..cfddb9ef
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
@@ -0,0 +1,308 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from string import Template
+
+import util
+
+dto_template = Template("""
+package $plugin_package.$dto_package;
+
+/**
+ * 
This class represents $description.
+ * 
It was generated by dto_gen.py based on $inputfile preparsed data:
+ * 
+$docs
+ * 

+ */
+public final class $cls_name implements $base_package.$dto_package.$base_type {
+
+$fields
+$methods
+}
+""")
+
+field_template = Template("""    public $type $name;\n""")
+
+send_template = Template("""    @Override
+    public int send(final $base_package.JVpp jvpp) throws io.fd.vpp.jvpp.VppInvocationException {
+        return (($plugin_package.JVpp${plugin_name})jvpp).$method_name($args);
+    }""")
+
+
+def generate_dtos(func_list, base_package, plugin_package, plugin_name, dto_package, inputfile):
+    """ Generates dto objects in a dedicated package """
+    print "Generating DTOs"
+
+    if not os.path.exists(dto_package):
+        os.mkdir(dto_package)
+
+    for func in func_list:
+        camel_case_dto_name = util.underscore_to_camelcase_upper(func['name'])
+        camel_case_method_name = util.underscore_to_camelcase(func['name'])
+        dto_path = os.path.join(dto_package, camel_case_dto_name + ".java")
+
+        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_dto_name):
+            continue
+
+        fields = generate_dto_fields(camel_case_dto_name, func)
+        methods = generate_dto_base_methods(camel_case_dto_name, func)
+        base_type = ""
+
+        # Generate request/reply or dump/dumpReply even if structure can be used as notification
+        if not util.is_just_notification(func["name"]):
+            if util.is_reply(camel_case_dto_name):
+                description = "reply DTO"
+                request_dto_name = get_request_name(camel_case_dto_name, func['name'])
+                if util.is_details(camel_case_dto_name):
+                    # FIXME assumption that dump calls end with "Dump" suffix. Not enforced in vpe.api
+                    base_type += "JVppReply<%s.%s.%s>" % (plugin_package, dto_package, request_dto_name + "Dump")
+                    generate_dump_reply_dto(request_dto_name, base_package, plugin_package, dto_package,
+                                            camel_case_dto_name, camel_case_method_name, func)
+                else:
+                    base_type += "JVppReply<%s.%s.%s>" % (plugin_package, dto_package, request_dto_name)
+            else:
+                args = "" if fields is "" else "this"
+                methods += send_template.substitute(method_name=camel_case_method_name,
+                                                    base_package=base_package,
+                                                    plugin_package=plugin_package,
+                                                    plugin_name=plugin_name,
+                                                    args=args)
+                if util.is_dump(camel_case_dto_name):
+                    base_type += "JVppDump"
+                    description = "dump request DTO"
+                else:
+                    base_type += "JVppRequest"
+                    description = "request DTO"
+
+            write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package,
+                           dto_path, fields, func, inputfile, methods)
+
+        # for structures that are also used as notifications, generate dedicated notification DTO
+        if util.is_notification(func["name"]):
+            base_type = "JVppNotification"
+            description = "notification DTO"
+            camel_case_dto_name = util.add_notification_suffix(camel_case_dto_name)
+            dto_path = os.path.join(dto_package, camel_case_dto_name + ".java")
+            methods = generate_dto_base_methods(camel_case_dto_name, func)
+            write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package,
+                           dto_path, fields, func, inputfile, methods)
+
+    flush_dump_reply_dtos(inputfile)
+
+
+def generate_dto_base_methods(camel_case_dto_name, func):
+    methods = generate_dto_hash(func)
+    methods += generate_dto_equals(camel_case_dto_name, func)
+    methods += generate_dto_tostring(camel_case_dto_name, func)
+    return methods
+
+
+def generate_dto_fields(camel_case_dto_name, func):
+    fields = ""
+    for t in zip(func['types'], func['args']):
+        # for retval don't generate dto field in Reply
+        field_name = util.underscore_to_camelcase(t[1])
+        if util.is_reply(camel_case_dto_name) and util.is_retval_field(field_name):
+            continue
+        fields += field_template.substitute(type=util.jni_2_java_type_mapping[t[0]],
+                                            name=field_name)
+    return fields
+
+
+tostring_field_template = Template("""                \"$field_name=\" + $field_name + ", " +\n""")
+tostring_array_field_template = Template("""                \"$field_name=\" + java.util.Arrays.toString($field_name) + ", " +\n""")
+tostring_template = Template("""    @Override
+    public String toString() {
+        return "$cls_name{" +
+$fields_tostring "}";
+    }\n\n""")
+
+
+def generate_dto_tostring(camel_case_dto_name, func):
+    tostring_fields = ""
+    for t in zip(func['types'], func['args']):
+
+        field_name = util.underscore_to_camelcase(t[1])
+        # for retval don't generate dto field in Reply
+        if util.is_retval_field(field_name):
+            continue
+
+        # handle array types
+        if util.is_array(util.jni_2_java_type_mapping[t[0]]):
+            tostring_fields += tostring_array_field_template.substitute(field_name=field_name)
+        else:
+            tostring_fields += tostring_field_template.substitute(field_name=field_name)
+
+    return tostring_template.substitute(cls_name=camel_case_dto_name,
+                                        fields_tostring=tostring_fields[:-8])
+
+equals_other_template = Template("""
+        final $cls_name other = ($cls_name) o;
+\n""")
+equals_field_template = Template("""        if (!java.util.Objects.equals(this.$field_name, other.$field_name)) {
+            return false;
+        }\n""")
+equals_array_field_template = Template("""        if (!java.util.Arrays.equals(this.$field_name, other.$field_name)) {
+            return false;
+        }\n""")
+equals_template = Template("""    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+$comparisons
+        return true;
+    }\n\n""")
+
+
+def generate_dto_equals(camel_case_dto_name, func):
+    equals_fields = ""
+    for t in zip(func['types'], func['args']):
+        field_name = util.underscore_to_camelcase(t[1])
+        # for retval don't generate dto field in Reply
+        if util.is_retval_field(field_name):
+            continue
+
+        # handle array types
+        if util.is_array(util.jni_2_java_type_mapping[t[0]]):
+            equals_fields += equals_array_field_template.substitute(field_name=field_name)
+        else:
+            equals_fields += equals_field_template.substitute(field_name=field_name)
+
+    if equals_fields != "":
+        equals_fields = equals_other_template.substitute(cls_name=camel_case_dto_name) + equals_fields
+
+    return equals_template.substitute(comparisons=equals_fields)
+
+
+hash_template = Template("""    @Override
+    public int hashCode() {
+        return java.util.Objects.hash($fields);
+    }\n\n""")
+hash_single_array_type_template = Template("""    @Override
+    public int hashCode() {
+        return java.util.Arrays.hashCode($fields);
+    }\n\n""")
+
+
+def generate_dto_hash(func):
+    hash_fields = ""
+
+    # Special handling for hashCode in case just a single array field is present. Cannot use Objects.equals since the
+    # array is mistaken for a varargs parameter. Instead use Arrays.hashCode in such case.
+    if len(func['args']) == 1:
+        single_type = func['types'][0]
+        single_type_name = func['args'][0]
+        if util.is_array(util.jni_2_java_type_mapping[single_type]):
+            return hash_single_array_type_template.substitute(fields=util.underscore_to_camelcase(single_type_name))
+
+    for t in zip(func['types'], func['args']):
+        field_name = util.underscore_to_camelcase(t[1])
+        # for retval don't generate dto field in Reply
+        if util.is_retval_field(field_name):
+            continue
+
+        hash_fields += field_name + ", "
+
+    return hash_template.substitute(fields=hash_fields[:-2])
+
+
+def write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package, dto_path,
+                   fields, func, inputfile, methods):
+    dto_file = open(dto_path, 'w')
+    dto_file.write(dto_template.substitute(inputfile=inputfile,
+                                           description=description,
+                                           docs=util.api_message_to_javadoc(func),
+                                           cls_name=camel_case_dto_name,
+                                           fields=fields,
+                                           methods=methods,
+                                           base_package=base_package,
+                                           plugin_package=plugin_package,
+                                           base_type=base_type,
+                                           dto_package=dto_package))
+    dto_file.flush()
+    dto_file.close()
+
+
+dump_dto_suffix = "ReplyDump"
+dump_reply_artificial_dtos = {}
+
+
+# Returns request name or special one from unconventional_naming_rep_req map
+def get_request_name(camel_case_dto_name, func_name):
+    return util.underscore_to_camelcase_upper(
+        util.unconventional_naming_rep_req[func_name]) if func_name in util.unconventional_naming_rep_req \
+        else util.remove_reply_suffix(camel_case_dto_name)
+
+
+def flush_dump_reply_dtos(inputfile):
+    for dump_reply_artificial_dto in dump_reply_artificial_dtos.values():
+        dto_path = os.path.join(dump_reply_artificial_dto['dto_package'],
+                                dump_reply_artificial_dto['cls_name'] + ".java")
+        dto_file = open(dto_path, 'w')
+        dto_file.write(dto_template.substitute(inputfile=inputfile,
+                                               description="dump reply wrapper",
+                                               docs=dump_reply_artificial_dto['docs'],
+                                               cls_name=dump_reply_artificial_dto['cls_name'],
+                                               fields=dump_reply_artificial_dto['fields'],
+                                               methods=dump_reply_artificial_dto['methods'],
+                                               plugin_package=dump_reply_artificial_dto['plugin_package'],
+                                               base_package=dump_reply_artificial_dto['base_package'],
+                                               base_type=dump_reply_artificial_dto['base_type'],
+                                               dto_package=dump_reply_artificial_dto['dto_package']))
+        dto_file.flush()
+        dto_file.close()
+
+
+def generate_dump_reply_dto(request_dto_name, base_package, plugin_package, dto_package, camel_case_dto_name,
+                            camel_case_method_name, func):
+    base_type = "JVppReplyDump<%s.%s.%s, %s.%s.%s>" % (
+        plugin_package, dto_package, util.remove_reply_suffix(camel_case_dto_name) + "Dump",
+        plugin_package, dto_package, camel_case_dto_name)
+    fields = "    public java.util.List<%s> %s = new java.util.ArrayList<>();" % (camel_case_dto_name, camel_case_method_name)
+    cls_name = camel_case_dto_name + dump_dto_suffix
+    # using artificial type for fields, just to bypass the is_array check in base methods generators
+    # the type is not really used
+    artificial_type = 'u8'
+
+    # In case of already existing artificial reply dump DTO, just update it
+    # Used for sub-dump dtos
+    if request_dto_name in dump_reply_artificial_dtos.keys():
+        dump_reply_artificial_dtos[request_dto_name]['fields'] += '\n' + fields
+        dump_reply_artificial_dtos[request_dto_name]['field_names'].append(func['name'])
+        dump_reply_artificial_dtos[request_dto_name]['field_types'].append(artificial_type)
+        methods = '\n' + generate_dto_base_methods(dump_reply_artificial_dtos[request_dto_name]['cls_name'],
+                                            {'args': dump_reply_artificial_dtos[request_dto_name]['field_names'],
+                                             'types': dump_reply_artificial_dtos[request_dto_name]['field_types']})
+        dump_reply_artificial_dtos[request_dto_name]['methods'] = methods
+    else:
+        methods = '\n' + generate_dto_base_methods(cls_name, {'args': [func['name']],
+                                                              'types': [artificial_type]})
+        dump_reply_artificial_dtos[request_dto_name] = ({'docs': util.api_message_to_javadoc(func),
+                                                         'cls_name': cls_name,
+                                                         'fields': fields,
+                                                         'field_names': [func['name']],
+                                                         'field_types': [artificial_type],
+                                                         # strip too many newlines at the end of base method block
+                                                         'methods': methods,
+                                                         'plugin_package': plugin_package,
+                                                         'base_package': base_package,
+                                                         'base_type': base_type,
+                                                         'dto_package': dto_package})
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
new file mode 100644
index 00000000..328cc8d3
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from string import Template
+
+import util
+
+variable_length_array_value_template = Template("""mp->${length_var_name}""")
+variable_length_array_template = Template("""clib_net_to_host_${length_field_type}(${value})""")
+
+dto_field_id_template = Template("""
+    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${class_ref_name}Class, "${field_name}", "${jni_signature}");""")
+
+default_dto_field_setter_template = Template("""
+    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, mp->${c_name});
+""")
+
+variable_length_array_value_template = Template("""mp->${length_var_name}""")
+variable_length_array_template = Template("""clib_net_to_host_${length_field_type}(${value})""")
+
+u16_dto_field_setter_template = Template("""
+    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u16(mp->${c_name}));
+""")
+
+u32_dto_field_setter_template = Template("""
+    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u32(mp->${c_name}));
+""")
+
+u64_dto_field_setter_template = Template("""
+    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u64(mp->${c_name}));
+""")
+
+u8_array_dto_field_setter_template = Template("""
+    jbyteArray ${field_reference_name} = (*env)->NewByteArray(env, ${field_length});
+    (*env)->SetByteArrayRegion(env, ${field_reference_name}, 0, ${field_length}, (const jbyte*)mp->${c_name});
+    (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
+    (*env)->DeleteLocalRef(env, ${field_reference_name});
+""")
+
+u16_array_dto_field_setter_template = Template("""
+    {
+        jshortArray ${field_reference_name} = (*env)->NewShortArray(env, ${field_length});
+        jshort * ${field_reference_name}ArrayElements = (*env)->GetShortArrayElements(env, ${field_reference_name}, NULL);
+        unsigned int _i;
+        for (_i = 0; _i < ${field_length}; _i++) {
+            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u16(mp->${c_name}[_i]);
+        }
+
+        (*env)->ReleaseShortArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
+        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
+        (*env)->DeleteLocalRef(env, ${field_reference_name});
+    }
+""")
+
+u32_array_dto_field_setter_template = Template("""
+    {
+        jintArray ${field_reference_name} = (*env)->NewIntArray(env, ${field_length});
+        jint * ${field_reference_name}ArrayElements = (*env)->GetIntArrayElements(env, ${field_reference_name}, NULL);
+        unsigned int _i;
+        for (_i = 0; _i < ${field_length}; _i++) {
+            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u32(mp->${c_name}[_i]);
+        }
+
+        (*env)->ReleaseIntArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
+        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
+        (*env)->DeleteLocalRef(env, ${field_reference_name});
+    }
+""")
+
+# For each u64 array we get its elements. Then we convert values to host byte order.
+# All changes to  jlong* buffer are written to jlongArray (isCopy is set to NULL)
+u64_array_dto_field_setter_template = Template("""
+    {
+        jlongArray ${field_reference_name} = (*env)->NewLongArray(env, ${field_length});
+        jlong * ${field_reference_name}ArrayElements = (*env)->GetLongArrayElements(env, ${field_reference_name}, NULL);
+        unsigned int _i;
+        for (_i = 0; _i < ${field_length}; _i++) {
+            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u64(mp->${c_name}[_i]);
+        }
+
+        (*env)->ReleaseLongArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
+        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
+        (*env)->DeleteLocalRef(env, ${field_reference_name});
+    }
+""")
+
+dto_field_setter_templates = {'u8': default_dto_field_setter_template,
+                              'u16': u16_dto_field_setter_template,
+                              'u32': u32_dto_field_setter_template,
+                              'i32': u32_dto_field_setter_template,
+                              'u64': u64_dto_field_setter_template,
+                              'f64': default_dto_field_setter_template,  # fixme
+                              'u8[]': u8_array_dto_field_setter_template,
+                              'u16[]': u16_array_dto_field_setter_template,
+                              'u32[]': u32_array_dto_field_setter_template,
+                              'u64[]': u64_array_dto_field_setter_template
+                              }
+
+
+def jni_reply_handler_for_type(handler_name, ref_name, field_type, c_name, field_reference_name,
+                               field_name, field_length, is_variable_len_array, length_field_type,
+                               object_name="dto"):
+    """
+    Generates jni code that initializes a field of java object (dto or custom type).
+    To be used in reply message handlers.
+    :param field_type: type of the field to be initialized (as defined in vpe.api)
+    :param c_name: name of the message struct member that stores initialization value
+    :param field_reference_name: name of the field reference in generated code
+    :param field_name: name of the field (camelcase)
+    :param field_length: integer or name of variable that stores field length
+    :param object_name: name of the object to be initialized
+    """
+
+    # todo move validation to vppapigen
+    if field_type.endswith('[]') and field_length == '0':
+        raise Exception('Variable array \'%s\' defined in \'%s\' '
+                        'should have defined length (e.g. \'%s[%s_length]\''
+                        % (c_name, handler_name, c_name, c_name))
+
+    if is_variable_len_array:
+        length_var_name = field_length
+        field_length = variable_length_array_value_template.substitute(length_var_name=length_var_name)
+        if length_field_type != 'u8':  # we need net to host conversion:
+            field_length = variable_length_array_template.substitute(
+                    length_field_type=length_field_type, value=field_length)
+
+    # for retval don't generate setters
+    if util.is_retval_field(c_name):
+        return ""
+
+    jni_signature = util.jni_2_signature_mapping[field_type]
+    jni_setter = util.jni_field_accessors[field_type]
+
+    result = dto_field_id_template.substitute(
+            field_reference_name=field_reference_name,
+            field_name=field_name,
+            class_ref_name=ref_name,
+            jni_signature=jni_signature)
+
+    dto_setter_template = dto_field_setter_templates[field_type]
+
+    result += dto_setter_template.substitute(
+            jni_signature=jni_signature,
+            object_name=object_name,
+            field_reference_name=field_reference_name,
+            c_name=c_name,
+            jni_setter=jni_setter,
+            field_length=field_length)
+    return result
+
+
+request_field_identifier_template = Template("""
+    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${object_name}Class, "${field_name}", "${jni_signature}");
+    ${jni_type} ${field_reference_name} = (*env)->Get${jni_getter}(env, ${object_name}, ${field_reference_name}FieldId);
+    """)
+
+array_length_enforcement_template = Template("""
+        size_t max_size = ${field_length};
+        if (cnt > max_size) cnt = max_size;""")
+
+u8_struct_setter_template = Template("""
+    mp->${c_name} = ${field_reference_name};""")
+
+u16_struct_setter_template = Template("""
+    mp->${c_name} = clib_host_to_net_u16(${field_reference_name});""")
+
+u32_struct_setter_template = Template("""
+    mp->${c_name} = clib_host_to_net_u32(${field_reference_name});""")
+
+i32_struct_setter_template = Template("""
+    mp->${c_name} = clib_host_to_net_i32(${field_reference_name});!""")
+
+u64_struct_setter_template = Template("""
+    mp->${c_name} = clib_host_to_net_u64(${field_reference_name});""")
+
+array_length_enforcement_template = Template("""
+        size_t max_size = ${field_length};
+        if (cnt > max_size) cnt = max_size;""")
+
+u8_array_struct_setter_template = Template("""
+    if (${field_reference_name}) {
+        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
+        ${field_length_check}
+        (*env)->GetByteArrayRegion(env, ${field_reference_name}, 0, cnt, (jbyte *)mp->${c_name});
+    }
+""")
+
+u16_array_struct_setter_template = Template("""
+    if (${field_reference_name}) {
+        jshort * ${field_reference_name}ArrayElements = (*env)->GetShortArrayElements(env, ${field_reference_name}, NULL);
+        size_t _i;
+        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
+        ${field_length_check}
+        for (_i = 0; _i < cnt; _i++) {
+            mp->${c_name}[_i] = clib_host_to_net_u16(${field_reference_name}ArrayElements[_i]);
+        }
+        (*env)->ReleaseShortArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
+    }
+    """)
+
+u32_array_struct_setter_template = Template("""
+    if (${field_reference_name}) {
+        jint * ${field_reference_name}ArrayElements = (*env)->GetIntArrayElements(env, ${field_reference_name}, NULL);
+        size_t _i;
+        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
+        ${field_length_check}
+        for (_i = 0; _i < cnt; _i++) {
+            mp->${c_name}[_i] = clib_host_to_net_u32(${field_reference_name}ArrayElements[_i]);
+        }
+        (*env)->ReleaseIntArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
+    }
+    """)
+
+u64_array_struct_setter_template = Template("""
+    if (${field_reference_name}) {
+        jlong * ${field_reference_name}ArrayElements = (*env)->GetLongArrayElements(env, ${field_reference_name}, NULL);
+        size_t _i;
+        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
+        ${field_length_check}
+        for (_i = 0; _i < cnt; _i++) {
+            mp->${c_name}[_i] = clib_host_to_net_u64(${field_reference_name}ArrayElements[_i]);
+        }
+        (*env)->ReleaseLongArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
+    }
+    """)
+
+struct_setter_templates = {'u8': u8_struct_setter_template,
+                           'u16': u16_struct_setter_template,
+                           'u32': u32_struct_setter_template,
+                           'i32': u32_struct_setter_template,
+                           'u64': u64_struct_setter_template,
+                           'u8[]': u8_array_struct_setter_template,
+                           'u16[]': u16_array_struct_setter_template,
+                           'u32[]': u32_array_struct_setter_template,
+                           'u64[]': u64_array_struct_setter_template
+                           }
+
+
+def jni_request_binding_for_type(field_type, c_name, field_reference_name, field_name, field_length,
+                                 is_variable_len_array, object_name="request"):
+    """
+    Generates jni code that initializes C structure that corresponds to a field of java object
+    (dto or custom type). To be used in request message handlers.
+    :param field_type: type of the field to be initialized (as defined in vpe.api)
+    :param c_name: name of the message struct member to be initialized
+    :param field_reference_name: name of the field reference in generated code
+    :param field_name: name of the field (camelcase)
+    :param field_length: integer or name of variable that stores field length
+    :param object_name: name of the object to be initialized
+    """
+    # field identifiers
+    jni_type = util.vpp_2_jni_type_mapping[field_type]
+    jni_signature = util.jni_2_signature_mapping[field_type]
+    jni_getter = util.jni_field_accessors[field_type]
+
+    # field identifier
+    msg_initialization = request_field_identifier_template.substitute(
+            jni_type=jni_type,
+            field_reference_name=field_reference_name,
+            field_name=field_name,
+            jni_signature=jni_signature,
+            jni_getter=jni_getter,
+            object_name=object_name)
+
+    # field setter
+    field_length_check = ""
+
+    # check if we are processing variable length array:
+    if is_variable_len_array:
+        field_length = util.underscore_to_camelcase(field_length)
+
+    # enforce max length if array has fixed length or uses variable length syntax
+    if str(field_length) != "0":
+        field_length_check = array_length_enforcement_template.substitute(field_length=field_length)
+
+    struct_setter_template = struct_setter_templates[field_type]
+
+    msg_initialization += struct_setter_template.substitute(
+            c_name=c_name,
+            field_reference_name=field_reference_name,
+            field_length_check=field_length_check)
+
+    return msg_initialization
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py
new file mode 100644
index 00000000..611171c4
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+# l
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, util
+from string import Template
+
+import jni_gen
+
+
+def is_manually_generated(f_name, plugin_name):
+    return f_name in {'control_ping_reply'}
+
+
+class_reference_template = Template("""jclass ${ref_name}Class;
+""")
+
+find_class_invocation_template = Template("""
+    ${ref_name}Class = (jclass)(*env)->NewGlobalRef(env, (*env)->FindClass(env, "io/fd/vpp/jvpp/${plugin_name}/dto/${class_name}"));
+    if ((*env)->ExceptionCheck(env)) {
+        (*env)->ExceptionDescribe(env);
+        return JNI_ERR;
+    }""")
+
+find_class_template = Template("""
+    ${ref_name}Class = (jclass)(*env)->NewGlobalRef(env, (*env)->FindClass(env, "${class_name}"));
+    if ((*env)->ExceptionCheck(env)) {
+        (*env)->ExceptionDescribe(env);
+        return JNI_ERR;
+    }""")
+
+delete_class_invocation_template = Template("""
+    if (${ref_name}Class) {
+        (*env)->DeleteGlobalRef(env, ${ref_name}Class);
+    }""")
+
+class_cache_template = Template("""
+$class_references
+static int cache_class_references(JNIEnv* env) {
+    $find_class_invocations
+    return 0;
+}
+
+static void delete_class_references(JNIEnv* env) {
+    $delete_class_invocations
+}""")
+
+
+def generate_class_cache(func_list, plugin_name):
+    class_references = []
+    find_class_invocations = []
+    delete_class_invocations = []
+    for f in func_list:
+        c_name = f['name']
+        class_name = util.underscore_to_camelcase_upper(c_name)
+        ref_name = util.underscore_to_camelcase(c_name)
+
+        if util.is_ignored(c_name) or util.is_control_ping(class_name):
+            continue
+
+        if util.is_reply(class_name):
+            class_references.append(class_reference_template.substitute(
+                ref_name=ref_name))
+            find_class_invocations.append(find_class_invocation_template.substitute(
+                plugin_name=plugin_name,
+                ref_name=ref_name,
+                class_name=class_name))
+            delete_class_invocations.append(delete_class_invocation_template.substitute(ref_name=ref_name))
+        elif util.is_notification(c_name):
+            class_references.append(class_reference_template.substitute(
+                ref_name=util.add_notification_suffix(ref_name)))
+            find_class_invocations.append(find_class_invocation_template.substitute(
+                plugin_name=plugin_name,
+                ref_name=util.add_notification_suffix(ref_name),
+                class_name=util.add_notification_suffix(class_name)))
+            delete_class_invocations.append(delete_class_invocation_template.substitute(
+                ref_name=util.add_notification_suffix(ref_name)))
+
+    # add exception class to class cache
+    ref_name = 'callbackException'
+    class_name = 'io/fd/vpp/jvpp/VppCallbackException'
+    class_references.append(class_reference_template.substitute(
+            ref_name=ref_name))
+    find_class_invocations.append(find_class_template.substitute(
+            ref_name=ref_name,
+            class_name=class_name))
+    delete_class_invocations.append(delete_class_invocation_template.substitute(ref_name=ref_name))
+
+    return class_cache_template.substitute(
+        class_references="".join(class_references), find_class_invocations="".join(find_class_invocations),
+        delete_class_invocations="".join(delete_class_invocations))
+
+
+# TODO: cache method and field identifiers to achieve better performance
+# https://jira.fd.io/browse/HONEYCOMB-42
+request_class_template = Template("""
+    jclass requestClass = (*env)->FindClass(env, "io/fd/vpp/jvpp/${plugin_name}/dto/${java_name_upper}");""")
+
+request_field_identifier_template = Template("""
+    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${object_name}Class, "${field_name}", "${jni_signature}");
+    ${jni_type} ${field_reference_name} = (*env)->Get${jni_getter}(env, ${object_name}, ${field_reference_name}FieldId);
+    """)
+
+
+jni_impl_template = Template("""
+/**
+ * JNI binding for sending ${c_name} message.
+ * Generated based on $inputfile preparsed data:
+$api_data
+ */
+JNIEXPORT jint JNICALL Java_io_fd_vpp_jvpp_${plugin_name}_JVpp${java_plugin_name}Impl_${field_name}0
+(JNIEnv * env, jclass clazz$args) {
+    ${plugin_name}_main_t *plugin_main = &${plugin_name}_main;
+    vl_api_${c_name}_t * mp;
+    u32 my_context_id = vppjni_get_context_id (&jvpp_main);
+    $request_class
+
+    // create message:
+    mp = vl_msg_api_alloc(sizeof(*mp));
+    memset (mp, 0, sizeof (*mp));
+    mp->_vl_msg_id = ntohs (VL_API_${c_name_uppercase} + plugin_main->msg_id_base);
+    mp->client_index = plugin_main->my_client_index;
+    mp->context = clib_host_to_net_u32 (my_context_id);
+
+    $msg_initialization
+
+    // send message:
+    vl_msg_api_send_shmem (plugin_main->vl_input_queue, (u8 *)&mp);
+    if ((*env)->ExceptionCheck(env)) {
+        return JNI_ERR;
+    }
+    return my_context_id;
+}""")
+
+def generate_jni_impl(func_list, plugin_name, inputfile):
+    jni_impl = []
+    for f in func_list:
+        f_name = f['name']
+        camel_case_function_name = util.underscore_to_camelcase(f_name)
+        if is_manually_generated(f_name, plugin_name) or util.is_reply(camel_case_function_name) \
+                or util.is_ignored(f_name) or util.is_just_notification(f_name):
+            continue
+
+        arguments = ''
+        request_class = ''
+        msg_initialization = ''
+        f_name_uppercase = f_name.upper()
+
+        if f['args']:
+            arguments = ', jobject request'
+            camel_case_function_name_upper = util.underscore_to_camelcase_upper(f_name)
+
+            request_class = request_class_template.substitute(
+                    java_name_upper=camel_case_function_name_upper,
+                    plugin_name=plugin_name)
+
+            for t in zip(f['types'], f['args'], f['lengths']):
+                field_name = util.underscore_to_camelcase(t[1])
+                msg_initialization += jni_gen.jni_request_binding_for_type(field_type=t[0], c_name=t[1],
+                                                                           field_reference_name=field_name,
+                                                                           field_name=field_name,
+                                                                           field_length=t[2][0],
+                                                                           is_variable_len_array=t[2][1])
+
+        jni_impl.append(jni_impl_template.substitute(
+                inputfile=inputfile,
+                api_data=util.api_message_to_javadoc(f),
+                field_reference_name=camel_case_function_name,
+                field_name=camel_case_function_name,
+                c_name_uppercase=f_name_uppercase,
+                c_name=f_name,
+                plugin_name=plugin_name,
+                java_plugin_name=plugin_name.title(),
+                request_class=request_class,
+                msg_initialization=msg_initialization,
+                args=arguments))
+
+    return "\n".join(jni_impl)
+
+# code fragment for checking result of the operation before sending request reply
+callback_err_handler_template = Template("""
+    // for negative result don't send callback message but send error callback
+    if (mp->retval<0) {
+        call_on_error("${handler_name}", mp->context, mp->retval, plugin_main->callbackClass, plugin_main->callbackObject, callbackExceptionClass);
+        return;
+    }
+    if (mp->retval == VNET_API_ERROR_IN_PROGRESS) {
+        clib_warning("Result in progress");
+        return;
+    }
+""")
+
+msg_handler_template = Template("""
+/**
+ * Handler for ${handler_name} message.
+ * Generated based on $inputfile preparsed data:
+$api_data
+ */
+static void vl_api_${handler_name}_t_handler (vl_api_${handler_name}_t * mp)
+{
+    ${plugin_name}_main_t *plugin_main = &${plugin_name}_main;
+    JNIEnv *env = jvpp_main.jenv;
+
+    $err_handler
+
+    jmethodID constructor = (*env)->GetMethodID(env, ${class_ref_name}Class, "", "()V");
+    jmethodID callbackMethod = (*env)->GetMethodID(env, plugin_main->callbackClass, "on${dto_name}", "(Lio/fd/vpp/jvpp/${plugin_name}/dto/${dto_name};)V");
+
+    jobject dto = (*env)->NewObject(env, ${class_ref_name}Class, constructor);
+    $dto_setters
+
+    (*env)->CallVoidMethod(env, plugin_main->callbackObject, callbackMethod, dto);
+    // free DTO as per http://stackoverflow.com/questions/1340938/memory-leak-when-calling-java-code-from-c-using-jni
+    (*env)->DeleteLocalRef(env, dto);
+}""")
+
+
+def generate_msg_handlers(func_list, plugin_name, inputfile):
+    handlers = []
+    for f in func_list:
+        handler_name = f['name']
+        dto_name = util.underscore_to_camelcase_upper(handler_name)
+        ref_name = util.underscore_to_camelcase(handler_name)
+
+        if is_manually_generated(handler_name, plugin_name) or util.is_ignored(handler_name):
+            continue
+
+        if not util.is_reply(dto_name) and not util.is_notification(handler_name):
+            continue
+
+        if util.is_notification(handler_name):
+            dto_name = util.add_notification_suffix(dto_name)
+            ref_name = util.add_notification_suffix(ref_name)
+
+        dto_setters = ''
+        err_handler = ''
+        # dto setters
+        for t in zip(f['types'], f['args'], f['lengths']):
+            c_name = t[1]
+            java_name = util.underscore_to_camelcase(c_name)
+            field_length = t[2][0]
+            is_variable_len_array = t[2][1]
+            length_field_type = None
+            if is_variable_len_array:
+                length_field_type = f['types'][f['args'].index(field_length)]
+            dto_setters += jni_gen.jni_reply_handler_for_type(handler_name=handler_name, ref_name=ref_name,
+                                                              field_type=t[0], c_name=t[1],
+                                                              field_reference_name=java_name,
+                                                              field_name=java_name, field_length=field_length,
+                                                              is_variable_len_array=is_variable_len_array,
+                                                              length_field_type=length_field_type)
+
+            # for retval don't generate setters and generate retval check
+            if util.is_retval_field(c_name):
+                err_handler = callback_err_handler_template.substitute(
+                    handler_name=handler_name
+                )
+                continue
+
+        handlers.append(msg_handler_template.substitute(
+            inputfile=inputfile,
+            api_data=util.api_message_to_javadoc(f),
+            handler_name=handler_name,
+            plugin_name=plugin_name,
+            dto_name=dto_name,
+            class_ref_name=ref_name,
+            dto_setters=dto_setters,
+            err_handler=err_handler))
+
+    return "\n".join(handlers)
+
+
+handler_registration_template = Template("""_(${upercase_name}, ${name}) \\
+""")
+
+
+def generate_handler_registration(func_list):
+    handler_registration = ["#define foreach_api_reply_handler \\\n"]
+    for f in func_list:
+        name = f['name']
+        camelcase_name = util.underscore_to_camelcase(f['name'])
+
+        if (not util.is_reply(camelcase_name) and not util.is_notification(name)) or util.is_ignored(name) \
+                or util.is_control_ping(camelcase_name):
+            continue
+
+        handler_registration.append(handler_registration_template.substitute(
+            name=name,
+            upercase_name=name.upper()))
+
+    return "".join(handler_registration)
+
+
+jvpp_c_template = Template("""/**
+ * This file contains JNI bindings for jvpp Java API.
+ * It was generated by jvpp_c_gen.py based on $inputfile
+ * (python representation of api file generated by vppapigen).
+ */
+
+// JAVA class reference cache
+$class_cache
+
+// JNI bindings
+$jni_implementations
+
+// Message handlers
+$msg_handlers
+
+// Registration of message handlers in vlib
+$handler_registration
+""")
+
+def generate_jvpp(func_list, plugin_name, inputfile, path):
+    """ Generates jvpp C file """
+    print "Generating jvpp C"
+
+    class_cache = generate_class_cache(func_list, plugin_name)
+    jni_impl = generate_jni_impl(func_list, plugin_name, inputfile)
+    msg_handlers = generate_msg_handlers(func_list, plugin_name, inputfile)
+    handler_registration = generate_handler_registration(func_list)
+
+    jvpp_c_file = open("%s/jvpp_%s_gen.h" % (path, plugin_name), 'w')
+    jvpp_c_file.write(jvpp_c_template.substitute(
+            inputfile=inputfile,
+            class_cache=class_cache,
+            jni_implementations=jni_impl,
+            msg_handlers=msg_handlers,
+            handler_registration=handler_registration))
+    jvpp_c_file.flush()
+    jvpp_c_file.close()
+
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py
new file mode 100644
index 00000000..ac096a71
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py
@@ -0,0 +1,324 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, util
+from string import Template
+
+import callback_gen
+import dto_gen
+
+jvpp_ifc_template = Template("""
+package $plugin_package.$callback_facade_package;
+
+/**
+ * 
Callback Java API representation of $plugin_package plugin.
+ * 
It was generated by jvpp_callback_facade_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface CallbackJVpp${plugin_name} extends $base_package.$notification_package.NotificationRegistryProvider, java.lang.AutoCloseable {
+
+    // TODO add send
+
+$methods
+}
+""")
+
+jvpp_impl_template = Template("""
+package $plugin_package.$callback_facade_package;
+
+/**
+ * 
Default implementation of Callback${plugin_name}JVpp interface.
+ * 
It was generated by jvpp_callback_facade_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public final class CallbackJVpp${plugin_name}Facade implements CallbackJVpp${plugin_name} {
+
+    private final $plugin_package.JVpp${plugin_name} jvpp;
+    private final java.util.Map callbacks;
+    private final $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl notificationRegistry = new $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl();
+    /**
+     * 
Create CallbackJVpp${plugin_name}Facade object for provided JVpp instance.
+     * Constructor internally creates CallbackJVppFacadeCallback class for processing callbacks
+     * and then connects to provided JVpp instance
+     *
+     * @param jvpp provided $base_package.JVpp instance
+     *
+     * @throws java.io.IOException in case instance cannot connect to JVPP
+     */
+    public CallbackJVpp${plugin_name}Facade(final $base_package.JVppRegistry registry, final $plugin_package.JVpp${plugin_name} jvpp) throws java.io.IOException {
+        this.jvpp = java.util.Objects.requireNonNull(jvpp,"jvpp is null");
+        this.callbacks = new java.util.HashMap<>();
+        java.util.Objects.requireNonNull(registry, "JVppRegistry should not be null");
+        registry.register(jvpp, new CallbackJVpp${plugin_name}FacadeCallback(this.callbacks, notificationRegistry));
+    }
+
+    @Override
+    public $plugin_package.$notification_package.${plugin_name}NotificationRegistry getNotificationRegistry() {
+        return notificationRegistry;
+    }
+
+    @Override
+    public void close() throws Exception {
+        jvpp.close();
+    }
+
+    // TODO add send()
+
+$methods
+}
+""")
+
+method_template = Template(
+    """    void $name($plugin_package.$dto_package.$request request, $plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException;""")
+
+method_impl_template = Template("""    public final void $name($plugin_package.$dto_package.$request request, $plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException {
+        synchronized (callbacks) {
+            callbacks.put(jvpp.$name(request), callback);
+        }
+    }
+""")
+
+no_arg_method_template = Template("""    void $name($plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException;""")
+no_arg_method_impl_template = Template("""    public final void $name($plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException {
+        synchronized (callbacks) {
+            callbacks.put(jvpp.$name(), callback);
+        }
+    }
+""")
+
+
+def generate_jvpp(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, callback_facade_package, inputfile):
+    """ Generates callback facade """
+    print "Generating JVpp callback facade"
+
+    if os.path.exists(callback_facade_package):
+        util.remove_folder(callback_facade_package)
+
+    os.mkdir(callback_facade_package)
+
+    methods = []
+    methods_impl = []
+    for func in func_list:
+
+        if util.is_notification(func['name']) or util.is_ignored(func['name']):
+            continue
+
+        camel_case_name = util.underscore_to_camelcase(func['name'])
+        camel_case_name_upper = util.underscore_to_camelcase_upper(func['name'])
+        if util.is_reply(camel_case_name) or util.is_control_ping(camel_case_name):
+            continue
+
+        # Strip suffix for dump calls
+        callback_type = get_request_name(camel_case_name_upper, func['name']) + callback_gen.callback_suffix
+
+        if len(func['args']) == 0:
+            methods.append(no_arg_method_template.substitute(name=camel_case_name,
+                                                             base_package=base_package,
+                                                             plugin_package=plugin_package,
+                                                             dto_package=dto_package,
+                                                             callback_package=callback_package,
+                                                             callback=callback_type))
+            methods_impl.append(no_arg_method_impl_template.substitute(name=camel_case_name,
+                                                                       base_package=base_package,
+                                                                       plugin_package=plugin_package,
+                                                                       dto_package=dto_package,
+                                                                       callback_package=callback_package,
+                                                                       callback=callback_type))
+        else:
+            methods.append(method_template.substitute(name=camel_case_name,
+                                                      request=camel_case_name_upper,
+                                                      base_package=base_package,
+                                                      plugin_package=plugin_package,
+                                                      dto_package=dto_package,
+                                                      callback_package=callback_package,
+                                                      callback=callback_type))
+            methods_impl.append(method_impl_template.substitute(name=camel_case_name,
+                                                                request=camel_case_name_upper,
+                                                                base_package=base_package,
+                                                                plugin_package=plugin_package,
+                                                                dto_package=dto_package,
+                                                                callback_package=callback_package,
+                                                                callback=callback_type))
+
+    join = os.path.join(callback_facade_package, "CallbackJVpp%s.java" % plugin_name)
+    jvpp_file = open(join, 'w')
+    jvpp_file.write(
+        jvpp_ifc_template.substitute(inputfile=inputfile,
+                                     methods="\n".join(methods),
+                                     base_package=base_package,
+                                     plugin_package=plugin_package,
+                                     plugin_name=plugin_name,
+                                     dto_package=dto_package,
+                                     notification_package=notification_package,
+                                     callback_facade_package=callback_facade_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+    jvpp_file = open(os.path.join(callback_facade_package, "CallbackJVpp%sFacade.java" % plugin_name), 'w')
+    jvpp_file.write(jvpp_impl_template.substitute(inputfile=inputfile,
+                                                  methods="\n".join(methods_impl),
+                                                  base_package=base_package,
+                                                  plugin_package=plugin_package,
+                                                  plugin_name=plugin_name,
+                                                  dto_package=dto_package,
+                                                  notification_package=notification_package,
+                                                  callback_package=callback_package,
+                                                  callback_facade_package=callback_facade_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+    generate_callback(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, callback_facade_package, inputfile)
+
+
+jvpp_facade_callback_template = Template("""
+package $plugin_package.$callback_facade_package;
+
+/**
+ * 
Implementation of JVppGlobalCallback interface for Java Callback API.
+ * 
It was generated by jvpp_callback_facade_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public final class CallbackJVpp${plugin_name}FacadeCallback implements $plugin_package.$callback_package.JVpp${plugin_name}GlobalCallback {
+
+    private final java.util.Map requests;
+    private final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback;
+    private static final java.util.logging.Logger LOG = java.util.logging.Logger.getLogger(CallbackJVpp${plugin_name}FacadeCallback.class.getName());
+
+    public CallbackJVpp${plugin_name}FacadeCallback(final java.util.Map requestMap,
+                                      final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback) {
+        this.requests = requestMap;
+        this.notificationCallback = notificationCallback;
+    }
+
+    @Override
+    public void onError($base_package.VppCallbackException reply) {
+
+        $base_package.$callback_package.JVppCallback failedCall;
+        synchronized(requests) {
+            failedCall = requests.remove(reply.getCtxId());
+        }
+
+        if(failedCall != null) {
+            try {
+                failedCall.onError(reply);
+            } catch(RuntimeException ex) {
+                ex.addSuppressed(reply);
+                LOG.log(java.util.logging.Level.WARNING, String.format("Callback: %s failed while handling exception: %s", failedCall, reply), ex);
+            }
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public void onControlPingReply($base_package.$dto_package.ControlPingReply reply) {
+
+        $base_package.$callback_package.ControlPingCallback callback;
+        synchronized(requests) {
+            callback = ($base_package.$callback_package.ControlPingCallback) requests.remove(reply.context);
+        }
+
+        if(callback != null) {
+            callback.onControlPingReply(reply);
+        }
+    }
+
+$methods
+}
+""")
+
+jvpp_facade_callback_method_template = Template("""
+    @Override
+    @SuppressWarnings("unchecked")
+    public void on$callback_dto($plugin_package.$dto_package.$callback_dto reply) {
+
+        $plugin_package.$callback_package.$callback callback;
+        synchronized(requests) {
+            callback = ($plugin_package.$callback_package.$callback) requests.remove(reply.context);
+        }
+
+        if(callback != null) {
+            callback.on$callback_dto(reply);
+        }
+    }
+""")
+
+jvpp_facade_callback_notification_method_template = Template("""
+    @Override
+    @SuppressWarnings("unchecked")
+    public void on$callback_dto($plugin_package.$dto_package.$callback_dto notification) {
+        notificationCallback.on$callback_dto(notification);
+    }
+""")
+
+
+def generate_callback(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, callback_facade_package, inputfile):
+    callbacks = []
+    for func in func_list:
+
+        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
+
+        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_name_with_suffix):
+            continue
+
+        if util.is_reply(camel_case_name_with_suffix):
+            callbacks.append(jvpp_facade_callback_method_template.substitute(plugin_package=plugin_package,
+                                                                             dto_package=dto_package,
+                                                                             callback_package=callback_package,
+                                                                             callback=util.remove_reply_suffix(camel_case_name_with_suffix) + callback_gen.callback_suffix,
+                                                                             callback_dto=camel_case_name_with_suffix))
+
+        if util.is_notification(func["name"]):
+            with_notification_suffix = util.add_notification_suffix(camel_case_name_with_suffix)
+            callbacks.append(jvpp_facade_callback_notification_method_template.substitute(plugin_package=plugin_package,
+                                                                             dto_package=dto_package,
+                                                                             callback_package=callback_package,
+                                                                             callback=with_notification_suffix + callback_gen.callback_suffix,
+                                                                             callback_dto=with_notification_suffix))
+
+    jvpp_file = open(os.path.join(callback_facade_package, "CallbackJVpp%sFacadeCallback.java" % plugin_name), 'w')
+    jvpp_file.write(jvpp_facade_callback_template.substitute(inputfile=inputfile,
+                                                             base_package=base_package,
+                                                             plugin_package=plugin_package,
+                                                             plugin_name=plugin_name,
+                                                             dto_package=dto_package,
+                                                             notification_package=notification_package,
+                                                             callback_package=callback_package,
+                                                             methods="".join(callbacks),
+                                                             callback_facade_package=callback_facade_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+
+# Returns request name or special one from unconventional_naming_rep_req map
+def get_request_name(camel_case_dto_name, func_name):
+    if func_name in reverse_dict(util.unconventional_naming_rep_req):
+        request_name = util.underscore_to_camelcase_upper(reverse_dict(util.unconventional_naming_rep_req)[func_name])
+    else:
+        request_name = camel_case_dto_name
+    return remove_suffix(request_name)
+
+
+def reverse_dict(map):
+    return dict((v, k) for k, v in map.iteritems())
+
+
+def remove_suffix(name):
+    if util.is_reply(name):
+        return util.remove_reply_suffix(name)
+    else:
+        if util.is_dump(name):
+            return util.remove_suffix(name, util.dump_suffix)
+        else:
+            return name
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py
new file mode 100644
index 00000000..26b31e22
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py
@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from string import Template
+
+import dto_gen
+import util
+
+jvpp_facade_callback_template = Template("""
+package $plugin_package.$future_package;
+
+/**
+ * 
Async facade callback setting values to future objects
+ * 
It was generated by jvpp_future_facade_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public final class FutureJVpp${plugin_name}FacadeCallback implements $plugin_package.$callback_package.JVpp${plugin_name}GlobalCallback {
+
+    private final java.util.Map>> requests;
+    private final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback;
+
+    public FutureJVpp${plugin_name}FacadeCallback(
+        final java.util.Map>> requestMap,
+        final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback) {
+        this.requests = requestMap;
+        this.notificationCallback = notificationCallback;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public void onError($base_package.VppCallbackException reply) {
+        final java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply> completableFuture;
+
+        synchronized(requests) {
+            completableFuture = (java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply>) requests.get(reply.getCtxId());
+        }
+
+        if(completableFuture != null) {
+            completableFuture.completeExceptionally(reply);
+
+            synchronized(requests) {
+                requests.remove(reply.getCtxId());
+            }
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public void onControlPingReply($base_package.$dto_package.ControlPingReply reply) {
+        final java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply> completableFuture;
+
+        synchronized(requests) {
+            completableFuture = (java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply>) requests.get(reply.context);
+        }
+
+        if(completableFuture != null) {
+            // Finish dump call
+            if (completableFuture instanceof $base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture) {
+                completableFuture.complete((($base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture) completableFuture).getReplyDump());
+                // Remove future mapped to dump call context id
+                synchronized(requests) {
+                    requests.remove((($base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture) completableFuture).getContextId());
+                }
+            } else {
+                completableFuture.complete(reply);
+            }
+
+            synchronized(requests) {
+                requests.remove(reply.context);
+            }
+        }
+    }
+
+$methods
+}
+""")
+
+jvpp_facade_callback_method_template = Template("""
+    @Override
+    @SuppressWarnings("unchecked")
+    public void on$callback_dto($plugin_package.$dto_package.$callback_dto reply) {
+        final java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply> completableFuture;
+
+        synchronized(requests) {
+            completableFuture = (java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply>) requests.get(reply.context);
+        }
+
+        if(completableFuture != null) {
+            completableFuture.complete(reply);
+
+            synchronized(requests) {
+                requests.remove(reply.context);
+            }
+        }
+    }
+""")
+
+jvpp_facade_callback_notification_method_template = Template("""
+    @Override
+    public void on$callback_dto($plugin_package.$dto_package.$callback_dto notification) {
+        notificationCallback.on$callback_dto(notification);
+    }
+""")
+
+jvpp_facade_details_callback_method_template = Template("""
+    @Override
+    @SuppressWarnings("unchecked")
+    public void on$callback_dto($plugin_package.$dto_package.$callback_dto reply) {
+        final $base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture<$plugin_package.$dto_package.$callback_dto_reply_dump> completableFuture;
+
+        synchronized(requests) {
+            completableFuture = ($base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture<$plugin_package.$dto_package.$callback_dto_reply_dump>) requests.get(reply.context);
+        }
+
+        if(completableFuture != null) {
+            completableFuture.getReplyDump().$callback_dto_field.add(reply);
+        }
+    }
+""")
+
+
+def generate_jvpp(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, future_facade_package, inputfile):
+    """ Generates JVpp interface and JNI implementation """
+    print "Generating JVpp future facade"
+
+    if not os.path.exists(future_facade_package):
+        os.mkdir(future_facade_package)
+
+    methods = []
+    methods_impl = []
+    callbacks = []
+    for func in func_list:
+        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
+
+        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_name_with_suffix):
+            continue
+
+        if not util.is_reply(camel_case_name_with_suffix) and not util.is_notification(func['name']):
+            continue
+
+        camel_case_method_name = util.underscore_to_camelcase(func['name'])
+
+        if not util.is_notification(func["name"]):
+            camel_case_request_method_name = util.remove_reply_suffix(util.underscore_to_camelcase(func['name']))
+            if util.is_details(camel_case_name_with_suffix):
+                camel_case_reply_name = get_standard_dump_reply_name(util.underscore_to_camelcase_upper(func['name']),
+                                                                     func['name'])
+                callbacks.append(jvpp_facade_details_callback_method_template.substitute(base_package=base_package,
+                                                                                         plugin_package=plugin_package,
+                                                                                         dto_package=dto_package,
+                                                                                         callback_dto=camel_case_name_with_suffix,
+                                                                                         callback_dto_field=camel_case_method_name,
+                                                                                         callback_dto_reply_dump=camel_case_reply_name + dto_gen.dump_dto_suffix,
+                                                                                         future_package=future_facade_package))
+
+                methods.append(future_jvpp_method_template.substitute(plugin_package=plugin_package,
+                                                                      dto_package=dto_package,
+                                                                      method_name=camel_case_request_method_name +
+                                                                                  util.underscore_to_camelcase_upper(util.dump_suffix),
+                                                                      reply_name=camel_case_reply_name + dto_gen.dump_dto_suffix,
+                                                                      request_name=util.remove_reply_suffix(camel_case_reply_name) +
+                                                                                   util.underscore_to_camelcase_upper(util.dump_suffix)))
+                methods_impl.append(future_jvpp_dump_method_impl_template.substitute(plugin_package=plugin_package,
+                                                                                     dto_package=dto_package,
+                                                                                     method_name=camel_case_request_method_name +
+                                                                                                 util.underscore_to_camelcase_upper(util.dump_suffix),
+                                                                                     reply_name=camel_case_reply_name + dto_gen.dump_dto_suffix,
+                                                                                     request_name=util.remove_reply_suffix(camel_case_reply_name) +
+                                                                                                  util.underscore_to_camelcase_upper(util.dump_suffix)))
+            else:
+                request_name = util.underscore_to_camelcase_upper(util.unconventional_naming_rep_req[func['name']]) \
+                    if func['name'] in util.unconventional_naming_rep_req else util.remove_reply_suffix(camel_case_name_with_suffix)
+
+                methods.append(future_jvpp_method_template.substitute(plugin_package=plugin_package,
+                                                                      dto_package=dto_package,
+                                                                      method_name=camel_case_request_method_name,
+                                                                      reply_name=camel_case_name_with_suffix,
+                                                                      request_name=request_name))
+                methods_impl.append(future_jvpp_method_impl_template.substitute(plugin_package=plugin_package,
+                                                                                dto_package=dto_package,
+                                                                                method_name=camel_case_request_method_name,
+                                                                                reply_name=camel_case_name_with_suffix,
+                                                                                request_name=request_name))
+
+                callbacks.append(jvpp_facade_callback_method_template.substitute(base_package=base_package,
+                                                                                 plugin_package=plugin_package,
+                                                                                 dto_package=dto_package,
+                                                                                 callback_dto=camel_case_name_with_suffix))
+
+        if util.is_notification(func["name"]):
+            callbacks.append(jvpp_facade_callback_notification_method_template.substitute(plugin_package=plugin_package,
+                                                                                          dto_package=dto_package,
+                                                                                          callback_dto=util.add_notification_suffix(camel_case_name_with_suffix)))
+
+    jvpp_file = open(os.path.join(future_facade_package, "FutureJVpp%sFacadeCallback.java" % plugin_name), 'w')
+    jvpp_file.write(jvpp_facade_callback_template.substitute(inputfile=inputfile,
+                                                             base_package=base_package,
+                                                             plugin_package=plugin_package,
+                                                             plugin_name=plugin_name,
+                                                             dto_package=dto_package,
+                                                             notification_package=notification_package,
+                                                             callback_package=callback_package,
+                                                             methods="".join(callbacks),
+                                                             future_package=future_facade_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+    jvpp_file = open(os.path.join(future_facade_package, "FutureJVpp%s.java" % plugin_name), 'w')
+    jvpp_file.write(future_jvpp_template.substitute(inputfile=inputfile,
+                                                    base_package=base_package,
+                                                    plugin_package=plugin_package,
+                                                    plugin_name=plugin_name,
+                                                    notification_package=notification_package,
+                                                    methods="".join(methods),
+                                                    future_package=future_facade_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+    jvpp_file = open(os.path.join(future_facade_package, "FutureJVpp%sFacade.java" % plugin_name), 'w')
+    jvpp_file.write(future_jvpp_facade_template.substitute(inputfile=inputfile,
+                                                           base_package=base_package,
+                                                           plugin_package=plugin_package,
+                                                           plugin_name=plugin_name,
+                                                           dto_package=dto_package,
+                                                           notification_package=notification_package,
+                                                           methods="".join(methods_impl),
+                                                           future_package=future_facade_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+
+future_jvpp_template = Template('''
+package $plugin_package.$future_package;
+
+/**
+ * 
Async facade extension adding specific methods for each request invocation
+ * 
It was generated by jvpp_future_facade_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface FutureJVpp${plugin_name} extends $base_package.$future_package.FutureJVppInvoker {
+$methods
+
+    @Override
+    public $plugin_package.$notification_package.${plugin_name}NotificationRegistry getNotificationRegistry();
+
+}
+''')
+
+future_jvpp_method_template = Template('''
+    java.util.concurrent.CompletionStage<$plugin_package.$dto_package.$reply_name> $method_name($plugin_package.$dto_package.$request_name request);
+''')
+
+
+future_jvpp_facade_template = Template('''
+package $plugin_package.$future_package;
+
+/**
+ * 
Implementation of FutureJVpp based on AbstractFutureJVppInvoker
+ * 
It was generated by jvpp_future_facade_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public class FutureJVpp${plugin_name}Facade extends $base_package.$future_package.AbstractFutureJVppInvoker implements FutureJVpp${plugin_name} {
+
+    private final $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl notificationRegistry = new $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl();
+
+    /**
+     * 
Create FutureJVpp${plugin_name}Facade object for provided JVpp instance.
+     * Constructor internally creates FutureJVppFacadeCallback class for processing callbacks
+     * and then connects to provided JVpp instance
+     *
+     * @param jvpp provided $base_package.JVpp instance
+     *
+     * @throws java.io.IOException in case instance cannot connect to JVPP
+     */
+    public FutureJVpp${plugin_name}Facade(final $base_package.JVppRegistry registry, final $base_package.JVpp jvpp) throws java.io.IOException {
+        super(jvpp, registry, new java.util.HashMap<>());
+        java.util.Objects.requireNonNull(registry, "JVppRegistry should not be null");
+        registry.register(jvpp, new FutureJVpp${plugin_name}FacadeCallback(getRequests(), notificationRegistry));
+    }
+
+    @Override
+    public $plugin_package.$notification_package.${plugin_name}NotificationRegistry getNotificationRegistry() {
+        return notificationRegistry;
+    }
+
+$methods
+}
+''')
+
+future_jvpp_method_impl_template = Template('''
+    @Override
+    public java.util.concurrent.CompletionStage<$plugin_package.$dto_package.$reply_name> $method_name($plugin_package.$dto_package.$request_name request) {
+        return send(request);
+    }
+''')
+
+future_jvpp_dump_method_impl_template = Template('''
+    @Override
+    public java.util.concurrent.CompletionStage<$plugin_package.$dto_package.$reply_name> $method_name($plugin_package.$dto_package.$request_name request) {
+        return send(request, new $plugin_package.$dto_package.$reply_name());
+    }
+''')
+
+
+# Returns request name or special one from unconventional_naming_rep_req map
+def get_standard_dump_reply_name(camel_case_dto_name, func_name):
+    # FIXME this is a hotfix for sub-details callbacks
+    # FIXME also for L2FibTableEntry
+    # It's all because unclear mapping between
+    #  request -> reply,
+    #  dump -> reply, details,
+    #  notification_start -> reply, notifications
+
+    # vpe.api needs to be "standardized" so we can parse the information and create maps before generating java code
+    suffix = func_name.split("_")[-1]
+    return util.underscore_to_camelcase_upper(
+        util.unconventional_naming_rep_req[func_name]) + util.underscore_to_camelcase_upper(suffix) if func_name in util.unconventional_naming_rep_req \
+        else camel_case_dto_name
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py
new file mode 100644
index 00000000..7bf91138
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, util
+from string import Template
+
+jvpp_ifc_template = Template("""
+package $plugin_package;
+
+/**
+ * 
Java representation of plugin's api file.
+ * 
It was generated by jvpp_impl_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface JVpp${plugin_name} extends $base_package.JVpp {
+
+    /**
+     * Generic dispatch method for sending requests to VPP
+     *
+     * @throws io.fd.vpp.jvpp.VppInvocationException if send request had failed
+     */
+    int send($base_package.$dto_package.JVppRequest request) throws io.fd.vpp.jvpp.VppInvocationException;
+
+$methods
+}
+""")
+
+jvpp_impl_template = Template("""
+package $plugin_package;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.logging.Logger;
+import $base_package.callback.JVppCallback;
+import $base_package.VppConnection;
+import $base_package.JVppRegistry;
+
+/**
+ * 
Default implementation of JVpp interface.
+ * 
It was generated by jvpp_impl_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public final class JVpp${plugin_name}Impl implements $plugin_package.JVpp${plugin_name} {
+
+    private final static Logger LOG = Logger.getLogger(JVpp${plugin_name}Impl.class.getName());
+    private static final String LIBNAME = "libjvpp_${plugin_name_underscore}.so";
+
+    // FIXME using NativeLibraryLoader makes load fail could not find (WantInterfaceEventsReply).
+    static {
+        try {
+            loadLibrary();
+        } catch (Exception e) {
+            LOG.severe("Can't find jvpp jni library: " + LIBNAME);
+            throw new ExceptionInInitializerError(e);
+        }
+    }
+
+    private static void loadStream(final InputStream is) throws IOException {
+        final Set perms = PosixFilePermissions.fromString("rwxr-x---");
+        final Path p = Files.createTempFile(LIBNAME, null, PosixFilePermissions.asFileAttribute(perms));
+        try {
+            Files.copy(is, p, StandardCopyOption.REPLACE_EXISTING);
+
+            try {
+                Runtime.getRuntime().load(p.toString());
+            } catch (UnsatisfiedLinkError e) {
+                throw new IOException("Failed to load library " + p, e);
+            }
+        } finally {
+            try {
+                Files.deleteIfExists(p);
+            } catch (IOException e) {
+            }
+        }
+    }
+
+    private static void loadLibrary() throws IOException {
+        try (final InputStream is = JVpp${plugin_name}Impl.class.getResourceAsStream('/' + LIBNAME)) {
+            if (is == null) {
+                throw new IOException("Failed to open library resource " + LIBNAME);
+            }
+            loadStream(is);
+        }
+    }
+
+    private VppConnection connection;
+    private JVppRegistry registry;
+
+    private static native void init0(final JVppCallback callback, final long queueAddress, final int clientIndex);
+    @Override
+    public void init(final JVppRegistry registry, final JVppCallback callback, final long queueAddress, final int clientIndex) {
+        this.registry = java.util.Objects.requireNonNull(registry, "registry should not be null");
+        this.connection = java.util.Objects.requireNonNull(registry.getConnection(), "connection should not be null");
+        connection.checkActive();
+        init0(callback, queueAddress, clientIndex);
+    }
+
+    private static native void close0();
+    @Override
+    public void close() {
+        close0();
+    }
+
+    @Override
+    public int send($base_package.$dto_package.JVppRequest request) throws io.fd.vpp.jvpp.VppInvocationException {
+        return request.send(this);
+    }
+
+    @Override
+    public final int controlPing(final io.fd.vpp.jvpp.dto.ControlPing controlPing) throws io.fd.vpp.jvpp.VppInvocationException {
+        return registry.controlPing(JVpp${plugin_name}Impl.class);
+    }
+
+$methods
+}
+""")
+
+method_template = Template("""    int $name($plugin_package.$dto_package.$request request) throws io.fd.vpp.jvpp.VppInvocationException;""")
+method_native_template = Template(
+    """    private static native int ${name}0($plugin_package.$dto_package.$request request);""")
+method_impl_template = Template("""    public final int $name($plugin_package.$dto_package.$request request) throws io.fd.vpp.jvpp.VppInvocationException {
+        java.util.Objects.requireNonNull(request,"Null request object");
+        connection.checkActive();
+        int result=${name}0(request);
+        if(result<0){
+            throw new io.fd.vpp.jvpp.VppInvocationException("${name}",result);
+        }
+        return result;
+    }
+""")
+
+no_arg_method_template = Template("""    int $name() throws io.fd.vpp.jvpp.VppInvocationException;""")
+no_arg_method_native_template = Template("""    private static native int ${name}0() throws io.fd.vpp.jvpp.VppInvocationException;""")
+no_arg_method_impl_template = Template("""    public final int $name() throws io.fd.vpp.jvpp.VppInvocationException {
+        connection.checkActive();
+        int result=${name}0();
+        if(result<0){
+            throw new io.fd.vpp.jvpp.VppInvocationException("${name}",result);
+        }
+        return result;
+    }
+""")
+
+
+def generate_jvpp(func_list, base_package, plugin_package, plugin_name_underscore, dto_package, inputfile):
+    """ Generates JVpp interface and JNI implementation """
+    print "Generating JVpp"
+    plugin_name = util.underscore_to_camelcase_upper(plugin_name_underscore)
+
+    methods = []
+    methods_impl = []
+    for func in func_list:
+
+        # Skip structures that are used only as notifications
+        if util.is_just_notification(func['name']) or util.is_ignored(func['name']):
+            continue
+
+        camel_case_name = util.underscore_to_camelcase(func['name'])
+        camel_case_name_upper = util.underscore_to_camelcase_upper(func['name'])
+        if util.is_reply(camel_case_name):
+            continue
+
+        if len(func['args']) == 0:
+            methods.append(no_arg_method_template.substitute(name=camel_case_name))
+            methods_impl.append(no_arg_method_native_template.substitute(name=camel_case_name))
+            methods_impl.append(no_arg_method_impl_template.substitute(name=camel_case_name))
+        else:
+            methods.append(method_template.substitute(name=camel_case_name,
+                                                      request=camel_case_name_upper,
+                                                      plugin_package=plugin_package,
+                                                      dto_package=dto_package))
+            methods_impl.append(method_native_template.substitute(name=camel_case_name,
+                                                                  request=camel_case_name_upper,
+                                                                  plugin_package=plugin_package,
+                                                                  dto_package=dto_package))
+            methods_impl.append(method_impl_template.substitute(name=camel_case_name,
+                                                                request=camel_case_name_upper,
+                                                                plugin_package=plugin_package,
+                                                                dto_package=dto_package))
+
+    jvpp_file = open("JVpp%s.java" % plugin_name, 'w')
+    jvpp_file.write(
+        jvpp_ifc_template.substitute(inputfile=inputfile,
+                                     methods="\n".join(methods),
+                                     base_package=base_package,
+                                     plugin_package=plugin_package,
+                                     plugin_name=plugin_name,
+                                     dto_package=dto_package))
+    jvpp_file.flush()
+    jvpp_file.close()
+
+    jvpp_file = open("JVpp%sImpl.java" % plugin_name, 'w')
+    jvpp_file.write(jvpp_impl_template.substitute(inputfile=inputfile,
+                                                  methods="\n".join(methods_impl),
+                                                  base_package=base_package,
+                                                  plugin_package=plugin_package,
+                                                  plugin_name=plugin_name,
+                                                  plugin_name_underscore=plugin_name_underscore,
+                                                  dto_package=dto_package))
+    jvpp_file.flush()
+    jvpp_file.close()
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py
new file mode 100644
index 00000000..94302d56
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import callback_gen
+import util
+from string import Template
+
+notification_registry_template = Template("""
+package $plugin_package.$notification_package;
+
+/**
+ * 
Registry for notification callbacks defined in ${plugin_name}.
+ * 
It was generated by notification_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface ${plugin_name}NotificationRegistry extends $base_package.$notification_package.NotificationRegistry {
+
+    $register_callback_methods
+
+    @Override
+    void close();
+}
+""")
+
+global_notification_callback_template = Template("""
+package $plugin_package.$notification_package;
+
+/**
+ * 
Aggregated callback interface for notifications only.
+ * 
It was generated by notification_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface Global${plugin_name}NotificationCallback$callbacks {
+
+}
+""")
+
+notification_registry_impl_template = Template("""
+package $plugin_package.$notification_package;
+
+/**
+ * 
Notification registry delegating notification processing to registered callbacks.
+ * 
It was generated by notification_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public final class ${plugin_name}NotificationRegistryImpl implements ${plugin_name}NotificationRegistry, Global${plugin_name}NotificationCallback {
+
+    // TODO add a special NotificationCallback interface and only allow those to be registered
+    private final java.util.concurrent.ConcurrentMap, $base_package.$callback_package.JVppNotificationCallback> registeredCallbacks =
+        new java.util.concurrent.ConcurrentHashMap<>();
+
+    $register_callback_methods
+    $handler_methods
+
+    @Override
+    public void close() {
+        registeredCallbacks.clear();
+    }
+}
+""")
+
+register_callback_impl_template = Template("""
+    public java.lang.AutoCloseable register$callback(final $plugin_package.$callback_package.$callback callback){
+        if(null != registeredCallbacks.putIfAbsent($plugin_package.$dto_package.$notification.class, callback)){
+            throw new IllegalArgumentException("Callback for " + $plugin_package.$dto_package.$notification.class +
+                "notification already registered");
+        }
+        return () -> registeredCallbacks.remove($plugin_package.$dto_package.$notification.class);
+    }
+""")
+
+handler_impl_template = Template("""
+    @Override
+    public void on$notification(
+        final $plugin_package.$dto_package.$notification notification) {
+        final $base_package.$callback_package.JVppNotificationCallback jVppNotificationCallback = registeredCallbacks.get($plugin_package.$dto_package.$notification.class);
+        if (null != jVppNotificationCallback) {
+            (($plugin_package.$callback_package.$callback) registeredCallbacks
+                .get($plugin_package.$dto_package.$notification.class))
+                .on$notification(notification);
+        }
+    }
+""")
+
+notification_provider_template = Template("""
+package $plugin_package.$notification_package;
+
+ /**
+ * Provides ${plugin_name}NotificationRegistry.
+ * 
The file was generated by notification_gen.py based on $inputfile
+ * 
(python representation of api file generated by vppapigen).
+ */
+public interface ${plugin_name}NotificationRegistryProvider extends $base_package.$notification_package.NotificationRegistryProvider {
+
+    @Override
+    public ${plugin_name}NotificationRegistry getNotificationRegistry();
+}
+""")
+
+
+def generate_notification_registry(func_list, base_package, plugin_package, plugin_name, notification_package, callback_package, dto_package, inputfile):
+    """ Generates notification registry interface and implementation """
+    print "Generating Notification interfaces and implementation"
+
+    if not os.path.exists(notification_package):
+        os.mkdir(notification_package)
+
+    callbacks = []
+    register_callback_methods = []
+    register_callback_methods_impl = []
+    handler_methods = []
+    for func in func_list:
+
+        if not util.is_notification(func['name']):
+            continue
+
+        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
+        notification_dto = util.add_notification_suffix(camel_case_name_with_suffix)
+        callback_ifc = notification_dto + callback_gen.callback_suffix
+        fully_qualified_callback_ifc = "{0}.{1}.{2}".format(plugin_package, callback_package, callback_ifc)
+        callbacks.append(fully_qualified_callback_ifc)
+
+        # TODO create NotificationListenerRegistration and return that instead of AutoCloseable to better indicate
+        # that the registration should be closed
+        register_callback_methods.append("java.lang.AutoCloseable register{0}({1} callback);"
+                                         .format(callback_ifc, fully_qualified_callback_ifc))
+        register_callback_methods_impl.append(register_callback_impl_template.substitute(plugin_package=plugin_package,
+                                                                                         callback_package=callback_package,
+                                                                                         dto_package=dto_package,
+                                                                                         notification=notification_dto,
+                                                                                         callback=callback_ifc))
+        handler_methods.append(handler_impl_template.substitute(base_package=base_package,
+                                                                plugin_package=plugin_package,
+                                                                callback_package=callback_package,
+                                                                dto_package=dto_package,
+                                                                notification=notification_dto,
+                                                                callback=callback_ifc))
+
+
+    callback_file = open(os.path.join(notification_package, "%sNotificationRegistry.java" % plugin_name), 'w')
+    callback_file.write(notification_registry_template.substitute(inputfile=inputfile,
+                                                                register_callback_methods="\n    ".join(register_callback_methods),
+                                                                base_package=base_package,
+                                                                plugin_package=plugin_package,
+                                                                plugin_name=plugin_name,
+                                                                notification_package=notification_package))
+    callback_file.flush()
+    callback_file.close()
+
+    callback_file = open(os.path.join(notification_package, "Global%sNotificationCallback.java" % plugin_name), 'w')
+
+    global_notification_callback_callbacks = ""
+    if (callbacks):
+        global_notification_callback_callbacks = " extends " + ", ".join(callbacks)
+
+    callback_file.write(global_notification_callback_template.substitute(inputfile=inputfile,
+                                                                       callbacks=global_notification_callback_callbacks,
+                                                                       plugin_package=plugin_package,
+                                                                       plugin_name=plugin_name,
+                                                                       notification_package=notification_package))
+    callback_file.flush()
+    callback_file.close()
+
+    callback_file = open(os.path.join(notification_package, "%sNotificationRegistryImpl.java" % plugin_name), 'w')
+    callback_file.write(notification_registry_impl_template.substitute(inputfile=inputfile,
+                                                                     callback_package=callback_package,
+                                                                     dto_package=dto_package,
+                                                                     register_callback_methods="".join(register_callback_methods_impl),
+                                                                     handler_methods="".join(handler_methods),
+                                                                     base_package=base_package,
+                                                                     plugin_package=plugin_package,
+                                                                     plugin_name=plugin_name,
+                                                                     notification_package=notification_package))
+    callback_file.flush()
+    callback_file.close()
+
+    callback_file = open(os.path.join(notification_package, "%sNotificationRegistryProvider.java" % plugin_name), 'w')
+    callback_file.write(notification_provider_template.substitute(inputfile=inputfile,
+                                                                     base_package=base_package,
+                                                                     plugin_package=plugin_package,
+                                                                     plugin_name=plugin_name,
+                                                                     notification_package=notification_package))
+    callback_file.flush()
+    callback_file.close()
+
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/types_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/types_gen.py
new file mode 100644
index 00000000..7a5eec37
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/types_gen.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from string import Template
+
+import util
+import jni_gen
+import dto_gen
+
+type_template = Template("""
+package $plugin_package.$type_package;
+
+/**
+ * 
This class represents $c_type_name type definition.
+ * 
It was generated by types_gen.py based on $inputfile preparsed data:
+ * 
+$docs
+ * 

+ */
+public final class $java_type_name {
+$fields
+$methods
+}
+""")
+
+field_template = Template("""    public $type $name;\n""")
+
+
+def generate_type_fields(type_definition):
+    """
+    Generates fields for class representing typeonly definition
+    :param type_definition: python representation of typeonly definition
+    :return: string representing class fields
+    """
+    fields = ""
+    for t in zip(type_definition['types'], type_definition['args']):
+        field_name = util.underscore_to_camelcase(t[1])
+        fields += field_template.substitute(type=util.jni_2_java_type_mapping[t[0]],
+                                            name=field_name)
+    return fields
+
+object_struct_setter_template = Template("""
+    {
+        jclass ${field_reference_name}Class = (*env)->FindClass(env, "${class_FQN}");
+        memset (&(mp->${c_name}), 0, sizeof (mp->${c_name}));
+        ${struct_initialization}
+    }
+""")
+
+object_array_struct_setter_template = Template("""
+    {
+        jclass ${field_reference_name}ArrayElementClass = (*env)->FindClass(env, "${class_FQN}");
+        if (${field_reference_name}) {
+            size_t _i;
+            jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
+            ${field_length_check}
+            for (_i = 0; _i < cnt; _i++) {
+                jobject ${field_reference_name}ArrayElement = (*env)->GetObjectArrayElement(env, ${field_reference_name}, _i);
+                memset (&(mp->${c_name}[_i]), 0, sizeof (mp->${c_name}[_i]));
+                ${struct_initialization}
+            }
+        }
+    }
+""")
+
+object_dto_field_setter_template = Template("""
+    {
+        jclass ${field_reference_name}Class = (*env)->FindClass(env, "${class_FQN}");
+        jmethodID ${field_reference_name}Constructor = (*env)->GetMethodID(env, ${field_reference_name}Class, "", "()V");
+        jobject ${field_reference_name} = (*env)->NewObject(env, ${field_reference_name}Class,  ${field_reference_name}Constructor);
+        ${type_initialization}
+        (*env)->SetObjectField(env, dto, ${field_reference_name}FieldId, ${field_reference_name});
+    }
+""")
+
+object_array_dto_field_setter_template = Template("""
+    {
+        jclass ${field_reference_name}Class = (*env)->FindClass(env, "${class_FQN}");
+        jobjectArray ${field_reference_name} = (*env)->NewObjectArray(env, ${field_length}, ${field_reference_name}Class, 0);
+        unsigned int _i;
+        for (_i = 0; _i < ${field_length}; _i++) {
+            jmethodID ${field_reference_name}Constructor = (*env)->GetMethodID(env, ${field_reference_name}Class, "", "()V");
+            jobject ${field_reference_name}ArrayElement = (*env)->NewObject(env, ${field_reference_name}Class,  ${field_reference_name}Constructor);
+            ${type_initialization}
+            (*env)->SetObjectArrayElement(env, ${field_reference_name}, _i, ${field_reference_name}ArrayElement);
+        }
+        (*env)->SetObjectField(env, dto, ${field_reference_name}FieldId, ${field_reference_name});
+    }
+""")
+
+
+def generate_struct_initialization(type_def, c_name_prefix, object_name, indent):
+    struct_initialization = ""
+    # field identifiers
+    for t in zip(type_def['types'], type_def['args'], type_def['lengths']):
+        field_reference_name = "${c_name}" + util.underscore_to_camelcase_upper(t[1])
+        field_name = util.underscore_to_camelcase(t[1])
+        struct_initialization += jni_gen.jni_request_binding_for_type(field_type=t[0], c_name=c_name_prefix + t[1],
+                                                                     field_reference_name=field_reference_name,
+                                                                     field_name=field_name,
+                                                                     field_length=t[2][0],
+                                                                     is_variable_len_array=t[2][1],
+                                                                     object_name=object_name)
+    return indent + struct_initialization.replace('\n', '\n' + indent)
+
+
+def generate_type_setter(handler_name, type_def, c_name_prefix, object_name, indent):
+    type_initialization = ""
+    for t in zip(type_def['types'], type_def['args'], type_def['lengths']):
+        field_length = t[2][0]
+        is_variable_len_array = t[2][1]
+        length_field_type = None
+        if is_variable_len_array:
+            length_field_type = type_def['types'][type_def['args'].index(field_length)]
+        type_initialization += jni_gen.jni_reply_handler_for_type(handler_name=handler_name,
+                                                                  ref_name="${field_reference_name}",
+                                                                  field_type=t[0], c_name=c_name_prefix + t[1],
+                                                                  field_reference_name="${c_name}" + util.underscore_to_camelcase_upper(t[1]),
+                                                                  field_name=util.underscore_to_camelcase(t[1]),
+                                                                  field_length=field_length,
+                                                                  is_variable_len_array=is_variable_len_array,
+                                                                  length_field_type=length_field_type,
+                                                                  object_name=object_name)
+    return indent + type_initialization.replace('\n', '\n' + indent)
+
+
+def generate_types(types_list, plugin_package, types_package, inputfile):
+    """
+    Generates Java representation of custom types defined in api file.
+    """
+
+    #
+    if not types_list:
+        print "Skipping custom types generation (%s does not define custom types)." % inputfile
+        return
+
+    print "Generating custom types"
+
+    if not os.path.exists(types_package):
+        os.mkdir(types_package)
+
+    for type in types_list:
+        c_type_name = type['name']
+        java_type_name = util.underscore_to_camelcase_upper(type['name'])
+        dto_path = os.path.join(types_package, java_type_name + ".java")
+
+        fields = generate_type_fields(type)
+
+        dto_file = open(dto_path, 'w')
+        dto_file.write(type_template.substitute(plugin_package=plugin_package,
+                                                type_package=types_package,
+                                                c_type_name=c_type_name,
+                                                inputfile=inputfile,
+                                                docs=util.api_message_to_javadoc(type),
+                                                java_type_name=java_type_name,
+                                                fields=fields,
+                                                methods=dto_gen.generate_dto_base_methods(java_type_name, type)
+                                                ))
+
+        # update type mappings:
+        # todo fix vpe.api to use type_name instead of vl_api_type_name_t
+        type_name = "vl_api_" + c_type_name + "_t"
+        java_fqn = "%s.%s.%s" % (plugin_package, types_package, java_type_name)
+        util.vpp_2_jni_type_mapping[type_name] = "jobject"
+        util.vpp_2_jni_type_mapping[type_name + "[]"] = "jobjectArray"
+        util.jni_2_java_type_mapping[type_name] = java_fqn
+        util.jni_2_java_type_mapping[type_name + "[]"] = java_fqn + "[]"
+        jni_name = java_fqn.replace('.', "/")
+        jni_signature = "L" + jni_name + ";"
+        util.jni_2_signature_mapping[type_name] = "L" + jni_name + ";"
+        util.jni_2_signature_mapping[type_name + "[]"] = "[" + jni_signature
+        util.jni_field_accessors[type_name] = "ObjectField"
+        util.jni_field_accessors[type_name + "[]"] = "ObjectField"
+
+        jni_gen.struct_setter_templates[type_name] = Template(
+                object_struct_setter_template.substitute(
+                        c_name="${c_name}",
+                        field_reference_name="${field_reference_name}",
+                        class_FQN=jni_name,
+                        struct_initialization=generate_struct_initialization(type, "${c_name}.",
+                                                                           "${field_reference_name}", ' ' * 4))
+        )
+
+        jni_gen.struct_setter_templates[type_name+ "[]"] = Template(
+                object_array_struct_setter_template.substitute(
+                        c_name="${c_name}",
+                        field_reference_name="${field_reference_name}",
+                        field_length_check="${field_length_check}",
+                        class_FQN=jni_name,
+                        struct_initialization=generate_struct_initialization(type, "${c_name}[_i].",
+                                                                           "${field_reference_name}ArrayElement", ' ' * 8))
+        )
+
+        jni_gen.dto_field_setter_templates[type_name] = Template(
+                object_dto_field_setter_template.substitute(
+                        field_reference_name="${field_reference_name}",
+                        field_length="${field_length}",
+                        class_FQN=jni_name,
+                        type_initialization=generate_type_setter(c_type_name, type, "${c_name}.",
+                                                                 "${field_reference_name}", ' ' * 4))
+        )
+
+        jni_gen.dto_field_setter_templates[type_name + "[]"] = Template(
+                object_array_dto_field_setter_template.substitute(
+                        field_reference_name="${field_reference_name}",
+                        field_length="${field_length}",
+                        class_FQN=jni_name,
+                        type_initialization=generate_type_setter(c_type_name, type, "${c_name}[_i].",
+                                                                 "${field_reference_name}ArrayElement", ' ' * 8))
+        )
+
+        dto_file.flush()
+        dto_file.close()
+
diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/util.py b/src/vpp-api/java/jvpp/gen/jvppgen/util.py
new file mode 100644
index 00000000..fc971c17
--- /dev/null
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/util.py
@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, pprint
+from os import removedirs
+
+
+def underscore_to_camelcase(name):
+    name = name.title().replace("_", "")
+    return name[0].lower() + name[1:]
+
+
+def underscore_to_camelcase_upper(name):
+    name = name.title().replace("_", "")
+    return name[0].upper() + name[1:]
+
+
+def remove_folder(folder):
+    """ Remove folder with all its files """
+    for root, dirs, files in os.walk(folder, topdown=False):
+        for name in files:
+            os.remove(os.path.join(root, name))
+        removedirs(folder)
+
+
+reply_suffixes = ("reply", "details", "l2fibtableentry")
+
+
+def is_reply(name):
+    return name.lower().endswith(reply_suffixes)
+
+
+def is_details(name):
+    return name.lower().endswith(reply_suffixes[1]) or name.lower().endswith(reply_suffixes[2])
+
+
+def is_retval_field(name):
+    return name == 'retval'
+
+dump_suffix = "dump"
+
+
+def is_dump(name):
+    return name.lower().endswith(dump_suffix)
+
+
+def get_reply_suffix(name):
+    for reply_suffix in reply_suffixes:
+        if name.lower().endswith(reply_suffix):
+            if reply_suffix == reply_suffixes[2]:
+                # FIXME workaround for l2_fib_table_entry
+                return 'entry'
+            else:
+                return reply_suffix
+
+# Mapping according to:
+# http://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/types.html
+#
+# Unsigned types are converted to signed java types that have the same size.
+# It is the API user responsibility to interpret them correctly.
+jni_2_java_type_mapping = {'u8': 'byte',
+                           'u8[]': 'byte[]',
+                           'i8': 'byte',
+                           'i8[]': 'byte[]',
+                           'u16': 'short',
+                           'u16[]': 'short[]',
+                           'i16': 'short',
+                           'i16[]': 'short[]',
+                           'u32': 'int',
+                           'u32[]': 'int[]',
+                           'i32': 'int',
+                           'i32[]': 'int[]',
+                           'u64': 'long',
+                           'u64[]': 'long[]',
+                           'i64': 'long',
+                           'i64[]': 'long[]',
+                           'f64': 'double',
+                           'f64[]': 'double[]'
+                           }
+
+vpp_2_jni_type_mapping = {'u8': 'jbyte',
+                          'u8[]': 'jbyteArray',
+                          'i8': 'jbyte',
+                          'u8[]': 'jbyteArray',
+                          'u16': 'jshort',
+                          'u16[]': 'jshortArray',
+                          'i16': 'jshort',
+                          'i16[]': 'jshortArray',
+                          'u32': 'jint',
+                          'u32[]': 'jintArray',
+                          'i32': 'jint',
+                          'i32[]': 'jintArray',
+                          'u64': 'jlong',
+                          'u64[]': 'jlongArray',
+                          'i64': 'jlong',
+                          'i64[]': 'jlongArray',
+                          'f64': 'jdouble',
+                          'f64[]': 'jdoubleArray'
+                          }
+
+# https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#type_signatures
+jni_2_signature_mapping = {'u8': 'B',
+                           'u8[]': '[B',
+                           'i8': 'B',
+                           'i8[]': '[B',
+                           'u16': 'S',
+                           'u16[]': '[S',
+                           'i16': 'S',
+                           'i16[]': '[S',
+                           'u32': 'I',
+                           'u32[]': '[I',
+                           'i32': 'I',
+                           'i32[]': '[I',
+                           'u64': 'J',
+                           'u64[]': '[J',
+                           'i64': 'J',
+                           'i64[]': '[J',
+                           'f64': 'D',
+                           'f64[]': '[D'
+                           }
+
+# https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#Get_type_Field_routines
+jni_field_accessors =  {'u8': 'ByteField',
+                        'u8[]': 'ObjectField',
+                        'i8': 'ByteField',
+                        'i8[]': 'ObjectField',
+                        'u16': 'ShortField',
+                        'u16[]': 'ObjectField',
+                        'i16': 'ShortField',
+                        'i16[]': 'ObjectField',
+                        'u32': 'IntField',
+                        'u32[]': 'ObjectField',
+                        'i32': 'IntField',
+                        'i32[]': 'ObjectField',
+                        'u64': 'LongField',
+                        'u64[]': 'ObjectField',
+                        'i64': 'LongField',
+                        'i64[]': 'ObjectField',
+                        'f64': 'DoubleField',
+                        'f64[]': 'ObjectField'
+                        }
+
+
+# vpe.api calls that do not follow naming conventions and have to be handled exceptionally when finding reply -> request mapping
+# FIXME in vpe.api
+unconventional_naming_rep_req = {
+                                 'cli_reply': 'cli_request',
+                                 'vnet_summary_stats_reply': 'vnet_get_summary_stats',
+                                 # This below is actually a sub-details callback. We cannot derive the mapping of dump request
+                                 # belonging to this sub-details from naming conventions. We need special mapping
+                                 'bridge_domain_sw_if_details': 'bridge_domain',
+                                 # This is standard dump call + details reply. However it's not called details but entry
+                                 'l2_fib_table_entry': 'l2_fib_table'
+                                 }
+
+#
+# FIXME no convention in the naming of events (notifications) in vpe.api
+notifications_message_suffixes = ("event", "counters")
+notification_messages_reused = ["sw_interface_set_flags"]
+
+# messages that must be ignored. These messages are INSUFFICIENTLY marked as disabled in vpe.api
+# FIXME
+ignored_messages = ["is_address_reachable"]
+
+
+def is_notification(name):
+    """ Returns true if the structure is a notification regardless of its no other use """
+    return is_just_notification(name) or name.lower() in notification_messages_reused
+
+
+def is_just_notification(name):
+    """ Returns true if the structure is just a notification and has no other use """
+    return name.lower().endswith(notifications_message_suffixes)
+
+
+def is_ignored(param):
+    return param.lower() in ignored_messages
+
+
+def remove_reply_suffix(camel_case_name_with_suffix):
+    return remove_suffix(camel_case_name_with_suffix, get_reply_suffix(camel_case_name_with_suffix))
+
+
+def remove_suffix(camel_case_name_with_suffix, suffix):
+    suffix_length = len(suffix)
+    return camel_case_name_with_suffix[:-suffix_length] if suffix_length != 0 else camel_case_name_with_suffix
+
+
+def is_control_ping(camel_case_name_with_suffix):
+    return camel_case_name_with_suffix.lower().startswith("controlping");
+
+
+def api_message_to_javadoc(api_message):
+    """ Converts vpe.api message description to javadoc """
+    str = pprint.pformat(api_message, indent=4, width=120, depth=None)
+    return " * " + str.replace("\n", "\n * ")
+
+
+notification_dto_suffix = "Notification"
+
+
+def add_notification_suffix(camel_case_dto_name):
+    camel_case_dto_name += notification_dto_suffix
+    return camel_case_dto_name
+
+
+def is_array(java_type_as_string):
+    return java_type_as_string.endswith("[]")
diff --git a/src/vpp-api/lua/README.md b/src/vpp-api/lua/README.md
new file mode 100644
index 00000000..4ecdb34d
--- /dev/null
+++ b/src/vpp-api/lua/README.md
@@ -0,0 +1,50 @@
+This is the experimental version of Lua API, aimed for the luajit use.
+
+Please take a look and send the feedback to ayourtch@gmail.com.
+
+To run the examples here:
+
+1) install luajit - "sudo apt-get install luajit" on ubuntu
+
+2) "make build-vpp-api" in the top VPP directory
+
+3) "make run" in a separate terminal window
+   This ensures you have an instance of VPP running
+
+4) sudo luajit examples/example-cli.lua
+
+This will result in something like this:
+
+Version:
+00000000  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  ................
+00000010  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  ................
+
+{ [1] = { ["luaapi_message_name"] = show_version_reply,["program"] = vpe,["version"] = ,["build_date"] = Fri Nov 25 10:58:48 UTC 2016,["retval"] = 0,["build_directory"] = /home/ubuntu/vpp,["_vl_msg_id"] = 170,["context"] = 0,} ,}
+---
+{ [1] = { ["luaapi_message_name"] = cli_inband_reply,["_vl_msg_id"] = 94,["length"] = 66,["reply"] = vpp v built by ubuntu on vpp-toys at Fri Nov 25 10:58:48 UTC 2016
+,["retval"] = 0,["context"] = 0,} ,}
+---
+
+5) You can also run the performance test bench:
+
+$ sudo luajit bench.lua
+10001 iterations, average speed 5624LL per second
+10001 iterations, average speed 6650LL per second
+10001 iterations, average speed 6053LL per second
+10001 iterations, average speed 7056LL per second
+10001 iterations, average speed 6388LL per second
+10001 iterations, average speed 5849LL per second
+10001 iterations, average speed 6321LL per second
+10001 iterations, average speed 6368LL per second
+10001 iterations, average speed 5958LL per second
+10001 iterations, average speed 6482LL per second
+Average tps across the tests: 6274LL
+
+Note: the above is run in an lxd container running inside 2-core
+xhyve VM on a Macbook Pro, so I would not take the performance numbers for granted :)
+
+The "examples" directory contains a few naive examples, as well as a couple of more 
+advanced ones - a tab-completing CLI for VPP that can call both the APIs and CLI,
+and also a small test utility which I use for automating some small tests using
+VPP.
+
diff --git a/src/vpp-api/lua/bench.lua b/src/vpp-api/lua/bench.lua
new file mode 100644
index 00000000..8e5a0b4b
--- /dev/null
+++ b/src/vpp-api/lua/bench.lua
@@ -0,0 +1,70 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+local vpp = require "vpp-lapi"
+
+local ffi = require "ffi"
+
+ffi.cdef([[
+   struct timespec {
+               long tv_sec;        /* seconds */
+               long tv_nsec;       /* nanoseconds */
+           };
+
+   int clock_gettime(int clk_id, struct timespec *tp);
+]])
+
+
+local time_cache = ffi.new("struct timespec[1]")
+local time_cache_1 = time_cache[0]
+function get_ns()
+  ffi.C.clock_gettime(0, time_cache)
+  return time_cache_1.tv_nsec + 1000000000 * time_cache_1.tv_sec
+end
+
+function do_bench()
+  local cycle_start = get_ns()
+  local n_iterations =  10000
+  local count = 1
+  for i = 1,n_iterations do
+    -- print(i)
+    vpp:api_call("show_version")
+    count = count + 1
+    -- print(i, "done")
+  end
+  cycle_end = get_ns()
+  local tps = n_iterations*1000000000LL/(cycle_end - cycle_start)
+  print (tostring(count) .. " iterations, average speed " .. tostring(tps) .. " per second")
+  return tps
+end
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+vpp:init({ pneum_path = pneum_path })
+vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+
+vpp:connect("lua-bench")
+local n_tests = 10
+local tps_acc = 0LL
+for i=1,n_tests do
+  tps_acc = tps_acc + do_bench()
+end
+print("Average tps across the tests: " .. tostring(tps_acc/n_tests))
+
+vpp:disconnect()
+
+
diff --git a/src/vpp-api/lua/examples/cli/README.md b/src/vpp-api/lua/examples/cli/README.md
new file mode 100644
index 00000000..3a5f8ee9
--- /dev/null
+++ b/src/vpp-api/lua/examples/cli/README.md
@@ -0,0 +1,5 @@
+This is a small experiment to have a wrapper CLI which can call both API functions as well as debug CLI.
+
+To facilitate tab completion and help, the API call names are broken up with spaces replacing the underscores.
+
+
diff --git a/src/vpp-api/lua/examples/cli/lua-cli.lua b/src/vpp-api/lua/examples/cli/lua-cli.lua
new file mode 100644
index 00000000..b3a24d7d
--- /dev/null
+++ b/src/vpp-api/lua/examples/cli/lua-cli.lua
@@ -0,0 +1,747 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+-- Experimental prototype CLI using API to VPP, with tab completion
+--
+-- Written by Andrew Yourtchenko (ayourtch@cisco.com) 2010,2016
+--
+
+vpp = require "vpp-lapi"
+
+
+local dotdotdot = "..."
+
+-- First the "readline" routine
+
+readln = {
+split = function(str, pat)
+  local t = {}  -- NOTE: use {n = 0} in Lua-5.0
+  local fpat = "(.-)" .. pat
+  local last_end = 1
+  if str then
+    local s, e, cap = str:find(fpat, 1)
+    while s do
+      if s ~= 1 or cap ~= "" then
+        table.insert(t,cap)
+      end
+      last_end = e+1
+      s, e, cap = str:find(fpat, last_end)
+    end
+    if last_end <= #str then
+      cap = str:sub(last_end)
+      table.insert(t, cap)
+    end
+  end
+  return t
+end,
+
+reader = function()
+  local rl = {}
+
+  rl.init = function()
+    os.execute("stty -icanon min 1 -echo")
+    rl.rawmode = true
+  end
+
+  rl.done = function()
+    os.execute("stty icanon echo")
+    rl.rawmode = false
+  end
+
+  rl.prompt = ">"
+  rl.history = { "" }
+  rl.history_index = 1
+  rl.history_length = 1
+
+  rl.hide_cmd = function()
+    local bs = string.char(8) .. " " .. string.char(8)
+    for i = 1, #rl.command do
+      io.stdout:write(bs)
+    end
+  end
+
+  rl.show_cmd = function()
+    if rl.command then
+      io.stdout:write(rl.command)
+    end
+  end
+
+  rl.store_history = function(cmd)
+    if cmd == "" then
+      return
+    end
+    rl.history[rl.history_length] = cmd
+    rl.history_length = rl.history_length + 1
+    rl.history_index = rl.history_length
+    rl.history[rl.history_length] = ""
+  end
+
+  rl.readln = function()
+    local done = false
+    local need_prompt = true
+    rl.command = ""
+
+    if not rl.rawmode then
+      rl.init()
+    end
+
+    while not done do
+      if need_prompt then
+        io.stdout:write(rl.prompt)
+	io.stdout:write(rl.command)
+        need_prompt = false
+      end
+
+      local ch = io.stdin:read(1)
+      if ch:byte(1) == 27 then
+        -- CONTROL
+        local ch2 = io.stdin:read(1)
+        -- arrows
+        if ch2:byte(1) == 91 then
+          local ch3 = io.stdin:read(1)
+          local b = ch3:byte(1)
+          if b == 65 then
+            ch = "UP"
+          elseif b == 66 then
+            ch = "DOWN"
+          elseif b == 67 then
+            ch = "RIGHT"
+          elseif b == 68 then
+            ch = "LEFT"
+          end
+          -- print("Byte: " .. ch3:byte(1))
+          -- if ch3:byte(1)
+        end
+      end
+
+      if ch == "?" then
+        io.stdout:write(ch)
+        io.stdout:write("\n")
+        if rl.help then
+          rl.help(rl)
+        end
+        need_prompt = true
+      elseif ch == "\t" then
+        if rl.tab_complete then
+          rl.tab_complete(rl)
+        end
+        io.stdout:write("\n")
+        need_prompt = true
+      elseif ch == "\n" then
+        io.stdout:write(ch)
+        done = true
+      elseif ch == "\004" then
+        io.stdout:write("\n")
+        rl.command = nil
+	done = true
+      elseif ch == string.char(127) then
+        if rl.command ~= "" then
+          io.stdout:write(string.char(8) .. " " .. string.char(8))
+          rl.command = string.sub(rl.command, 1, -2)
+        end
+      elseif #ch > 1 then
+        -- control char
+        if ch == "UP" then
+          rl.hide_cmd()
+          if rl.history_index == #rl.history then
+            rl.history[rl.history_index] = rl.command
+          end
+          if rl.history_index > 1 then
+            rl.history_index = rl.history_index - 1
+            rl.command = rl.history[rl.history_index]
+          end
+          rl.show_cmd()
+        elseif ch == "DOWN" then
+          rl.hide_cmd()
+          if rl.history_index < rl.history_length then
+            rl.history_index = rl.history_index + 1
+            rl.command = rl.history[rl.history_index]
+          end
+          rl.show_cmd()
+        end
+      else
+        io.stdout:write(ch)
+        rl.command = rl.command .. ch
+      end
+    end
+    if rl.command then
+      rl.store_history(rl.command)
+    end
+    return rl.command
+  end
+  return rl
+end
+
+}
+
+--[[
+
+r = reader()
+
+local done = false
+
+while not done do
+  local cmd = r.readln()
+  print("Command: " .. tostring(cmd))
+  if not cmd or cmd == "quit" then
+    done = true
+  end
+end
+
+r.done()
+
+]]
+
+--------- MDS show tech parser
+
+local print_section = nil
+local list_sections = false
+
+local curr_section = "---"
+local curr_parser = nil
+
+-- by default operate in batch mode
+local batch_mode = true
+
+local db = {}
+local device = {}
+device.output = {}
+local seen_section = {}
+
+function start_collection(name)
+  device = {}
+  seen_section = {}
+end
+
+function print_error(errmsg)
+  print("@#$:" .. errmsg)
+end
+
+function keys(tbl)
+  local t = {}
+  for k, v in pairs(tbl) do
+    table.insert(t, k)
+  end
+  return t
+end
+
+function tset (parent, ...)
+
+  -- print ('set', ...)
+
+  local len = select ('#', ...)
+  local key, value = select (len-1, ...)
+  local cutpoint, cutkey
+
+  for i=1,len-2 do
+
+    local key = select (i, ...)
+    local child = parent[key]
+
+    if value == nil then
+      if child == nil then  return
+      elseif next (child, next (child)) then  cutpoint = nil  cutkey = nil
+      elseif cutpoint == nil then  cutpoint = parent  cutkey = key  end
+
+    elseif child == nil then  child = {}  parent[key] = child  end
+
+    parent = child
+    end
+
+  if value == nil and cutpoint then  cutpoint[cutkey] = nil
+  else  parent[key] = value  return value  end
+  end
+
+
+function tget (parent, ...)
+  local len = select ('#', ...)
+  for i=1,len do
+    parent = parent[select (i, ...)]
+    if parent == nil then  break  end
+    end
+  return parent
+  end
+
+
+local pager_lines = 23
+local pager_printed = 0
+local pager_skipping = false
+local pager_filter_pipe = nil
+
+function pager_reset()
+  pager_printed = 0
+  pager_skipping = false
+  if pager_filter_pipe then
+    pager_filter_pipe:close()
+    pager_filter_pipe = nil
+  end
+end
+
+
+function print_more()
+  io.stdout:write(" --More-- ")
+end
+
+function print_nomore()
+  local bs = string.char(8)
+  local bs10 = bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs
+  io.stdout:write(bs10 .. "          " .. bs10)
+end
+
+function print_line(txt)
+  if pager_filter_pipe then
+    pager_filter_pipe:write(txt .. "\n")
+    return
+  end
+  if pager_printed >= pager_lines then
+    print_more()
+    local ch = io.stdin:read(1)
+    if ch == " " then
+      pager_printed = 0
+    elseif ch == "\n" then
+      pager_printed = pager_printed - 1
+    elseif ch == "q" then
+      pager_printed = 0
+      pager_skipping = true
+    end
+    print_nomore()
+  end
+  if not pager_skipping then
+    print(txt)
+    pager_printed = pager_printed + 1
+  else
+    -- skip printing
+  end
+end
+
+function paged_write(text)
+  local t = readln.split(text, "[\n]")
+  if string.sub(text, -1) == "\n" then
+    table.insert(t, "")
+  end
+  for i, v in ipairs(t) do
+    if i < #t then
+      print_line(v)
+    else
+      if pager_filter_pipe then
+        pager_filter_pipe:write(v)
+      else
+        io.stdout:write(v)
+      end
+    end
+  end
+end
+
+
+
+
+
+function get_choices(tbl, key)
+  local res = {}
+  for k, v in pairs(tbl) do
+    if string.sub(k, 1, #key) == key then
+      table.insert(res, k)
+    elseif 0 < #key and dotdotdot == k then
+      table.insert(res, k)
+    end
+  end
+  return res
+end
+
+function get_exact_choice(choices, val)
+  local exact_idx = nil
+  local substr_idx = nil
+  local substr_seen = false
+
+  if #choices == 1 then
+    if choices[1] == dotdotdot then
+      return 1
+    elseif string.sub(choices[1], 1, #val) == val then
+      return 1
+    else
+      return nil
+    end
+  else
+    for i, v in ipairs(choices) do
+      if v == val then
+        exact_idx = i
+        substr_seen = true
+      elseif choices[i] ~= dotdotdot and string.sub(choices[i], 1, #val) == val then
+        if substr_seen then
+          substr_idx = nil
+        else
+          substr_idx = i
+          substr_seen = true
+        end
+      elseif choices[i] == dotdotdot then
+        if substr_seen then
+          substr_idx = nil
+        else
+          substr_idx = i
+          substr_seen = true
+        end
+      end
+    end
+  end
+  return exact_idx or substr_idx
+end
+
+function device_cli_help(rl)
+  local key = readln.split(rl.command, "[ ]+")
+  local tree = rl.tree
+  local keylen = #key
+  local fullcmd = ""
+  local error = false
+  local terse = true
+
+  if ((#rl.command >= 1) and (string.sub(rl.command, -1) == " ")) or (#rl.command == 0) then
+    table.insert(key, "")
+    terse = false
+  end
+
+  for i, v in ipairs(key) do
+    local choices = get_choices(tree, v)
+    local idx = get_exact_choice(choices, v)
+    if idx then
+      local choice = choices[idx]
+      tree = tree[choice]
+      fullcmd = fullcmd .. choice .. " "
+    else
+      if i < #key then
+        error = true
+      end
+    end
+
+    if i == #key and not error then
+      for j, w in ipairs(choices) do
+        if terse then
+          paged_write(w .. "\t")
+        else
+          paged_write("  " .. w .. "\n")
+        end
+      end
+      paged_write("\n")
+      if terse then
+        paged_write(" \n")
+      end
+    end
+  end
+  pager_reset()
+end
+
+function device_cli_tab_complete(rl)
+  local key = readln.split(rl.command, "[ ]+")
+  local tree = rl.tree
+  local keylen = #key
+  local fullcmd = ""
+  local error = false
+
+  for i, v in ipairs(key) do
+    local choices = get_choices(tree, v)
+    local idx = get_exact_choice(choices, v)
+    if idx and choices[idx] ~= dotdotdot then
+      local choice = choices[idx]
+      tree = tree[choice]
+      -- print("level " .. i .. " '" .. choice .. "'")
+      fullcmd = fullcmd .. choice .. " "
+    else
+      -- print("level " .. i .. " : " .. table.concat(choices, " ") .. " ")
+      error = true
+    end
+  end
+  if not error then
+    rl.command = fullcmd
+  else
+    -- print("\n\nerror\n")
+  end
+  pager_reset()
+end
+
+function device_cli_exec(rl)
+
+  local cmd_nopipe = rl.command
+  local cmd_pipe = nil
+
+  local pipe1, pipe2 = string.find(rl.command, "[|]")
+  if pipe1 then
+    cmd_nopipe = string.sub(rl.command, 1, pipe1-1)
+    cmd_pipe = string.sub(rl.command, pipe2+1, -1)
+  end
+
+  local key = readln.split(cmd_nopipe .. " ", "[ ]+")
+  local tree = rl.tree
+  local keylen = #key
+  local fullcmd = ""
+  local error = false
+  local func = nil
+
+  if cmd_pipe then
+    pager_filter_pipe = io.popen(cmd_pipe, "w")
+  end
+
+
+  rl.choices = {}
+
+  for i, v in ipairs(key) do
+    local choices = get_choices(tree, v)
+    local idx = get_exact_choice(choices, v)
+    if idx then
+      local choice = choices[idx]
+      if i == #key then
+        func = tree[choice]
+      else
+        if choice == dotdotdot then
+          -- keep the tree the same, update the choice value to match the input string
+          choices[idx] = v
+          choice = v
+        else
+          tree = tree[choice]
+        end
+      end
+      -- print("level " .. i .. " '" .. choice .. "'")
+      table.insert(rl.choices, choice)
+    else
+      -- print("level " .. i .. " : " .. table.concat(choices, " ") .. " ")
+      error = true
+      return nil
+    end
+  end
+  return func
+end
+
+function populate_tree(commands)
+  local tree = {}
+
+  for k, v in pairs(commands) do
+    local key = readln.split(k .. " ", "[ ]+")
+    local xtree = tree
+    for i, kk in ipairs(key) do
+      if i == 1 and kk == "sh" then
+        kk = "show"
+      end
+      if i == #key then
+        if type(v) == "function" then
+          xtree[kk] = v
+        else
+          xtree[kk] = function(rl) paged_write(table.concat(v, "\n") .. "\n") end
+        end
+      else
+        if not xtree[kk] then
+          xtree[kk] = {}
+        end
+        xtree = xtree[kk]
+      end
+    end
+  end
+  return tree
+end
+
+function trim (s)
+  return (string.gsub(s, "^%s*(.-)%s*$", "%1"))
+end
+
+
+function init_vpp(vpp)
+  local root_dir = "/home/ubuntu/vpp"
+  local pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+
+  vpp:init({ pneum_path = pneum_path })
+
+  vpp:init({ pneum_path = pneum_path })
+  vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+
+
+
+  vpp:connect("lua_cli")
+end
+
+function run_cli(vpp, cli)
+  local reply = vpp:api_call("cli_inband", { cmd = cli })
+  if reply and #reply == 1 then
+    local rep = reply[1]
+    if 0 == rep.retval then
+      return rep.reply
+    else
+      return "XXXXXLUACLI: API RETVAL ERROR : " .. tostring(rep.retval)
+    end
+  else
+    return "XXXXXLUACLI ERROR, RAW REPLY: " .. vpp.dump(reply)
+  end
+end
+
+
+function toprintablestring(s)
+  if type(s) == "string" then
+    return "\n"..vpp.hex_dump(s)
+  else
+    return tostring(s)
+  end
+end
+
+function interactive_cli(r)
+  while not done do
+    pager_reset()
+    local cmd = r.readln()
+    if not cmd then
+      done = true
+    elseif cmd == "quit" or cmd == "exit" then
+      done = true
+    else
+      local func = device_cli_exec(r)
+      if func then
+	func(r)
+      else
+	if trim(cmd) == "" then
+	else
+	  for i = 1, #r.prompt do
+	    paged_write(" ")
+	  end
+	  paged_write("^\n% Invalid input detected at '^' marker.\n\n")
+	end
+      end
+    end
+  end
+end
+
+device = {}
+device.output = {}
+
+init_vpp(vpp)
+cmds_str = run_cli(vpp, "?")
+vpp_cmds = readln.split(cmds_str, "\n")
+vpp_clis = {}
+
+for linenum, line in ipairs(vpp_cmds) do
+  local m,h = string.match(line, "^  (.-)  (.*)$")
+  if m and #m > 0 then
+    table.insert(vpp_clis, m)
+    device.output["vpp debug cli " .. m] = function(rl)
+      -- print("ARBITRARY CLI" .. vpp.dump(rl.choices))
+      print("LUACLI command: " .. table.concat(rl.choices, " "))
+      local sub = {}
+      --
+      for i=4, #rl.choices -1 do
+        table.insert(sub, rl.choices[i])
+      end
+      local cli = table.concat(sub, " ")
+      print("Running CLI: " .. tostring(cli))
+      paged_write(run_cli(vpp, cli))
+    end
+    device.output["vpp debug cli " .. m .. " " .. dotdotdot] = function(rl)
+      print("ARGH")
+    end
+
+    local ret = run_cli(vpp, "help " .. m)
+    device.output["help vpp debug cli " .. m] = { ret }
+  end
+end
+
+for linenum, line in ipairs(vpp_clis) do
+  -- print(line, ret)
+end
+
+for msgnum, msgname in pairs(vpp.msg_number_to_name) do
+  local cli, numspaces = string.gsub(msgname, "_", " ")
+  device.output["call " .. cli .. " " .. dotdotdot] = function(rl)
+    print("ARGH")
+  end
+  device.output["call " .. cli] = function(rl)
+    print("LUACLI command: " .. table.concat(rl.choices, " "))
+    print("Running API: " .. msgname) -- vpp.dump(rl.choices))
+    local out = {}
+    local args = {}
+    local ntaken = 0
+    local argname = ""
+    for i=(1+1+numspaces+1), #rl.choices-1 do
+      -- print(i, rl.choices[i])
+      if ntaken > 0 then
+        ntaken = ntaken -1
+      else
+        local fieldname = rl.choices[i]
+        local field = vpp.msg_name_to_fields[msgname][fieldname]
+        if field then
+          local s = rl.choices[i+1]
+          s=s:gsub("\\x(%x%x)",function (x) return string.char(tonumber(x,16)) end)
+          args[fieldname] = s
+          ntaken = 1
+        end
+      end
+    end
+    -- print("ARGS: ", vpp.dump(args))
+    local ret = vpp:api_call(msgname, args)
+    for i, reply in ipairs(ret) do
+      table.insert(out, "=================== Entry #" .. tostring(i))
+      for k, v in pairs(reply) do
+        table.insert(out, "   " .. tostring(k) .. " : " .. toprintablestring(v))
+      end
+    end
+    -- paged_write(vpp.dump(ret) .. "\n\n")
+    paged_write(table.concat(out, "\n").."\n\n")
+  end
+  device.output["call " .. cli .. " help"] = function(rl)
+    local out = {}
+    for k, v in pairs(vpp.msg_name_to_fields[msgname]) do
+      table.insert(out, tostring(k) .. " : " .. v["ctype"] .. " ; " .. tostring(vpp.dump(v)) )
+    end
+    -- paged_write(vpp.dump(vpp.msg_name_to_fields[msgname]) .. "\n\n")
+    paged_write(table.concat(out, "\n").."\n\n")
+  end
+-- vpp.msg_name_to_number = {}
+end
+
+
+
+local r = readln.reader()
+local done = false
+
+r.prompt = "VPP(luaCLI)#"
+
+r.help = device_cli_help
+r.tab_complete = device_cli_tab_complete
+print("===== CLI view, use ^D to end =====")
+
+r.tree = populate_tree(device.output)
+-- readln.pretty("xxxx", r.tree)
+
+
+for idx, an_arg in ipairs(arg) do
+  local fname = an_arg
+  if fname == "-i" then
+    pager_lines = 23
+    interactive_cli(r)
+  else
+    pager_lines = 100000000
+    for line in io.lines(fname) do
+      r.command = line
+      local func = device_cli_exec(r)
+      if func then
+	func(r)
+      end
+    end
+  end
+end
+
+if #arg == 0 then
+  print("You should specify '-i' as an argument for the interactive session,")
+  print("but with no other sources of commands, we start interactive session now anyway")
+   interactive_cli(r)
+end
+
+vpp:disconnect()
+r.done()
+
+
diff --git a/src/vpp-api/lua/examples/example-acl-plugin.lua b/src/vpp-api/lua/examples/example-acl-plugin.lua
new file mode 100644
index 00000000..ca01f18d
--- /dev/null
+++ b/src/vpp-api/lua/examples/example-acl-plugin.lua
@@ -0,0 +1,110 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+
+vpp = require "vpp-lapi"
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+vpp:connect("aytest")
+vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
+
+-- api calls
+reply = vpp:api_call("show_version")
+print("Version: ", reply[1].version)
+print(vpp.hex_dump(reply[1].version))
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 0 })
+print(vpp.dump(reply))
+print("---")
+
+acl_index_to_delete = reply[1].acl_index
+print("Deleting " .. tostring(acl_index_to_delete))
+reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+for ri, rv in ipairs(reply) do 
+  print("Reply message #" .. tostring(ri))
+  print(vpp.dump(rv))
+  for ai, av in ipairs(rv.r) do
+    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
+  end
+   
+end
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+print(vpp.dump(reply))
+print("---")
+
+
+vpp:disconnect()
+
+
diff --git a/src/vpp-api/lua/examples/example-classifier.lua b/src/vpp-api/lua/examples/example-classifier.lua
new file mode 100644
index 00000000..ec9c3d3e
--- /dev/null
+++ b/src/vpp-api/lua/examples/example-classifier.lua
@@ -0,0 +1,51 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+
+local vpp = require "vpp-lapi"
+local bit = require("bit")
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+
+vpp:connect("aytest")
+
+-- api calls
+
+print("Calling API to add a new classifier table")
+reply = vpp:api_call("classify_add_del_table", {
+  context = 43,
+  memory_size = bit.lshift(2, 20),
+  client_index = 42,
+  is_add = 1,
+  nbuckets = 32,
+  skip_n_vectors = 0,
+  match_n_vectors = 1,
+  mask = "\255\255\255\255\255\255\255\255" .. "\255\255\255\255\255\255\255\255"
+})
+print(vpp.dump(reply))
+print("---")
+
+
+vpp:disconnect()
+
+
diff --git a/src/vpp-api/lua/examples/example-cli.lua b/src/vpp-api/lua/examples/example-cli.lua
new file mode 100644
index 00000000..8b84989f
--- /dev/null
+++ b/src/vpp-api/lua/examples/example-cli.lua
@@ -0,0 +1,44 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+vpp = require "vpp-lapi"
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+
+vpp:connect("aytest")
+
+-- api calls
+reply = vpp:api_call("show_version")
+print("Version: ", reply[1].version)
+print(vpp.hex_dump(reply[1].version))
+print(vpp.dump(reply))
+print("---")
+
+
+reply = vpp:api_call("cli_inband", { cmd = "show vers" }) 
+print(vpp.dump(reply))
+print("---")
+
+
+vpp:disconnect()
+
+
diff --git a/src/vpp-api/lua/examples/lute/README.md b/src/vpp-api/lua/examples/lute/README.md
new file mode 100644
index 00000000..8d37250a
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/README.md
@@ -0,0 +1,66 @@
+LUTE: Lua Unit Test Environment
+
+This is a small helper utility to automate some simple tests
+that one might need to do.
+
+Think of it as a hybrid of a screen and expect who
+also took some habits from HTML inline code.
+
+It is quite probably useless for building anything serious,
+but practice shows it is quite efficient at allowing
+convenient temporary quick tests, and for something
+that was written over a course of a couple of evenings it
+is quite a nice little helper tool.
+
+It allows do launch and drive multiple shell sessions,
+and by virtue of having been written in Lua, it of course
+also allows to add the business logic using the Lua code.
+
+If you launch the lute without parameters, it gives you
+the interactive shell to execute the commands in.
+
+If you launch it with an argument, it will attempt to
+read and execute the commands from the file.
+
+Commands:
+
+shell FOO
+
+  spawn a shell in a new PTY under the label FOO.
+
+run FOO bar
+
+  Send "bar" keystrokes followed by "ENTER" to the session FOO
+
+  Special case: "break" word on its own gets translated into ^C being sent.
+
+cd FOO
+
+  "change domain" into session FOO. All subsequent inputs will go,
+  line-buffered, into the session FOO. To jump back up, use ^D (Control-D),
+  or within the file, use ^D^D^D (caret D caret D caret D on its own line)
+
+expect FOO blablabla
+
+  Pause further interpretation of the batch mode until you see "blablabla"
+  in the output of session FOO, or until timeout happens.
+
+sleep N
+
+  Sleep an integer N seconds, if you are in batch mode.
+
+echo blabla
+
+  Echo the remainder of the line to standard output.
+
+For Lua code, there is a pre-existing pseudo-session called "lua",
+which accepts "run lua" command which does what you would expect
+(evaluate the rest of the string in Lua context - being the same
+as lute itself). Also you can do "cd lua" and get into a
+multiline-enabled interpreter shell.
+
+This way for the VPP case you can automate some of the things in your routine
+that you would have to have done manually, and test drive API as well
+as use the realistic native OS components to create the environment around it.
+
+
diff --git a/src/vpp-api/lua/examples/lute/lute.lua b/src/vpp-api/lua/examples/lute/lute.lua
new file mode 100644
index 00000000..89b9924b
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/lute.lua
@@ -0,0 +1,777 @@
+--[[
+version = 1
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+-- LUTE: Lua Unit Test Environment
+-- AKA what happens when screen tries to marry with lua and expect,
+-- but escapes mid-ceremony.
+--
+-- comments: @ayourtch
+
+ffi = require("ffi")
+
+vpp = {}
+function vpp.dump(o)
+   if type(o) == 'table' then
+      local s = '{ '
+      for k,v in pairs(o) do
+         if type(k) ~= 'number' then k = '"'..k..'"' end
+         s = s .. '['..k..'] = ' .. vpp.dump(v) .. ','
+      end
+      return s .. '} '
+   else
+      return tostring(o)
+   end
+end
+
+
+ffi.cdef([[
+
+int posix_openpt(int flags);
+int grantpt(int fd);
+int unlockpt(int fd);
+char *ptsname(int fd);
+
+typedef long pid_t;
+typedef long ssize_t;
+typedef long size_t;
+typedef int nfds_t;
+typedef long time_t;
+typedef long suseconds_t;
+
+pid_t fork(void);
+pid_t setsid(void);
+
+int close(int fd);
+int open(char *pathname, int flags);
+
+int dup2(int oldfd, int newfd);
+
+ssize_t read(int fd, void *buf, size_t count);
+ssize_t write(int fd, const void *buf, size_t count);
+
+struct pollfd {
+               int   fd;         /* file descriptor */
+               short events;     /* requested events */
+               short revents;    /* returned events */
+           };
+
+int poll(struct pollfd *fds, nfds_t nfds, int timeout);
+
+struct timeval {
+               time_t      tv_sec;     /* seconds */
+               suseconds_t tv_usec;    /* microseconds */
+           };
+
+int gettimeofday(struct timeval *tv, struct timezone *tz);
+
+int inet_pton(int af, const char *src, void *dst);
+
+]])
+
+ffi.cdef([[
+void *memset(void *s, int c, size_t n);
+void *memcpy(void *dest, void *src, size_t n);
+void *memmove(void *dest, const void *src, size_t n);
+void *memmem(const void *haystack, size_t haystacklen,
+        const void *needle, size_t needlelen);
+]])
+
+
+
+local O_RDWR = 2
+
+
+function os_time()
+  local tv = ffi.new("struct timeval[1]")
+  local ret = ffi.C.gettimeofday(tv, nil)
+  return tonumber(tv[0].tv_sec) + (tonumber(tv[0].tv_usec)/1000000.0)
+end
+
+function sleep(n)
+  local when_wakeup = os_time() + n
+  while os_time() <= when_wakeup do
+    ffi.C.poll(nil, 0, 10)
+  end
+end
+
+
+function c_str(text_in)
+  local text = text_in 
+  local c_str = ffi.new("char[?]", #text+1)
+  ffi.copy(c_str, text)
+  return c_str
+end
+
+function ip46(addr_text)
+  local out = ffi.new("char [200]")
+  local AF_INET6 = 10
+  local AF_INET = 2
+  local is_ip6 = ffi.C.inet_pton(AF_INET6, c_str(addr_text), out)
+  if is_ip6 == 1 then
+    return ffi.string(out, 16), true
+  end
+  local is_ip4 = ffi.C.inet_pton(AF_INET, c_str(addr_text), out)
+  if is_ip4 then
+    return (string.rep("4", 12).. ffi.string(out, 4)), false
+  end
+end
+
+function pty_master_open()
+  local fd = ffi.C.posix_openpt(O_RDWR)
+  ffi.C.grantpt(fd)
+  ffi.C.unlockpt(fd)
+  local p = ffi.C.ptsname(fd)
+  print("PTS:" .. ffi.string(p))
+  return fd, ffi.string(p)
+end
+
+function pty_run(cmd)
+  local master_fd, pts_name = pty_master_open()
+  local child_pid = ffi.C.fork()
+  if (child_pid == -1) then
+    print("Error fork()ing")
+    return -1
+  end 
+ 
+  if child_pid ~= 0 then
+    -- print("Parent")
+    return master_fd, child_pid
+  end
+
+  -- print("Child")
+  if (ffi.C.setsid() == -1) then
+    print("Child error setsid")
+    os.exit(-1)
+  end
+
+  ffi.C.close(master_fd)
+
+  local slave_fd = ffi.C.open(c_str(pts_name), O_RDWR)
+  if slave_fd == -1 then
+    print("Child can not open slave fd")
+    os.exit(-2)
+  end
+
+  ffi.C.dup2(slave_fd, 0)
+  ffi.C.dup2(slave_fd, 1)
+  ffi.C.dup2(slave_fd, 2)
+  os.execute(cmd)
+end
+
+function readch()
+  local buf = ffi.new("char[1]")
+  local nread= ffi.C.read(0, buf, 1)
+  -- print("\nREADCH : " .. string.char(buf[0]))
+  return string.char(buf[0])
+end
+
+function stdout_write(str)
+  ffi.C.write(1, c_str(str), #str)
+end
+
+
+readln = {
+split = function(str, pat)
+  local t = {}  -- NOTE: use {n = 0} in Lua-5.0
+  local fpat = "(.-)" .. pat
+  local last_end = 1
+  if str then
+    local s, e, cap = str:find(fpat, 1)
+    while s do
+      if s ~= 1 or cap ~= "" then
+        table.insert(t,cap)
+      end
+      last_end = e+1
+      s, e, cap = str:find(fpat, last_end)
+    end
+    if last_end <= #str then
+      cap = str:sub(last_end)
+      table.insert(t, cap)
+    end
+  end
+  return t
+end,
+
+reader = function()
+  local rl = {}
+
+  rl.init = function()
+    os.execute("stty -icanon min 1 -echo")
+    rl.rawmode = true
+  end
+
+  rl.done = function()
+    os.execute("stty icanon echo")
+    rl.rawmode = false
+  end
+
+  rl.prompt = ">"
+  rl.history = { "" }
+  rl.history_index = 1
+  rl.history_length = 1
+
+  rl.hide_cmd = function()
+    local bs = string.char(8) .. " " .. string.char(8)
+    for i = 1, #rl.command do
+      stdout_write(bs)
+    end
+  end
+
+  rl.show_cmd = function()
+    if rl.command then
+      stdout_write(rl.command)
+    end
+  end
+
+  rl.store_history = function(cmd)
+    if cmd == "" then
+      return
+    end
+    rl.history[rl.history_length] = cmd
+    rl.history_length = rl.history_length + 1
+    rl.history_index = rl.history_length
+    rl.history[rl.history_length] = ""
+  end
+
+  rl.readln = function(stdin_select_fn, batch_cmd, batch_when, batch_expect)
+    local done = false
+    local need_prompt = true
+    rl.command = ""
+
+    if not rl.rawmode then
+      rl.init()
+    end
+
+    while not done do
+      local indent_value = #rl.prompt + #rl.command
+      if need_prompt then
+        stdout_write(rl.prompt)
+        stdout_write(rl.command)
+        need_prompt = false
+      end
+      if type(stdin_select_fn) == "function" then
+        while not stdin_select_fn(indent_value, batch_cmd, batch_when, batch_expect) do
+          stdout_write(rl.prompt)
+          stdout_write(rl.command)
+          indent_value = #rl.prompt + #rl.command
+        end
+        if batch_cmd and ((os_time() > batch_when) or (batch_expect and expect_success(batch_expect, buf, 0))) then
+          stdout_write("\n" .. rl.prompt .. batch_cmd .. "\n")
+          if batch_expect then
+            expect_done(batch_expect)
+          end
+          return batch_cmd, batch_expect
+        end
+      end
+      local ch = readch()
+      if ch:byte(1) == 27 then
+        -- CONTROL
+        local ch2 = readch()
+        -- arrows
+        if ch2:byte(1) == 91 then
+          local ch3 = readch()
+          local b = ch3:byte(1)
+          if b == 65 then
+            ch = "UP"
+          elseif b == 66 then
+            ch = "DOWN"
+          elseif b == 67 then
+            ch = "RIGHT"
+          elseif b == 68 then
+            ch = "LEFT"
+          end
+          -- print("Byte: " .. ch3:byte(1))
+          -- if ch3:byte(1)
+        end
+      end
+
+      if ch == "?" then
+        stdout_write(ch)
+        stdout_write("\n")
+        if rl.help then
+          rl.help(rl)
+        end
+        need_prompt = true
+      elseif ch == "\t" then
+        if rl.tab_complete then
+          rl.tab_complete(rl)
+        end
+        stdout_write("\n")
+        need_prompt = true
+      elseif ch == "\n" then
+        stdout_write(ch)
+        done = true
+      elseif ch == "\004" then
+        stdout_write("\n")
+        rl.command = nil
+        done = true
+      elseif ch == string.char(127) then
+        if rl.command ~= "" then
+          stdout_write(string.char(8) .. " " .. string.char(8))
+          rl.command = string.sub(rl.command, 1, -2)
+        end
+      elseif #ch > 1 then
+        -- control char
+        if ch == "UP" then
+          rl.hide_cmd()
+          if rl.history_index == #rl.history then
+            rl.history[rl.history_index] = rl.command
+          end
+          if rl.history_index > 1 then
+            rl.history_index = rl.history_index - 1
+            rl.command = rl.history[rl.history_index]
+          end
+          rl.show_cmd()
+        elseif ch == "DOWN" then
+          rl.hide_cmd()
+          if rl.history_index < rl.history_length then
+            rl.history_index = rl.history_index + 1
+            rl.command = rl.history[rl.history_index]
+          end
+          rl.show_cmd()
+        end
+      else
+        stdout_write(ch)
+        rl.command = rl.command .. ch
+      end
+    end
+    if rl.command then
+      rl.store_history(rl.command)
+    end
+    return rl.command
+  end
+  return rl
+end
+
+}
+
+local select_fds = {}
+local sessions = {}
+
+local line_erased = false
+
+function erase_line(indent)
+  if not line_erased then
+    line_erased = true
+    stdout_write(string.rep(string.char(8), indent)..string.rep(" ", indent)..string.rep(string.char(8), indent))
+  end
+end
+
+function do_select_stdin(indent, batch_cmd, batch_when, batch_expect)
+  while true do
+    local nfds = 1+#select_fds
+    local pfds = ffi.new("struct pollfd[?]", nfds)
+    pfds[0].fd = 0;
+    pfds[0].events = 1;
+    pfds[0].revents = 0;
+    for i = 1,#select_fds do
+      pfds[i].fd = select_fds[i].fd
+      pfds[i].events = 1
+      pfds[i].revents = 0
+    end
+    if batch_cmd and ((os_time() > batch_when) or (batch_expect and expect_success(batch_expect, buf, 0))) then
+      return true
+    end
+    while ffi.C.poll(pfds, nfds, 10) == 0 do
+      if batch_cmd and ((os_time() > batch_when) or (batch_expect and expect_success(batch_expect, buf, 0))) then
+        return true
+      end
+      if line_erased then
+        line_erased = false
+        return false
+      end
+    end
+    if pfds[0].revents == 1 then
+      return true
+    end
+    for i = 1,#select_fds do
+      if(pfds[i].revents > 0) then
+        if pfds[i].fd ~= select_fds[i].fd then
+          print("File descriptors unequal", pfds[i].fd, select_fds[i].fd)
+        end
+        select_fds[i].cb(select_fds[i], pfds[i].revents, indent)
+      end
+    end
+  end
+end
+
+local buf = ffi.new("char [32768]")
+
+function session_stdout_write(prefix, data)
+  data = prefix .. data:gsub("\n", "\n"..prefix):gsub("\n"..prefix.."$", "\n")
+  
+  stdout_write(data)
+end
+
+function expect_success(sok, buf, nread)
+  local expect_buf_sz = ffi.sizeof(sok.expect_buf) - 128
+  local expect_buf_avail = expect_buf_sz - sok.expect_buf_idx
+  -- print("EXPECT_SUCCESS: nread ".. tostring(nread).. " expect_buf_idx: " .. tostring(sok.expect_buf_idx) .. " expect_buf_avail: " .. tostring(expect_buf_avail) )
+  if expect_buf_avail < 0 then
+    print "EXPECT BUFFER OVERRUN ALREADY"
+    os.exit(1)
+  end
+  if expect_buf_avail < nread then
+    if (nread >= ffi.sizeof(sok.expect_buf)) then
+      print("Read too large of a chunk to fit into expect buffer")
+      return nil
+    end
+    local delta = nread - expect_buf_avail
+
+    ffi.C.memmove(sok.expect_buf, sok.expect_buf + delta, expect_buf_sz - delta)
+    sok.expect_buf_idx = sok.expect_buf_idx - delta
+    expect_buf_avail = nread 
+  end
+  if sok.expect_buf_idx + nread > expect_buf_sz then
+    print("ERROR, I have just overrun the buffer !")
+    os.exit(1)
+  end
+  ffi.C.memcpy(sok.expect_buf + sok.expect_buf_idx, buf, nread)
+  sok.expect_buf_idx = sok.expect_buf_idx + nread
+  if sok.expect_str == nil then
+    return true
+  end
+  local match_p = ffi.C.memmem(sok.expect_buf, sok.expect_buf_idx, sok.expect_str, sok.expect_str_len)
+  if match_p ~= nil then
+    return true
+  end
+  return false
+end
+
+function expect_done(sok)
+  local expect_buf_sz = ffi.sizeof(sok.expect_buf) - 128
+  if not sok.expect_str then 
+    return false
+  end
+  local match_p = ffi.C.memmem(sok.expect_buf, sok.expect_buf_idx, sok.expect_str, sok.expect_str_len)
+  if match_p ~= nil then
+    if sok.expect_cb then
+      sok.expect_cb(sok)
+    end
+    local match_idx = ffi.cast("char *", match_p) - ffi.cast("char *", sok.expect_buf)
+    ffi.C.memmove(sok.expect_buf, ffi.cast("char *", match_p) + sok.expect_str_len, expect_buf_sz - match_idx - sok.expect_str_len)
+    sok.expect_buf_idx = match_idx + sok.expect_str_len
+    sok.expect_success = true
+
+    sok.expect_str = nil
+    sok.expect_str_len = 0
+    return true
+  end
+end
+
+function slave_events(sok, revents, indent)
+  local fd = sok.fd
+  local nread = ffi.C.read(fd, buf, ffi.sizeof(buf)-128)
+  local idx = nread - 1
+  while idx >= 0 and buf[idx] ~= 10 do
+    idx = idx - 1
+  end
+  if idx >= 0 then
+    erase_line(indent)
+    session_stdout_write(sok.prefix, sok.buf .. ffi.string(buf, idx+1))
+    sok.buf = ""
+  end
+  sok.buf = sok.buf .. ffi.string(buf+idx+1, nread-idx-1)
+  -- print("\nRead: " .. tostring(nread))
+  -- stdout_write(ffi.string(buf, nread))
+  if expect_success(sok, buf, nread) then
+    return true
+  end
+  return false
+end
+
+
+function start_session(name)
+  local mfd, cpid = pty_run("/bin/bash")
+  local sok =  { ["fd"] = mfd, ["cb"] = slave_events, ["buf"] = "", ["prefix"] = name .. ":", ["expect_buf"] = ffi.new("char [165536]"), ["expect_buf_idx"] = 0, ["expect_str"] = nil }
+  table.insert(select_fds, sok)
+  sessions[name] = sok
+end
+
+function command_transform(exe)
+  if exe == "break" then
+    exe = string.char(3)
+  end
+  return exe
+end
+
+function session_write(a_session, a_str)
+  if has_session(a_session) then
+    return tonumber(ffi.C.write(sessions[a_session].fd, c_str(a_str), #a_str))
+  else
+    return 0
+  end
+end
+
+function session_exec(a_session, a_cmd)
+  local exe = command_transform(a_cmd) .. "\n"
+  session_write(a_session, exe)
+end
+
+function session_cmd(ui, a_session, a_cmd)
+  if not has_session(a_session) then
+    stdout_write("ERR: No such session '" .. tostring(a_session) .. "'\n")
+    return nil
+  end
+  if a_session == "lua" then
+    local func, msg = loadstring(ui.lua_acc .. a_cmd)
+    -- stdout_write("LOADSTR: " .. vpp.dump({ ret, msg }) .. "\n")
+    if not func and string.match(msg, "") then
+      if a_session ~= ui.in_session then
+         stdout_write("ERR LOADSTR: " .. tostring(msg) .. "\n")
+         return nil
+      end
+      ui.lua_acc = ui.lua_acc .. a_cmd  .. "\n"
+      return true
+    end
+    ui.lua_acc = ""
+    local ret, msg = pcall(func)
+    if ret then
+      return true
+    else
+      stdout_write("ERR: " .. msg .. "\n") 
+      return nil
+    end
+  else
+    session_exec(a_session, a_cmd)
+    if ui.session_cmd_delay then
+      return { "delay", ui.session_cmd_delay }
+    end
+    return true
+  end
+end
+
+function has_session(a_session)
+  if a_session == "lua" then
+    return true
+  end
+  return (sessions[a_session] ~= nil)
+end
+
+function command_match(list, input, output)
+  for i, v in ipairs(list) do
+    local m = {}
+    m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8], m[9] = string.match(input, v[1])
+    -- print("MATCH: ", vpp.dump(m))
+    if m[1] then
+       output["result"] = m
+       output["result_index"] = i
+       return m
+    end 
+  end
+  return nil
+end
+
+function cmd_spawn_shell(ui, a_arg)
+  start_session(a_arg[1])
+  return true
+end
+
+function cmd_run_cmd(ui, a_arg)
+  local a_sess = a_arg[1]
+  local a_cmd = a_arg[2]
+  return session_cmd(ui, a_sess, a_cmd)
+end
+
+function cmd_cd(ui, a_arg)
+  local a_sess = a_arg[1]
+  if has_session(a_sess) then
+    ui.in_session = a_sess
+    return true
+  else
+    stdout_write("ERR: Unknown session '".. tostring(a_sess) .. "'\n")
+    return nil
+  end
+end
+
+function cmd_sleep(ui, a_arg)
+  return { "delay", tonumber(a_arg[1]) }
+end
+
+function cmd_expect(ui, a_arg)
+  local a_sess = a_arg[1]
+  local a_expect = a_arg[2]
+  local sok = sessions[a_sess]
+  if not sok then
+    stdout_write("ERR: unknown session '" .. tostring(a_sess) .. "'\n")
+    return nil
+  end
+  sok.expect_str = c_str(a_expect)
+  sok.expect_str_len = #a_expect
+  return { "expect", a_sess }
+end
+
+function cmd_info(ui, a_arg)
+  local a_sess = a_arg[1]
+  local sok = sessions[a_sess]
+  if not sok then
+    stdout_write("ERR: unknown session '" .. tostring(a_sess) .. "'\n")
+    return nil
+  end
+  print("Info for session " .. tostring(a_sess) .. "\n")
+  print("Expect buffer index: " .. tostring(sok.expect_buf_idx))
+  print("Expect buffer: '" .. tostring(ffi.string(sok.expect_buf, sok.expect_buf_idx)) .. "'\n")
+  if sok.expect_str then
+    print("Expect string: '" .. tostring(ffi.string(sok.expect_str, sok.expect_str_len)) .. "'\n")
+  else
+    print("Expect string not set\n")
+  end
+end
+
+function cmd_echo(ui, a_arg)
+  local a_data = a_arg[1]
+  print("ECHO: " .. tostring(a_data))
+end
+
+main_command_table = {
+  { "^shell ([a-zA-Z0-9_]+)$", cmd_spawn_shell },
+  { "^run ([a-zA-Z0-9_]+) (.+)$", cmd_run_cmd },
+  { "^cd ([a-zA-Z0-9_]+)$", cmd_cd  },
+  { "^sleep ([0-9]+)$", cmd_sleep },
+  { "^expect ([a-zA-Z0-9_]+) (.-)$", cmd_expect },
+  { "^info ([a-zA-Z0-9_]+)$", cmd_info },
+  { "^echo (.-)$", cmd_echo }
+}
+
+
+
+function ui_set_prompt(ui)
+  if ui.in_session then 
+    if ui.in_session == "lua" then
+      if #ui.lua_acc > 0 then
+        ui.r.prompt = ui.in_session .. ">>"
+      else
+        ui.r.prompt = ui.in_session .. ">"
+      end 
+    else
+      ui.r.prompt = ui.in_session .. "> "
+    end
+  else
+    ui.r.prompt = "> "
+  end
+  return ui.r.prompt
+end
+
+function ui_run_command(ui, cmd)
+  -- stdout_write("Command: " .. tostring(cmd) .. "\n")
+  local ret = false
+  if ui.in_session then
+    if cmd then
+      if cmd == "^D^D^D" then
+        ui.in_session = nil
+        ret = true
+      else
+        ret = session_cmd(ui, ui.in_session, cmd)
+      end
+    else
+      ui.in_session = nil
+      ret = true
+    end
+  else  
+    if cmd then
+      local out = {}
+      if cmd == "" then
+        ret = true
+      end
+      if command_match(main_command_table, cmd, out) then
+        local i = out.result_index
+        local m = out.result
+        if main_command_table[i][2] then
+          ret = main_command_table[i][2](ui, m)
+        end
+      end
+    end
+    if not cmd or cmd == "quit" then
+      return "quit"
+    end
+  end
+  return ret
+end
+
+local ui = {}
+ui.in_session = nil
+ui.r = readln.reader() 
+ui.lua_acc = ""
+ui.session_cmd_delay = 0.3
+
+local lines = ""
+
+local done = false
+-- a helper function which always returns nil
+local no_next_line = function() return nil end
+
+-- a function which returns the next batch line
+local next_line = no_next_line
+
+local batchfile = arg[1]
+
+if batchfile then
+  local f = io.lines(batchfile)
+  next_line = function() 
+    local line = f()
+    if line then 
+      return line
+    else 
+      next_line = no_next_line
+      session_stdout_write(batchfile .. ":", "End of batch\n")
+      return nil
+    end
+  end
+end
+
+
+local batch_when = 0
+local batch_expect = nil
+while not done do
+  local prompt = ui_set_prompt(ui)
+  local batch_cmd = next_line() 
+  local cmd, expect_sok = ui.r.readln(do_select_stdin, batch_cmd, batch_when, batch_expect)
+  if expect_sok and not expect_success(expect_sok, buf, 0) then
+    if not cmd_ret and next_line ~= no_next_line then
+      print("ERR: expect timeout\n")
+      next_line = no_next_line
+    end
+  else 
+    local cmd_ret = ui_run_command(ui, cmd)
+    if not cmd_ret and next_line ~= no_next_line then
+      print("ERR: Error during batch execution\n")
+      next_line = no_next_line
+    end
+
+    if cmd_ret  == "quit" then
+      done = true
+    end
+    batch_expect = nil
+    batch_when = 0
+    if type(cmd_ret) == "table" then
+      if cmd_ret[1] == "delay" then
+	batch_when = os_time() + tonumber(cmd_ret[2])
+      end
+      if cmd_ret[1] == "expect" then
+	batch_expect = sessions[cmd_ret[2]]
+	batch_when = os_time() + 15
+      end
+    end
+  end
+end
+ui.r.done()
+
+os.exit(1)
+
+
+
diff --git a/src/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute b/src/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute
new file mode 100644
index 00000000..a24d04bf
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute
@@ -0,0 +1,329 @@
+shell vppbuild
+run vppbuild stty -echo
+run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
+expect vppbuild ALLGOOD
+
+shell s0
+shell s1
+shell s2
+
+
+cd s1
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+cd s2
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+
+cd lua
+
+function session_get_bash_pid(s)
+  if not has_session(s) then
+    return nil
+  end
+  local fname = "/tmp/lute-"..s.."-pid.txt"
+
+  session_exec(s, "echo $$ >" .. fname)
+  -- it's a dirty hack but it's quick
+  sleep(0.5)
+  local pid = io.lines(fname)()
+  print("Got pid for " .. s .. " : " .. tostring(pid))
+  return(tonumber(pid))
+end
+
+function session_connect_with(s0, s1)
+  -- local pid0 = tostring(session_get_bash_pid(s0))
+  local pid1 = tostring(session_get_bash_pid(s1))
+  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
+  local this_end = s0 .. "_" .. s1
+  local other_end = s1 .. "_" .. s0
+  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
+  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
+  for i, option in ipairs(eth_options) do
+    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
+    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
+  end
+  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
+  sleep(0.5)
+end
+
+^D^D^D
+run lua session_connect_with("s0", "s1")
+run lua session_connect_with("s0", "s2")
+
+cd s1
+ip -6 addr add dev s1_s0 2001:db8:1::1/64
+ip -4 addr add dev s1_s0 192.0.2.1/24
+ip link set dev s1_s0 up promisc on
+^D^D^D
+
+cd s2
+ip -6 addr add dev s2_s0 2001:db8:1::2/64
+ip -6 addr add dev s2_s0 2001:db8:1::3/64
+ip -6 addr add dev s2_s0 2001:db8:1::4/64
+ip -4 addr add dev s2_s0 192.0.2.2/24
+ip -4 addr add dev s2_s0:1 192.0.2.3/24
+ip -4 addr add dev s2_s0:2 192.0.2.4/24
+ip link set dev s2_s0 up promisc on
+^D^D^D
+
+run s1 ip addr
+run s2 ip addr
+shell VPP
+cd VPP
+cd /home/ubuntu/vpp
+make debug 
+r
+^D^D^D
+expect VPP DBGvpp#
+
+cd lua
+-- Initialization of the Lua environment for talking to VPP
+vpp = require("vpp-lapi")
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+vpp:init({ pneum_path = pneum_path })
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+vpp:connect("aytest")
+vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
+
+^D^D^D
+
+cd lua
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
+vpp_if_to_s1 = reply[1].sw_if_index
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
+vpp_if_to_s2 = reply[1].sw_if_index
+
+ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
+
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+
+bd_id = 42
+
+reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
+print(vpp.dump(reply))
+
+for i, v in ipairs(ifaces) do
+  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
+  print(vpp.dump(reply))
+end
+
+^D^D^D
+
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.4
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+
+
+cd lua
+--- ACL testing
+
+--[[ temporary comment out
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add_replace", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add_replace", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add_replace", { context = 42, count = 0 })
+print(vpp.dump(reply))
+print("---")
+
+acl_index_to_delete = reply[1].acl_index
+print("Deleting " .. tostring(acl_index_to_delete))
+reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+for ri, rv in ipairs(reply) do
+  print("Reply message #" .. tostring(ri))
+  print(vpp.dump(rv))
+  for ai, av in ipairs(rv.r) do
+    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
+  end
+
+end
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
+print(vpp.dump(reply))
+print("---")
+
+
+]] -- end of comment out
+
+---- Should be nothing ^^
+r = {
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
+  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
+  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
+}
+
+reply = vpp:api_call("acl_add_replace", { context = 42, count = 5, r = r })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add_replace", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_in
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+--reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+-- print(vpp.dump(reply))
+--print("---")
+
+^D^D^D
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 0
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run VPP show trace 
+expect VPP match: inacl 0 rule 1
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 2
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 3
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 4
+
+
+cd lua
+
+--- TEST OUTBOUND ACL
+
+r1 = {
+  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 }
+}
+
+reply = vpp:api_call("acl_add_replace", { context = 42, count = 3, r = r1 })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+
+^D^D^D
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: outacl 2 rule 0
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run VPP show trace 
+expect VPP match: inacl 0 rule 1
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+run VPP show trace
+expect VPP match: outacl 2 rule 1
+
+run lua print("ALL GOOD!")
+
diff --git a/src/vpp-api/lua/examples/lute/script-inout-acl-old.lute b/src/vpp-api/lua/examples/lute/script-inout-acl-old.lute
new file mode 100644
index 00000000..9edebf02
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/script-inout-acl-old.lute
@@ -0,0 +1,329 @@
+shell vppbuild
+run vppbuild stty -echo
+run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
+expect vppbuild ALLGOOD
+
+shell s0
+shell s1
+shell s2
+
+
+cd s1
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+cd s2
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+
+cd lua
+
+function session_get_bash_pid(s)
+  if not has_session(s) then
+    return nil
+  end
+  local fname = "/tmp/lute-"..s.."-pid.txt"
+
+  session_exec(s, "echo $$ >" .. fname)
+  -- it's a dirty hack but it's quick
+  sleep(0.5)
+  local pid = io.lines(fname)()
+  print("Got pid for " .. s .. " : " .. tostring(pid))
+  return(tonumber(pid))
+end
+
+function session_connect_with(s0, s1)
+  -- local pid0 = tostring(session_get_bash_pid(s0))
+  local pid1 = tostring(session_get_bash_pid(s1))
+  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
+  local this_end = s0 .. "_" .. s1
+  local other_end = s1 .. "_" .. s0
+  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
+  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
+  for i, option in ipairs(eth_options) do
+    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
+    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
+  end
+  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
+  sleep(0.5)
+end
+
+^D^D^D
+run lua session_connect_with("s0", "s1")
+run lua session_connect_with("s0", "s2")
+
+cd s1
+ip -6 addr add dev s1_s0 2001:db8:1::1/64
+ip -4 addr add dev s1_s0 192.0.2.1/24
+ip link set dev s1_s0 up promisc on
+^D^D^D
+
+cd s2
+ip -6 addr add dev s2_s0 2001:db8:1::2/64
+ip -6 addr add dev s2_s0 2001:db8:1::3/64
+ip -6 addr add dev s2_s0 2001:db8:1::4/64
+ip -4 addr add dev s2_s0 192.0.2.2/24
+ip -4 addr add dev s2_s0:1 192.0.2.3/24
+ip -4 addr add dev s2_s0:2 192.0.2.4/24
+ip link set dev s2_s0 up promisc on
+^D^D^D
+
+run s1 ip addr
+run s2 ip addr
+shell VPP
+cd VPP
+cd /home/ubuntu/vpp
+make debug 
+r
+^D^D^D
+expect VPP DBGvpp#
+
+cd lua
+-- Initialization of the Lua environment for talking to VPP
+vpp = require("vpp-lapi")
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+vpp:init({ pneum_path = pneum_path })
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+vpp:connect("aytest")
+vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
+
+^D^D^D
+
+cd lua
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
+vpp_if_to_s1 = reply[1].sw_if_index
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
+vpp_if_to_s2 = reply[1].sw_if_index
+
+ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
+
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+
+bd_id = 42
+
+reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
+print(vpp.dump(reply))
+
+for i, v in ipairs(ifaces) do
+  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
+  print(vpp.dump(reply))
+end
+
+^D^D^D
+
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.4
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+
+
+cd lua
+--- ACL testing
+
+--[[ temporary comment out
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 0 })
+print(vpp.dump(reply))
+print("---")
+
+acl_index_to_delete = reply[1].acl_index
+print("Deleting " .. tostring(acl_index_to_delete))
+reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+for ri, rv in ipairs(reply) do
+  print("Reply message #" .. tostring(ri))
+  print(vpp.dump(rv))
+  for ai, av in ipairs(rv.r) do
+    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
+  end
+
+end
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
+print(vpp.dump(reply))
+print("---")
+
+
+]] -- end of comment out
+
+---- Should be nothing ^^
+r = {
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
+  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
+  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
+}
+
+reply = vpp:api_call("acl_add", { context = 42, count = 5, r = r })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_in
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+--reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+-- print(vpp.dump(reply))
+--print("---")
+
+^D^D^D
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 0
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run VPP show trace 
+expect VPP match: inacl 0 rule 1
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 2
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 3
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 4
+
+
+cd lua
+
+--- TEST OUTBOUND ACL
+
+r1 = {
+  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 }
+}
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = r1 })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+
+^D^D^D
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: outacl 2 rule 0
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run VPP show trace 
+expect VPP match: inacl 0 rule 1
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+run VPP show trace
+expect VPP match: outacl 2 rule 1
+
+run lua print("ALL GOOD!")
+
diff --git a/src/vpp-api/lua/examples/lute/script-inout-acl.lute b/src/vpp-api/lua/examples/lute/script-inout-acl.lute
new file mode 100644
index 00000000..d7e7423c
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/script-inout-acl.lute
@@ -0,0 +1,329 @@
+shell vppbuild
+run vppbuild stty -echo
+run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
+expect vppbuild ALLGOOD
+
+shell s0
+shell s1
+shell s2
+
+
+cd s1
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+cd s2
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+
+cd lua
+
+function session_get_bash_pid(s)
+  if not has_session(s) then
+    return nil
+  end
+  local fname = "/tmp/lute-"..s.."-pid.txt"
+
+  session_exec(s, "echo $$ >" .. fname)
+  -- it's a dirty hack but it's quick
+  sleep(0.5)
+  local pid = io.lines(fname)()
+  print("Got pid for " .. s .. " : " .. tostring(pid))
+  return(tonumber(pid))
+end
+
+function session_connect_with(s0, s1)
+  -- local pid0 = tostring(session_get_bash_pid(s0))
+  local pid1 = tostring(session_get_bash_pid(s1))
+  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
+  local this_end = s0 .. "_" .. s1
+  local other_end = s1 .. "_" .. s0
+  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
+  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
+  for i, option in ipairs(eth_options) do
+    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
+    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
+  end
+  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
+  sleep(0.5)
+end
+
+^D^D^D
+run lua session_connect_with("s0", "s1")
+run lua session_connect_with("s0", "s2")
+
+cd s1
+ip -6 addr add dev s1_s0 2001:db8:1::1/64
+ip -4 addr add dev s1_s0 192.0.2.1/24
+ip link set dev s1_s0 up promisc on
+^D^D^D
+
+cd s2
+ip -6 addr add dev s2_s0 2001:db8:1::2/64
+ip -6 addr add dev s2_s0 2001:db8:1::3/64
+ip -6 addr add dev s2_s0 2001:db8:1::4/64
+ip -4 addr add dev s2_s0 192.0.2.2/24
+ip -4 addr add dev s2_s0:1 192.0.2.3/24
+ip -4 addr add dev s2_s0:2 192.0.2.4/24
+ip link set dev s2_s0 up promisc on
+^D^D^D
+
+run s1 ip addr
+run s2 ip addr
+shell VPP
+cd VPP
+cd /home/ubuntu/vpp
+make debug 
+r
+^D^D^D
+expect VPP DBGvpp#
+
+cd lua
+-- Initialization of the Lua environment for talking to VPP
+vpp = require("vpp-lapi")
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+vpp:init({ pneum_path = pneum_path })
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+vpp:connect("aytest")
+vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
+
+^D^D^D
+
+cd lua
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
+vpp_if_to_s1 = reply[1].sw_if_index
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
+vpp_if_to_s2 = reply[1].sw_if_index
+
+ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
+
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+
+bd_id = 42
+
+reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
+print(vpp.dump(reply))
+
+for i, v in ipairs(ifaces) do
+  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
+  print(vpp.dump(reply))
+end
+
+^D^D^D
+
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.4
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+
+
+cd lua
+--- ACL testing
+
+--[[ temporary comment out
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 0 })
+print(vpp.dump(reply))
+print("---")
+
+acl_index_to_delete = reply[1].acl_index
+print("Deleting " .. tostring(acl_index_to_delete))
+reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+for ri, rv in ipairs(reply) do
+  print("Reply message #" .. tostring(ri))
+  print(vpp.dump(rv))
+  for ai, av in ipairs(rv.r) do
+    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
+  end
+
+end
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
+print(vpp.dump(reply))
+print("---")
+
+
+]] -- end of comment out
+
+---- Should be nothing ^^
+r = {
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
+  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
+  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
+}
+
+reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 5, r = r })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_in
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+--reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+-- print(vpp.dump(reply))
+--print("---")
+
+^D^D^D
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 0
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run VPP show trace 
+expect VPP match: inacl 0 rule 1
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 2
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 3
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run VPP show trace
+expect VPP match: inacl 0 rule 4
+
+
+cd lua
+
+--- TEST OUTBOUND ACL
+
+r1 = {
+  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 }
+}
+
+reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 3, r = r1 })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+
+^D^D^D
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run VPP show trace
+expect VPP match: outacl 2 rule 0
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run VPP show trace 
+expect VPP match: inacl 0 rule 1
+
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+run VPP show trace
+expect VPP match: outacl 2 rule 1
+
+run lua print("ALL GOOD!")
+
diff --git a/src/vpp-api/lua/examples/lute/script.lute b/src/vpp-api/lua/examples/lute/script.lute
new file mode 100644
index 00000000..c3dd90f2
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/script.lute
@@ -0,0 +1,7 @@
+shell s1
+expect s1 $
+run s1 echo testing123
+expect s1 $
+run s1 echo done
+quit
+
diff --git a/src/vpp-api/lua/examples/lute/sessions-acl.lute b/src/vpp-api/lua/examples/lute/sessions-acl.lute
new file mode 100644
index 00000000..ac237ef9
--- /dev/null
+++ b/src/vpp-api/lua/examples/lute/sessions-acl.lute
@@ -0,0 +1,308 @@
+run lua -- collectgarbage("stop")
+
+shell vppbuild
+run vppbuild stty -echo
+run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
+expect vppbuild ALLGOOD
+
+shell s0
+shell s1
+shell s2
+
+
+cd s1
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+cd s2
+unshare -n /bin/bash
+/sbin/ifconfig -a
+^D^D^D
+
+
+cd lua
+
+function session_get_bash_pid(s)
+  if not has_session(s) then
+    return nil
+  end
+  local fname = "/tmp/lute-"..s.."-pid.txt"
+
+  session_exec(s, "echo $$ >" .. fname)
+  -- it's a dirty hack but it's quick
+  sleep(0.5)
+  local pid = io.lines(fname)()
+  print("Got pid for " .. s .. " : " .. tostring(pid))
+  return(tonumber(pid))
+end
+
+function session_connect_with(s0, s1)
+  -- local pid0 = tostring(session_get_bash_pid(s0))
+  local pid1 = tostring(session_get_bash_pid(s1))
+  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
+  local this_end = s0 .. "_" .. s1
+  local other_end = s1 .. "_" .. s0
+  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
+  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
+  for i, option in ipairs(eth_options) do
+    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
+    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
+  end
+  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
+  sleep(0.5)
+end
+
+^D^D^D
+run lua session_connect_with("s0", "s1")
+run lua session_connect_with("s0", "s2")
+
+cd s1
+ip -6 addr add dev s1_s0 2001:db8:1::1/64
+ip -4 addr add dev s1_s0 192.0.2.1/24
+ip link set dev s1_s0 up promisc on
+^D^D^D
+
+cd s2
+ip -6 addr add dev s2_s0 2001:db8:1::2/64
+ip -6 addr add dev s2_s0 2001:db8:1::3/64
+ip -6 addr add dev s2_s0 2001:db8:1::4/64
+ip -4 addr add dev s2_s0 192.0.2.2/24
+ip -4 addr add dev s2_s0:1 192.0.2.3/24
+ip -4 addr add dev s2_s0:2 192.0.2.4/24
+ip link set dev s2_s0 up promisc on
+^D^D^D
+
+run s1 ip addr
+run s2 ip addr
+shell VPP
+cd VPP
+cd /home/ubuntu/vpp
+make debug 
+r
+^D^D^D
+expect VPP DBGvpp#
+
+cd lua
+-- Initialization of the Lua environment for talking to VPP
+vpp = require("vpp-lapi")
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+vpp:init({ pneum_path = pneum_path })
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+vpp:connect("aytest")
+vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
+
+^D^D^D
+
+cd lua
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
+vpp_if_to_s1 = reply[1].sw_if_index
+
+reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
+vpp_if_to_s2 = reply[1].sw_if_index
+
+ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
+
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
+print(vpp.dump(reply))
+
+bd_id = 42
+
+reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
+print(vpp.dump(reply))
+
+for i, v in ipairs(ifaces) do
+  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
+  print(vpp.dump(reply))
+end
+
+^D^D^D
+
+run s1 ping -c 3 192.0.2.2
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.3
+expect s1 packet loss
+run s1 ping -c 3 192.0.2.4
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::2
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::3
+expect s1 packet loss
+run s1 ping6 -c 3 2001:db8:1::4
+expect s1 packet loss
+
+
+cd lua
+--- ACL testing
+
+--[[ temporary comment out
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 0 })
+print(vpp.dump(reply))
+print("---")
+
+acl_index_to_delete = reply[1].acl_index
+print("Deleting " .. tostring(acl_index_to_delete))
+reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+for ri, rv in ipairs(reply) do
+  print("Reply message #" .. tostring(ri))
+  print(vpp.dump(rv))
+  for ai, av in ipairs(rv.r) do
+    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
+  end
+
+end
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
+print(vpp.dump(reply))
+print("---")
+
+
+]] -- end of comment out
+
+---- Should be nothing ^^
+r = {
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
+  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
+  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
+  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
+}
+
+reply = vpp:api_call("acl_add", { context = 42, count = 5, r = r })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_in
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+--- TEST OUTBOUND ACL
+
+r1 = {
+  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
+  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 },
+  { is_permit = 2, is_ipv6 = 0 }
+}
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = r1 })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+r2 = {
+  { is_permit = 1, is_ipv6 = 1 },
+  { is_permit = 0, is_ipv6 = 0 }
+}
+
+reply = vpp:api_call("acl_add", { context = 42, count = 2, r = r2 })
+print(vpp.dump(reply))
+print("---")
+second_interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 1, acl_index = second_interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+^D^D^D
+
+run VPP show classify tables
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s2 nc -v -l -p 22
+run s1 nc 192.0.2.2 22
+run s1 echo
+sleep 1
+run s1 break
+sleep 1
+run VPP show trace
+expect VPP match: outacl 2 rule 2
+run VPP show classify tables
+
+
+run VPP show classify tables
+run VPP clear trace
+run VPP trace add af-packet-input 100
+run s2 nc -v -l -p 22
+run s1 nc 192.0.2.2 22
+run s1 echo
+sleep 1
+run s1 break
+sleep 1
+run VPP show trace
+expect VPP match: outacl 2 rule 2
+run VPP show classify tables
+
+
+run lua print("ALL GOOD!")
+
diff --git a/src/vpp-api/lua/vpp-lapi.lua b/src/vpp-api/lua/vpp-lapi.lua
new file mode 100644
index 00000000..ebfd032b
--- /dev/null
+++ b/src/vpp-api/lua/vpp-lapi.lua
@@ -0,0 +1,989 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+-- json decode/encode from https://gist.github.com/tylerneylon/59f4bcf316be525b30ab
+-- licensed by the author tylerneylon into public domain. Thanks!
+
+local json = {}
+
+-- Internal functions.
+
+local function kind_of(obj)
+  if type(obj) ~= 'table' then return type(obj) end
+  local i = 1
+  for _ in pairs(obj) do
+    if obj[i] ~= nil then i = i + 1 else return 'table' end
+  end
+  if i == 1 then return 'table' else return 'array' end
+end
+
+local function escape_str(s)
+  local in_char  = {'\\', '"', '/', '\b', '\f', '\n', '\r', '\t'}
+  local out_char = {'\\', '"', '/',  'b',  'f',  'n',  'r',  't'}
+  for i, c in ipairs(in_char) do
+    s = s:gsub(c, '\\' .. out_char[i])
+  end
+  return s
+end
+
+-- Returns pos, did_find; there are two cases:
+-- 1. Delimiter found: pos = pos after leading space + delim; did_find = true.
+-- 2. Delimiter not found: pos = pos after leading space;     did_find = false.
+-- This throws an error if err_if_missing is true and the delim is not found.
+local function skip_delim(str, pos, delim, err_if_missing)
+  pos = pos + #str:match('^%s*', pos)
+  if str:sub(pos, pos) ~= delim then
+    if err_if_missing then
+      error('Expected ' .. delim .. ' near position ' .. pos)
+    end
+    return pos, false
+  end
+  return pos + 1, true
+end
+
+-- Expects the given pos to be the first character after the opening quote.
+-- Returns val, pos; the returned pos is after the closing quote character.
+local function parse_str_val(str, pos, val)
+  val = val or ''
+  local early_end_error = 'End of input found while parsing string.'
+  if pos > #str then error(early_end_error) end
+  local c = str:sub(pos, pos)
+  if c == '"'  then return val, pos + 1 end
+  if c ~= '\\' then return parse_str_val(str, pos + 1, val .. c) end
+  -- We must have a \ character.
+  local esc_map = {b = '\b', f = '\f', n = '\n', r = '\r', t = '\t'}
+  local nextc = str:sub(pos + 1, pos + 1)
+  if not nextc then error(early_end_error) end
+  return parse_str_val(str, pos + 2, val .. (esc_map[nextc] or nextc))
+end
+
+-- Returns val, pos; the returned pos is after the number's final character.
+local function parse_num_val(str, pos)
+  local num_str = str:match('^-?%d+%.?%d*[eE]?[+-]?%d*', pos)
+  local val = tonumber(num_str)
+  if not val then error('Error parsing number at position ' .. pos .. '.') end
+  return val, pos + #num_str
+end
+
+
+-- Public values and functions.
+
+function json.stringify(obj, as_key)
+  local s = {}  -- We'll build the string as an array of strings to be concatenated.
+  local kind = kind_of(obj)  -- This is 'array' if it's an array or type(obj) otherwise.
+  if kind == 'array' then
+    if as_key then error('Can\'t encode array as key.') end
+    s[#s + 1] = '['
+    for i, val in ipairs(obj) do
+      if i > 1 then s[#s + 1] = ', ' end
+      s[#s + 1] = json.stringify(val)
+    end
+    s[#s + 1] = ']'
+  elseif kind == 'table' then
+    if as_key then error('Can\'t encode table as key.') end
+    s[#s + 1] = '{'
+    for k, v in pairs(obj) do
+      if #s > 1 then s[#s + 1] = ', ' end
+      s[#s + 1] = json.stringify(k, true)
+      s[#s + 1] = ':'
+      s[#s + 1] = json.stringify(v)
+    end
+    s[#s + 1] = '}'
+  elseif kind == 'string' then
+    return '"' .. escape_str(obj) .. '"'
+  elseif kind == 'number' then
+    if as_key then return '"' .. tostring(obj) .. '"' end
+    return tostring(obj)
+  elseif kind == 'boolean' then
+    return tostring(obj)
+  elseif kind == 'nil' then
+    return 'null'
+  else
+    error('Unjsonifiable type: ' .. kind .. '.')
+  end
+  return table.concat(s)
+end
+
+json.null = {}  -- This is a one-off table to represent the null value.
+
+function json.parse(str, pos, end_delim)
+  pos = pos or 1
+  if pos > #str then error('Reached unexpected end of input.') end
+  local pos = pos + #str:match('^%s*', pos)  -- Skip whitespace.
+  local first = str:sub(pos, pos)
+  if first == '{' then  -- Parse an object.
+    local obj, key, delim_found = {}, true, true
+    pos = pos + 1
+    while true do
+      key, pos = json.parse(str, pos, '}')
+      if key == nil then return obj, pos end
+      if not delim_found then error('Comma missing between object items.') end
+      pos = skip_delim(str, pos, ':', true)  -- true -> error if missing.
+      obj[key], pos = json.parse(str, pos)
+      pos, delim_found = skip_delim(str, pos, ',')
+    end
+  elseif first == '[' then  -- Parse an array.
+    local arr, val, delim_found = {}, true, true
+    pos = pos + 1
+    while true do
+      val, pos = json.parse(str, pos, ']')
+      if val == nil then return arr, pos end
+      if not delim_found then error('Comma missing between array items.') end
+      arr[#arr + 1] = val
+      pos, delim_found = skip_delim(str, pos, ',')
+    end
+  elseif first == '"' then  -- Parse a string.
+    return parse_str_val(str, pos + 1)
+  elseif first == '-' or first:match('%d') then  -- Parse a number.
+    return parse_num_val(str, pos)
+  elseif first == end_delim then  -- End of an object or array.
+    return nil, pos + 1
+  else  -- Parse true, false, or null.
+    local literals = {['true'] = true, ['false'] = false, ['null'] = json.null}
+    for lit_str, lit_val in pairs(literals) do
+      local lit_end = pos + #lit_str - 1
+      if str:sub(pos, lit_end) == lit_str then return lit_val, lit_end + 1 end
+    end
+    local pos_info_str = 'position ' .. pos .. ': ' .. str:sub(pos, pos + 10)
+    error('Invalid json syntax starting at ' .. pos_info_str)
+  end
+end
+
+
+local vpp = {}
+
+local ffi = require("ffi")
+
+--[[
+
+The basic type definitions. A bit of weird gymnastic with
+unionization of the hton* and ntoh* functions results
+is to make handling of signed and unsigned types a bit cleaner,
+essentially building typecasting into a C union.
+
+The vl_api_opaque_message_t is a synthetic type assumed to have
+enough storage to hold the entire API message regardless of the type.
+During the operation it is casted to the specific message struct types.
+
+]]
+
+
+ffi.cdef([[
+
+typedef uint8_t u8;
+typedef int8_t i8;
+typedef uint16_t u16;
+typedef int16_t i16;
+typedef uint32_t u32;
+typedef int32_t i32;
+typedef uint64_t u64;
+typedef int64_t i64;
+typedef double f64;
+typedef float f32;
+
+#pragma pack(1)
+typedef union {
+  u16 u16;
+  i16 i16;
+} lua_ui16t;
+
+#pragma pack(1)
+typedef union {
+  u32 u32;
+  i32 i32;
+} lua_ui32t;
+
+u16 ntohs(uint16_t hostshort);
+u16 htons(uint16_t hostshort);
+u32 htonl(uint32_t along);
+u32 ntohl(uint32_t along);
+void *memset(void *s, int c, size_t n);
+void *memcpy(void *dest, void *src, size_t n);
+
+#pragma pack(1)
+typedef struct _vl_api_opaque_message {
+  u16 _vl_msg_id;
+  u8  data[65536];
+} vl_api_opaque_message_t;
+]])
+
+
+-- CRC-based version stuff
+
+local crc32c_table = ffi.new('const uint32_t[256]',
+  { 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
+  0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
+  0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
+  0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
+  0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
+  0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
+  0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
+  0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
+  0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
+  0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
+  0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
+  0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
+  0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
+  0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
+  0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
+  0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
+  0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
+  0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
+  0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
+  0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
+  0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
+  0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
+  0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
+  0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
+  0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
+  0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
+  0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
+  0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
+  0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
+  0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
+  0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
+  0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
+  0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
+  0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
+  0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
+  0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
+  0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
+  0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
+  0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
+  0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
+  0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
+  0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
+  0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
+  0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
+  0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
+  0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
+  0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
+  0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
+  0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
+  0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
+  0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
+  0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
+  0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
+  0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
+  0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
+  0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
+  0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
+  0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
+  0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
+  0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
+  0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
+  0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
+  0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
+  0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 }
+);
+
+local function CRC8(crc, d)
+  return bit.bxor(bit.rshift(crc, 8), crc32c_table[bit.band(0xff, bit.bxor(crc, d))])
+end
+
+local function CRC16(crc, d)
+  crc = CRC8(crc, bit.band(d, 0xFF))
+  d = bit.rshift(d, 8)
+  crc = CRC8(crc, bit.band(d, 0xFF))
+  return crc
+end
+
+local function string_crc(str, crc)
+  for i=1,#str do
+    -- print("S", i, string.byte(str, i), string.char(string.byte(str, i)))
+    crc = CRC8(crc, string.byte(str, i))
+  end
+  return crc
+end
+
+local tokens = {
+  { ["match"] =' ', ["act"]             = { }  },
+  { ["match"] ='\n', ["act"]             = { }  },
+  { ["match"] ="manual_endian", ["act"]  = { "NODE_MANUAL_ENDIAN", "MANUAL_ENDIAN",    276 } },
+  { ["match"] ="define", ["act"]         = { "NODE_DEFINE",        "DEFINE",           267 } },
+  { ["match"] ="dont_trace", ["act"]     = { "NODE_DONT_TRACE",    "DONT_TRACE",       279 } },
+  { ["match"] ="f64", ["act"]            = { "NODE_F64",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i16", ["act"]            = { "NODE_I16",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i32", ["act"]            = { "NODE_I32",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i64", ["act"]            = { "NODE_I64",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i8", ["act"]             = { "NODE_I8",            "PRIMTYPE",         string_crc } },
+  { ["match"] ="manual_print", ["act"]   = { "NODE_MANUAL_PRINT",  "MANUAL_PRINT",     275 } },
+  { ["match"] ="noversion", ["act"]      = { "NODE_NOVERSION",     "NOVERSION",        274 } },
+  { ["match"] ="packed", ["act"]         = { "NODE_PACKED",        "TPACKED",          266 } },
+  { ["match"] ="typeonly", ["act"]       = { "NODE_TYPEONLY",      "TYPEONLY",         278 } },
+  { ["match"] ="u16", ["act"]            = { "NODE_U16",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="u32", ["act"]            = { "NODE_U32",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="u64", ["act"]            = { "NODE_U64",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="u8", ["act"]             = { "NODE_U8",            "PRIMTYPE",         string_crc } },
+  { ["match"] ="union", ["act"]          = { "NODE_UNION",         "UNION",            271 } },
+  { ["match"] ="uword", ["act"]          = { "NODE_UWORD",         "PRIMTYPE",         string_crc } },
+  { ["match"] ="%(", ["act"]             = { "NODE_LPAR",          "LPAR",             259 } },
+  { ["match"] ="%)", ["act"]             = { "NODE_RPAR",          "RPAR",             258 } },
+  { ["match"] =";", ["act"]              = { "NODE_SEMI",          "SEMI",             260 } },
+  { ["match"] ="%[", ["act"]             = { "NODE_LBRACK",        "LBRACK",           261 } },
+  { ["match"] ="%]", ["act"]             = { "NODE_RBRACK",        "RBRACK",           262 } },
+  { ["match"] ="%{", ["act"]             = { "NODE_LCURLY",        "LCURLY",           268 } },
+  { ["match"] ="%}", ["act"]             = { "NODE_RCURLY",        "RCURLY",           269 } },
+  { ["match"] ='%b""', ["act"]           = { "NODE_STRING",        "STRING",           string_crc } },
+  { ["match"] ='%b@@', ["act"]           = { "NODE_HELPER",        "HELPER_STRING",    string_crc } },
+  -- TODO: \ must be consumed
+  { ["match"] ='[_a-zA-Z][_a-zA-Z0-9]*',
+                       ["act"]           = { "NODE_NAME",          "NAME",             string_crc } },
+  { ["match"] ='[0-9]+', ["act"]         = { "NODE_NUMBER",        "NUMBER",           string_crc } },
+  { ["match"] ='#[^\n]+', ["act"]            = { "NODE_PRAGMA",        "PRAGMA",           nil } },
+}
+
+
+function vpp.crc_version_string(data)
+  local input_crc = 0
+  -- Get rid of comments
+  data = data:gsub("/%*.-%*/", "")
+  data = data:gsub("//[^\n]+", "")
+  -- print(data)
+  idx = 1
+  while (true) do
+    local matched = nil
+    for k, v in ipairs(tokens) do
+      if not matched then
+        local x, y, cap = string.find(data, v["match"], idx)
+        if x == idx then
+          matched = { ["node"] = v["act"], ["x"] = x, ["y"] = y, ["cap"] = cap, ["chars"] = string.sub(data, x, y)  }
+          -- print(k, v, x, y, cap, matched.chars, matched.node[0] )
+        end
+      end
+    end
+    if matched then
+      idx = idx + (matched.y - matched.x + 1)
+      if matched.node[1] then
+        local act = matched.node[3]
+        if type(act) == "function" then
+          input_crc = act(matched.chars, input_crc)
+        elseif type(act) == "number" then
+          input_crc = CRC16(input_crc, act)
+        end
+        -- print(vpp.dump(matched))
+      end
+    else
+      -- print("NOT MATCHED!")
+      local crc = CRC16(input_crc, 0xFFFFFFFF)
+      return string.sub(string.format("%x", crc), -8)
+    end
+  end
+end
+
+
+function vpp.dump(o)
+   if type(o) == 'table' then
+      local s = '{ '
+      for k,v in pairs(o) do
+         if type(k) ~= 'number' then k = '"'..k..'"' end
+         s = s .. '['..k..'] = ' .. vpp.dump(v) .. ','
+      end
+      return s .. '} '
+   else
+      return tostring(o)
+   end
+end
+
+function vpp.hex_dump(buf)
+  local ret = {}
+  for i=1,math.ceil(#buf/16) * 16 do
+    if (i-1) % 16 == 0 then table.insert(ret, string.format('%08X  ', i-1)) end
+    table.insert(ret, ( i > #buf and '   ' or string.format('%02X ', buf:byte(i)) ))
+    if i %  8 == 0 then table.insert(ret, ' ') end
+    if i % 16 == 0 then table.insert(ret, buf:sub(i-16+1, i):gsub('%c','.')..'\n' ) end
+  end
+  return table.concat(ret)
+end
+
+
+function vpp.c_str(text_in)
+  local text = text_in -- \000 will be helpfully added by ffi.copy
+  local c_str = ffi.new("char[?]", #text+1)
+  ffi.copy(c_str, text)
+  return c_str
+end
+
+
+function vpp.init(vpp, args)
+  local pneum_api = args.pneum_api or [[
+ int cough_pneum_attach(char *pneum_path, char *cough_path);
+ int pneum_connect(char *name, char *chroot_prefix, void *cb);
+ int pneum_disconnect(void);
+ int pneum_read(char **data, int *l);
+ int pneum_write(char *data, int len);
+ void pneum_free(char *data);
+ uint32_t pneum_get_msg_index(unsigned char * name);
+]]
+
+  vpp.pneum_path = args.pneum_path
+  ffi.cdef(pneum_api)
+  local init_res = 0
+  vpp.pneum = ffi.load(vpp.pneum_path)
+  if (init_res < 0) then
+    return nil
+  end
+
+  vpp.next_msg_num = 1
+  vpp.msg_name_to_number = {}
+  vpp.msg_name_to_fields = {}
+  vpp.msg_number_to_name = {}
+  vpp.msg_number_to_type = {}
+  vpp.msg_number_to_pointer_type = {}
+  vpp.msg_name_to_crc = {}
+  vpp.c_type_to_fields = {}
+  vpp.events = {}
+  vpp.plugin_version = {}
+  vpp.is_connected = false
+
+
+  vpp.t_lua2c = {}
+  vpp.t_c2lua = {}
+  vpp.t_lua2c["u8"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "string" then
+      -- ffi.copy adds a zero byte at the end. Grrr.
+      -- ffi.copy(dst_c_ptr, src)
+      ffi.C.memcpy(dst_c_ptr, vpp.c_str(src), #src)
+      return(#src)
+    elseif type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u8 *", dst_c_ptr)[i-1] = v
+      end
+      return(#src)
+    else
+      return 1, src -- ffi.cast("u8", src)
+    end
+  end
+  vpp.t_c2lua["u8"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      return ffi.string(src_ptr, src_len)
+    else
+      return (tonumber(src_ptr))
+    end
+  end
+
+  vpp.t_lua2c["u16"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u16 *", dst_c_ptr)[i-1] = ffi.C.htons(v)
+      end
+      return(2 * #src)
+    else
+      return 2, (ffi.C.htons(src))
+    end
+  end
+  vpp.t_c2lua["u16"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(ffi.C.ntohs(src_ptr[i]))
+      end
+      return out
+    else
+      return (tonumber(ffi.C.ntohs(src_ptr)))
+    end
+  end
+
+  vpp.t_lua2c["u32"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u32 *", dst_c_ptr)[i-1] = ffi.C.htonl(v)
+      end
+      return(4 * #src)
+    else
+      return 4, (ffi.C.htonl(src))
+    end
+  end
+  vpp.t_c2lua["u32"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(ffi.C.ntohl(src_ptr[i]))
+      end
+      return out
+    else
+      return (tonumber(ffi.C.ntohl(src_ptr)))
+    end
+  end
+  vpp.t_lua2c["i32"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("i32 *", dst_c_ptr)[i-1] = ffi.C.htonl(v)
+      end
+      return(4 * #src)
+    else
+      return 4, (ffi.C.htonl(src))
+    end
+  end
+  vpp.t_c2lua["i32"] = function(c_type, src_ptr, src_len)
+    local ntohl = function(src)
+      local u32val = ffi.cast("u32", src)
+      local ntohlval = (ffi.C.ntohl(u32val))
+      local out = tonumber(ffi.cast("i32", ntohlval + 0LL))
+      return out
+    end
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(ntohl(src_ptr[i]))
+      end
+    else
+      return (tonumber(ntohl(src_ptr)))
+    end
+  end
+
+  vpp.t_lua2c["u64"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u64 *", dst_c_ptr)[i-1] = v --- FIXME ENDIAN
+      end
+      return(8 * #src)
+    else
+      return 8, ffi.cast("u64", src) --- FIXME ENDIAN
+    end
+  end
+  vpp.t_c2lua["u64"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(src_ptr[i]) -- FIXME ENDIAN
+      end
+      return out
+    else
+      return (tonumber(src_ptr)) --FIXME ENDIAN
+    end
+  end
+
+
+
+
+  vpp.t_lua2c["__MSG__"] = function(c_type, src, dst_c_ptr)
+    local dst = ffi.cast(c_type .. " *", dst_c_ptr)
+    local additional_len = 0
+    local fields_info = vpp.c_type_to_fields[c_type]
+    -- print("__MSG__ type: " .. tostring(c_type))
+    ffi.C.memset(dst_c_ptr, 0, ffi.sizeof(dst[0]))
+    -- print(vpp.dump(fields_info))
+    -- print(vpp.dump(src))
+    for k,v in pairs(src) do
+      local field = fields_info[k]
+      if not field then
+        print("ERROR: field " .. tostring(k) .. " in message " .. tostring(c_type) .. " is unknown")
+      end
+      local lua2c = vpp.t_lua2c[field.c_type]
+      -- print("__MSG__ field " .. tostring(k) .. " : " .. vpp.dump(field))
+      -- if the field is not an array type, try to coerce the argument to a number
+      if not field.array and type(v) == "string" then
+        v = tonumber(v)
+      end
+      if not lua2c then
+        print("__MSG__ " .. tostring(c_type) .. " t_lua2c: can not store field " .. field.name ..
+              " type " .. field.c_type .. " dst " .. tostring(dst[k]))
+        return 0
+      end
+      local len = 0
+      local val = nil
+      if field.array and (type(v) == "table") then
+        -- print("NTFY: field " .. tostring(k) .. " in message " .. tostring(c_type) .. " is an array")
+        for field_i, field_v in ipairs(v) do
+          -- print("NTFY: setting member#" .. tostring(field_i) .. " to value " .. vpp.dump(field_v))
+          local field_len, field_val = lua2c(field.c_type, field_v, dst[k][field_i-1])
+          len = len + field_len
+        end
+      else
+        len, val = lua2c(field.c_type, v, dst[k])
+      end
+      if not field.array then
+        dst[k] = val
+      else
+        if 0 == field.array then
+          additional_len = additional_len + len
+          -- print("Adding " .. tostring(len) .. " bytes due to field " .. tostring(field.name))
+          -- If there is a variable storing the length
+          -- and the input table does not set it, do magic
+          if field.array_size and not src[field.array_size] then
+            local size_field = fields_info[field.array_size]
+            if size_field then
+              dst[field.array_size] = vpp.t_c2lua[size_field.c_type](size_field.c_type, len)
+            end
+          end
+        end
+      end
+      -- print("Full message:\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', req_store_cache), 64)))
+    end
+    return (ffi.sizeof(dst[0])+additional_len)
+  end
+
+  vpp.t_c2lua["__MSG__"] = function(c_type, src_ptr, src_len)
+    local out = {}
+    local reply_typed_ptr = ffi.cast(c_type .. " *", src_ptr)
+    local field_desc = vpp.c_type_to_fields[c_type]
+    if src_len then
+      for i = 0,src_len-1 do
+        out[i+1] = vpp.t_c2lua[c_type](c_type, src_ptr[i])
+      end
+      return out
+    end
+
+    for k, v in pairs(field_desc) do
+      local v_c2lua = vpp.t_c2lua[v.c_type]
+      if v_c2lua then
+        local len = v.array
+        -- print(dump(v))
+        if len then
+          local len_field_name = k .. "_length"
+          local len_field = field_desc[len_field_name]
+          if (len_field) then
+            local real_len = vpp.t_c2lua[len_field.c_type](len_field.c_type, reply_typed_ptr[len_field_name])
+            out[k] =  v_c2lua(v.c_type, reply_typed_ptr[k], real_len)
+          elseif len == 0 then
+            -- check if len = 0, then must be a field which contains the size
+            len_field =  field_desc[v.array_size]
+            local real_len = vpp.t_c2lua[len_field.c_type](len_field.c_type, reply_typed_ptr[v.array_size])
+            -- print("REAL length: " .. vpp.dump(v) .. " : " .. tostring(real_len))
+            out[k] = v_c2lua(v.c_type, reply_typed_ptr[k], real_len)
+          else
+            -- alas, just stuff the entire array
+            out[k] = v_c2lua(v.c_type, reply_typed_ptr[k], len)
+          end
+        else
+          out[k] =  v_c2lua(v.c_type, reply_typed_ptr[k])
+        end
+      else
+        out[k] = ""
+      end
+      -- print(k, out[k])
+    end
+    return out
+  end
+
+  return vpp
+end
+
+function vpp.resolve_message_number(msgname)
+  local name = msgname .. "_" .. vpp.msg_name_to_crc[msgname]
+  local idx = vpp.pneum.pneum_get_msg_index(vpp.c_str(name))
+  if vpp.debug_dump then
+    print("Index for " .. tostring(name) .. " is " .. tostring(idx))
+  end
+  vpp.msg_name_to_number[msgname] = idx
+  vpp.msg_number_to_name[idx] = msgname
+  vpp.msg_number_to_type[idx] = "vl_api_" .. msgname .. "_t"
+  vpp.msg_number_to_pointer_type[idx] = vpp.msg_number_to_type[idx] .. " *"
+  ffi.cdef("\n\n enum { vl_msg_" .. msgname .. " = " .. idx .. " };\n\n")
+end
+
+function vpp.connect(vpp, client_name)
+    local name = "lua_client"
+    if client_name then
+      name = client_name
+    end
+    local ret = vpp.pneum.pneum_connect(vpp.c_str(client_name), nil, nil)
+    if tonumber(ret) == 0 then
+      vpp.is_connected = true
+    end
+    for k, v in pairs(vpp.msg_name_to_number) do
+      vpp.resolve_message_number(k)
+    end
+  end
+
+function vpp.disconnect(vpp)
+    vpp.pneum.pneum_disconnect()
+  end
+
+function vpp.json_api(vpp, path, plugin_name)
+    -- print("Consuming the VPP api from "..path)
+    local ffii = {}
+    local f = io.open(path, "r")
+    if not f then
+      print("Could not open " .. path)
+      return nil
+    end
+    local data = f:read("*all")
+    local json = json.parse(data)
+    if not (json.types or json.messages) then
+      print("Can not parse " .. path)
+      return nil
+    end
+
+    local all_types = {}
+
+    for i, v in ipairs(json.types) do
+      table.insert(all_types, { typeonly = 1, desc = v })
+    end
+    for i, v in ipairs(json.messages) do
+      table.insert(all_types, { typeonly = 0, desc = v })
+    end
+    for i, v in ipairs(all_types) do
+      local typeonly = v.typeonly
+      local name = v.desc[1]
+      local c_type = "vl_api_" .. name .. "_t"
+
+      local fields = {}
+      -- vpp.msg_name_to_fields[name] = fields
+      -- print("CTYPE " .. c_type)
+      vpp.c_type_to_fields[c_type] = fields
+      vpp.t_lua2c[c_type] = vpp.t_lua2c["__MSG__"]
+      vpp.t_c2lua[c_type] = vpp.t_c2lua["__MSG__"]
+
+      local cdef = { "\n\n#pragma pack(1)\ntypedef struct _vl_api_", name, " {\n" }
+      for ii, vv in ipairs(v.desc) do
+        if type(vv) == "table" then
+          if vv.crc then
+            vpp.msg_name_to_crc[name] = string.sub(vv.crc, 3) -- strip the leading 0x
+          else
+            local fieldtype = vv[1]
+            local fieldname = vv[2]
+            local fieldcount = vv[3]
+            local fieldcountvar = vv[4]
+            local fieldrec = { name = fieldname, c_type = fieldtype, array = fieldcount, array_size = fieldcountvar }
+            if fieldcount then
+              table.insert(cdef, "  " .. fieldtype .. " " .. fieldname .. "[" .. fieldcount .. "];\n")
+              if fieldtype == "u8" then
+                -- any array of bytes is treated as a string
+              elseif vpp.t_lua2c[fieldtype] then
+                -- print("Array of " .. fieldtype .. " is ok!")
+              else
+                print("Unknown array type: ", name,  " : " , fieldname, " : ", fieldtype, ":", fieldcount, ":", fieldcountvar)
+              end
+            else
+              table.insert(cdef, "  " .. fieldtype .. " " .. fieldname .. ";\n")
+            end
+            fields[fieldname] = fieldrec
+          end
+        end
+      end
+
+      table.insert(cdef, "} vl_api_" .. name .. "_t;")
+      table.insert(ffii, table.concat(cdef))
+
+      if typeonly == 0 then
+        -- we will want to resolve this later
+        if vpp.debug_dump then
+          print("Remember to resolve " .. name)
+        end
+        vpp.msg_name_to_number[name] = -1
+        if vpp.is_connected then
+          vpp.resolve_message_number(name)
+        end
+      end
+
+    end
+    local cdef_full = table.concat(ffii)
+    ffi.cdef(cdef_full)
+end
+
+function vpp.consume_api(vpp, path, plugin_name)
+    -- print("Consuming the VPP api from "..path)
+    local ffii = {}
+    local f = io.open(path, "r")
+    if not f then
+      print("Could not open " .. path)
+      return nil
+    end
+    local data = f:read("*all")
+    -- Remove all C comments
+    data = data:gsub("/%*.-%*/", "")
+    if vpp.is_connected and not plugin_name then
+      print(path .. ": must specify plugin name!")
+      return
+    end
+    if plugin_name then
+      vpp.plugin_version[plugin_name] = vpp.crc_version_string(data)
+      local full_plugin_name = plugin_name .. "_" .. vpp.plugin_version[plugin_name]
+      local reply = vpp:api_call("get_first_msg_id", { name = full_plugin_name } )
+      vpp.next_msg_num = tonumber(reply[1].first_msg_id)
+      print("Plugin " .. full_plugin_name .. " first message is " .. tostring(vpp.next_msg_num))
+    end
+    -- print ("data len: ", #data)
+    data = data:gsub("\n(.-)(%S+)%s*{([^}]*)}", function (preamble, name, members)
+      local _, typeonly = preamble:gsub("typeonly", "")
+      local maybe_msg_id_field = { [0] = "u16 _vl_msg_id;", "" }
+      local onedef = "\n\n#pragma pack(1)\ntypedef struct _vl_api_"..name.. " {\n" ..
+	   -- "   u16 _vl_msg_id;" ..
+           maybe_msg_id_field[typeonly] ..
+	   members:gsub("%[[a-zA-Z_]+]", "[0]") ..
+	   "} vl_api_" .. name .. "_t;"
+
+      local c_type = "vl_api_" .. name .. "_t"
+
+      local fields = {}
+      -- vpp.msg_name_to_fields[name] = fields
+      -- print("CTYPE " .. c_type)
+      vpp.c_type_to_fields[c_type] = fields
+      vpp.t_lua2c[c_type] = vpp.t_lua2c["__MSG__"]
+      vpp.t_c2lua[c_type] = vpp.t_c2lua["__MSG__"]
+      local mirec = { name = "_vl_msg_id", c_type = "u16", array = nil, array_size = nil }
+      if typeonly == 0 then
+        fields[mirec.name] = mirec
+      end
+
+      -- populate the field reflection table for the message
+      -- sets the various type information as well as the accessors for lua<->C conversion
+      members:gsub("(%S+)%s+(%S+);", function (fieldtype, fieldname)
+          local fieldcount = nil
+          local fieldcountvar = nil
+          -- data = data:gsub("%[[a-zA-Z_]+]", "[0]")
+          fieldname = fieldname:gsub("(%b[])", function(cnt)
+              fieldcount = tonumber(cnt:sub(2, -2));
+              if not fieldcount then
+                fieldcount = 0
+                fieldcountvar = cnt:sub(2, -2)
+              end
+              return ""
+            end)
+	  local fieldrec = { name = fieldname, c_type = fieldtype, array = fieldcount, array_size = fieldcountvar }
+          if fieldcount then
+            if fieldtype == "u8" then
+              -- any array of bytes is treated as a string
+            elseif vpp.t_lua2c[fieldtype] then
+              -- print("Array of " .. fieldtype .. " is ok!")
+            else
+              print("Unknown array type: ", name,  " : " , fieldname, " : ", fieldtype, ":", fieldcount, ":", fieldcountvar)
+            end
+          end
+	  fields[fieldname] = fieldrec
+	end)
+
+      -- print(dump(fields))
+
+      if typeonly == 0 then
+	local this_message_number = vpp.next_msg_num
+	vpp.next_msg_num = vpp.next_msg_num + 1
+	vpp.msg_name_to_number[name] = this_message_number
+	vpp.msg_number_to_name[this_message_number] = name
+	vpp.msg_number_to_type[this_message_number] = "vl_api_" .. name .. "_t"
+	vpp.msg_number_to_pointer_type[this_message_number] = vpp.msg_number_to_type[this_message_number] .. " *"
+	onedef = onedef .. "\n\n enum { vl_msg_" .. name .. " = " .. this_message_number .. " };\n\n"
+      end
+      table.insert(ffii, onedef);
+      return "";
+      end)
+    local cdef = table.concat(ffii)
+    -- print(cdef)
+    ffi.cdef(cdef)
+  end
+
+
+function vpp.lua2c(vpp, c_type, src, dst_c_ptr)
+  -- returns the number of bytes written to memory pointed by dst
+  local lua2c = vpp.t_lua2c[c_type]
+  if lua2c then
+    return(lua2c(c_type, src, dst_c_ptr))
+  else
+    print("vpp.lua2c: do not know how to store type " .. tostring(c_type))
+    local x = "a" .. nil
+    return 0
+  end
+end
+
+function vpp.c2lua(vpp, c_type, src_ptr, src_len)
+  -- returns the lua data structure
+  local c2lua = vpp.t_c2lua[c_type]
+  if c2lua then
+    return(c2lua(c_type, src_ptr, src_len))
+  else
+    print("vpp.c2lua: do not know how to load type " .. c_type)
+    return nil
+  end
+end
+
+local req_store_cache = ffi.new("vl_api_opaque_message_t[1]")
+
+function vpp.api_write(vpp, api_name, req_table)
+    local msg_num = vpp.msg_name_to_number[api_name]
+    if not msg_num then
+      print ("API call "..api_name.." is not known")
+      return nil
+    end
+
+    if not req_table then
+      req_table = {}
+    end
+    req_table._vl_msg_id = msg_num
+
+    local packed_len = vpp:lua2c(vpp.msg_number_to_type[msg_num], req_table, req_store_cache)
+    if vpp.debug_dump then
+      print("Write Message length: " .. tostring(packed_len) .. "\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', req_store_cache), packed_len)))
+    end
+
+    res = vpp.pneum.pneum_write(ffi.cast('void *', req_store_cache), packed_len)
+    return res
+  end
+
+local rep_store_cache = ffi.new("vl_api_opaque_message_t *[1]")
+local rep_len_cache = ffi.new("int[1]")
+
+function vpp.api_read(vpp)
+    local rep_type = "vl_api_opaque_message_t"
+    local rep = rep_store_cache
+    local replen = rep_len_cache
+    res = vpp.pneum.pneum_read(ffi.cast("void *", rep), replen)
+    if vpp.debug_dump then
+      print("Read Message length: " .. tostring(replen[0]) .. "\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', rep[0]), replen[0])))
+    end
+
+    local reply_msg_num = ffi.C.ntohs(rep[0]._vl_msg_id)
+    local reply_msg_name = vpp.msg_number_to_name[reply_msg_num]
+
+    local reply_typed_ptr = ffi.cast(vpp.msg_number_to_pointer_type[reply_msg_num], rep[0])
+    local out = vpp:c2lua(vpp.msg_number_to_type[reply_msg_num], rep[0], nil, replen[0])
+    if type(out) == "table" then
+      out["luaapi_message_name"] = reply_msg_name
+    end
+
+    vpp.pneum.pneum_free(ffi.cast('void *',rep[0]))
+
+    return reply_msg_name, out
+  end
+
+function vpp.api_call(vpp, api_name, req_table, options_in)
+    local msg_num = vpp.msg_name_to_number[api_name]
+    local end_message_name = api_name .."_reply"
+    local replies = {}
+    local cstruct = ""
+    local options = options_in or {}
+    if msg_num then
+      if vpp.debug_dump then
+        print("Message #" .. tostring(msg_num) .. " for name " .. tostring(api_name))
+      end
+      vpp:api_write(api_name, req_table)
+      if not vpp.msg_name_to_number[end_message_name] or options.force_ping then
+        end_message_name = "control_ping_reply"
+        vpp:api_write("control_ping")
+      end
+      repeat
+        reply_message_name, reply = vpp:api_read()
+        if reply and not reply.context then
+          -- there may be async events inbetween
+          table.insert(vpp.events, reply)
+        else
+          if reply_message_name ~= "control_ping_reply" then
+            -- do not insert the control ping encapsulation
+            table.insert(replies, reply)
+          end
+        end
+        -- print(reply)
+      until reply_message_name == end_message_name
+    else
+      print(api_name .. " is an unknown API call")
+      return nil
+    end
+    return replies
+  end
+
+return vpp
diff --git a/src/vpp.am b/src/vpp.am
index a02206f8..1d07b90a 100644
--- a/src/vpp.am
+++ b/src/vpp.am
@@ -74,7 +74,7 @@ PLUGIN_DPDK_ARG=""
 endif
 
 vpp_plugin_configure:
-	@echo "PLUGIN CONFIGURE " $@ 
+	@echo "PLUGIN CFG" $@
 	@echo "#!/bin/bash" > $@
 	@echo " " >> $@
 	@echo "set +eu" >> $@
diff --git a/src/vppapigen.am b/src/vppapigen.am
index 083ee537..edde339d 100644
--- a/src/vppapigen.am
+++ b/src/vppapigen.am
@@ -16,9 +16,9 @@ bin_PROGRAMS += vppapigen
 BUILT_SOURCES += tools/vppapigen/gram.h
 
 tools/vppapigen/gram.h: tools/vppapigen/gram.y
-	$(YACC) -d @srcdir@/tools/vppapigen/gram.y
-	mv y.tab.h tools/vppapigen/gram.h
-	rm y.tab.c
+	@$(YACC) -d @srcdir@/tools/vppapigen/gram.y
+	@mv y.tab.h tools/vppapigen/gram.h
+	@rm y.tab.c
 
 vppapigen_SOURCES = tools/vppapigen/gram.y tools/vppapigen/lex.c tools/vppapigen/node.c
 vppapigen_LDADD = libvppinfra.la
diff --git a/vpp-api/Makefile.am b/vpp-api/Makefile.am
deleted file mode 100644
index b50522ef..00000000
--- a/vpp-api/Makefile.am
+++ /dev/null
@@ -1,4 +0,0 @@
-AUTOMAKE_OPTIONS = foreign
-SUBDIRS = java
-
-# vi:syntax=automake
diff --git a/vpp-api/configure.ac b/vpp-api/configure.ac
deleted file mode 100644
index 278041c7..00000000
--- a/vpp-api/configure.ac
+++ /dev/null
@@ -1,12 +0,0 @@
-AC_INIT(vpp-api, 1.1.0)
-LT_INIT
-AC_CONFIG_MACRO_DIR([m4])
-AC_CONFIG_SUBDIRS([java])
-AM_INIT_AUTOMAKE
-AM_SILENT_RULES([yes])
-
-AC_PROG_CC
-
-AC_CONFIG_FILES([Makefile])
-AC_OUTPUT
-
diff --git a/vpp-api/java/Makefile.am b/vpp-api/java/Makefile.am
deleted file mode 100644
index b0551965..00000000
--- a/vpp-api/java/Makefile.am
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright (c) 2015 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AUTOMAKE_OPTIONS = foreign subdir-objects
-ACLOCAL_AMFLAGS = -I m4
-AM_CFLAGS = -Wall
-
-noinst_PROGRAMS =
-BUILT_SOURCES =
-bin_PROGRAMS =
-CLEANFILES =
-lib_LTLIBRARIES =
-
-#
-# jvpp-common
-#
-
-nobase_include_HEADERS =		\
-  jvpp-common/jvpp_common.h
-
-lib_LTLIBRARIES += libjvpp_common.la
-
-libjvpp_common_la_SOURCES = jvpp-common/jvpp_common.c
-libjvpp_common_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra \
-	              -lpthread -lm -lrt
-libjvpp_common_la_LDFLAGS = -module
-libjvpp_common_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux
-
-
-#
-# jvpp-registry (connection management + plugin registry)
-#
-lib_LTLIBRARIES += libjvpp_registry.la
-
-libjvpp_registry_la_SOURCES = jvpp-registry/jvpp_registry.c
-libjvpp_registry_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra \
-	              -lpthread -lm -lrt -ljvpp_common
-libjvpp_registry_la_LDFLAGS = -module
-libjvpp_registry_la_CPPFLAGS = -Ijvpp-registry -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux
-EXTRA_libjvpp_registry_la_DEPENDENCIES=libjvpp_common.la
-
-jarfile_jvpp_registry = jvpp-registry-$(PACKAGE_VERSION).jar
-packagedir_jvpp_registry = io/fd/vpp/jvpp
-
-BUILT_SOURCES += jvpp-registry/io_fd_vpp_jvpp_VppJNIConnection.h
-
-jvpp_registry_src_files := $(wildcard @srcdir@/jvpp-registry/$(packagedir_jvpp_registry)/*.java) $(wildcard @srcdir@/jvpp-registry/$(packagedir_jvpp_registry)/**/*.java)
-
-jvpp-registry/io_fd_vpp_jvpp_VppJNIConnection.h: $(jvpp_registry_src_files)
-	@echo " jvpp-registry.jar generation"
-	mkdir -p jvpp-registry/target
-	$(JAVAC) -d jvpp-registry/target $(jvpp_registry_src_files)
-	$(JAVAH) -force -classpath jvpp-registry/target -d jvpp-registry io.fd.vpp.jvpp.VppJNIConnection
-	$(JAVAH) -force -classpath jvpp-registry/target -d jvpp-registry io.fd.vpp.jvpp.JVppRegistryImpl
-
-$(jarfile_jvpp_registry): libjvpp_registry.la
-	cp .libs/libjvpp_registry.so.0.0.0 jvpp-registry/target;	\
-	$(JAR) cf $(JARFLAGS) $@ -C jvpp-registry/target .;
-
-#
-# jvpp-core (Java wrapper for vpe.api)
-#
-lib_LTLIBRARIES += libjvpp_core.la
-
-libjvpp_core_la_SOURCES = jvpp-core/jvpp_core.c jvpp-core/jvpp_core_gen.h
-libjvpp_core_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra \
-	              -lpthread -lm -lrt -ljvpp_common
-libjvpp_core_la_LDFLAGS = -module
-libjvpp_core_la_CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux
-
-jarfile_jvpp_core = jvpp-core-$(PACKAGE_VERSION).jar
-packagedir_jvpp_core = io/fd/vpp/jvpp/core
-
-BUILT_SOURCES += jvpp-core/io_fd_vpp_jvpp_core_JVppCoreImpl.h
-
-JSON_FILES = \
-	$(wildcard $(prefix)/../vpp/share/vpp/api/core/*.api.json)
-
-jvpp-core/io_fd_vpp_jvpp_core_JVppCoreImpl.h:		\
-    jvpp-registry/io_fd_vpp_jvpp_VppJNIConnection.h	\
-    $(JSON_FILES)
-	cp -rf @srcdir@/jvpp-core/* -t jvpp-core/
-	mkdir -p jvpp-core/target
-	cd jvpp-core \
-		&& mkdir -p types dto future callfacade callback notification \
-		&& @srcdir@/jvpp/gen/jvpp_gen.py  --plugin_name core \
-		    -i $(JSON_FILES) \
-		&& cp -rf types dto future callfacade callback notification *.java -t $(packagedir_jvpp_core) \
-		&& rm -rf types dto future callfacade callback notification *.java
-
-	$(JAVAC) -classpath jvpp-registry/target -d jvpp-core/target jvpp-core/$(packagedir_jvpp_core)/*.java \
-		jvpp-core/$(packagedir_jvpp_core)/types/*.java \
-		jvpp-core/$(packagedir_jvpp_core)/dto/*.java \
-		jvpp-core/$(packagedir_jvpp_core)/callback/*.java \
-		jvpp-core/$(packagedir_jvpp_core)/notification/*.java \
-		jvpp-core/$(packagedir_jvpp_core)/future/*.java 	\
-		jvpp-core/$(packagedir_jvpp_core)/callfacade/*.java 	\
-		jvpp-core/$(packagedir_jvpp_core)/test/*.java 	\
-		|| (echo "jvpp-core compilation failed: $$?"; exit 1)
-	$(JAVAH) -force -classpath jvpp-registry/target:jvpp-core/target -d jvpp-core io.fd.vpp.jvpp.core.JVppCoreImpl
-
-$(jarfile_jvpp_core): libjvpp_core.la
-	cp .libs/libjvpp_core.so.0.0.0 jvpp-core/target
-	$(JAR) cf $(JARFLAGS) $@ -C jvpp-core/target .
-
-all-local: $(jarfile_jvpp_registry) $(jarfile_jvpp_core)
diff --git a/vpp-api/java/Readme.txt b/vpp-api/java/Readme.txt
deleted file mode 100644
index 689b9b37..00000000
--- a/vpp-api/java/Readme.txt
+++ /dev/null
@@ -1,236 +0,0 @@
-= JVpp
-
-JVpp is JNI based Java API for VPP.
-
-== Features
-It is:
-
-* Asynchronous
-* Fully generated
-* Lightweight
-
-== Architecture
-
-=== Plugin support
-
-  /-------------\        /--------------\          /---------------\
-  | JvppPlugin1 +<-------+ JVppRegistry +--------->+ VppConnection |
-  \-------------/  inits \--+-----------/   uses   \---------------/
-                            |
-  /-------------\           |
-  | JvppPlugin2 +<----------+ inits
-  \-------------/           |
-                            |
-  ...                       |
-                            |
-  /----------\              |
-  | JVppCore +<-------------+
-  \----------/
-
-
-VppRegistry opens connection to vpp (VppConnection) and manages jvpp plugins.
-Each plugin needs to be registered in the VppRegistry. Registration involves
-plugin initialization (providing JNI implementation with JVppCallback reference,
-vpp client identifier and vpp shared memory queue address).
-
-API user sends message by calling a method of appropriate plugin interface.
-The call is delegated to JNI implementation provided by the particular plugin.
-When JNI code receives reply, it invokes callback method of JVppCallback
-that corresponds to the received message reply.
-
-=== JVppCore as an example of JVpp plugin architecture
-
- JVpp Java
-
-  /--------------\             /----------\          /------------\    /------\
-  | JVppRegistry |             | JVppCore |          |  Callbacks |    | DTOs |
-  \----+---------/             \----+-----/          \------+-----/    \------/
-       ^                            ^                       ^
-       | implements                 | implements            | implements
-  /----+--------------\         /---+----------\      /-----+---------\
-  | JVppRegistryImpl* +-------->+ JVppCoreImpl |      |  JVppCallback |
-  \-------+-----------/  inits  \---+----------/      \-------+-------/
-          |                          |                       ^
-          |                          | uses                  | calls back
-          |                          |                       |
-----------|--------------------------|-----------------------|---------------------
-          |                          |                       |
- C JNI    |                          +-------------------+   |       /-----------------\
-          v                                              |   |   +-->+ jvpp_core_gen.h |
-  /--------+--------\                                    |   |   |   \-----------------/
-  | jpp_registry.c* +---+   /--------+----+----\         |   |   |
-  \-----------------/   |   | << shared lib >> |        /-+--+---+------\
-                        + ->+   jvpp_common*   <--------+  jvpp_core.c* |
-                      uses  \------------------/  uses  \---------------/
-
-
-* Components marked with an asterisk contain manually crafted code, which in addition
-to generated classes form jvpp. Exception applies to Callbacks and DTOs, since there are
-manually crafted marker interfaces in callback and dto package (dto/JVppRequest, dto/JVppReply,
-dto/JVppDump, dto/JVppReplyDump, callback/JVppCallback)
-
-Note: jvpp_core.c calls back the JVppCallback instance with every response. An instance of the
-JVppCallback is provided to jvpp_core.c by JVppRegistryImpl on JVppCoreImpl initialization.
-
-Part of the JVpp is also Future facade. It is asynchronous API returning Future objects
-on top of low level JVpp. It wraps dump reply messages in one DTO using control_ping message
-(provided by JVppRegistry).
-
-
-Future facade
-
-        /----------------\          /---------------\
-        | FutureJVppCore |      +-->+ JVppRegistry* |
-        \-----+----------/      |   \---------------/
-              ^                 |
-              | implements      | uses
-              |                 |
-     /--------+-------------\   |    /------------------------------\
-     | FutureJVppCoreFacade +---+--->+ FutureJVppCoreFacadeCallback |
-     \---------+------------/  uses  \-------+----------------------/
-               |                             |
----------------|-----------------------------|-------------------------------
-               | uses                        | implements
-JVpp Java      |                             |
-               |                             |
- /----------\  |                             |
- | JVppCore +<-+                             |
- \----+-----/                                |
-      ^                                      |
-      | implements                           v
- /----+---------\                   /--------+---------------\
- | JVppCoreImpl |                   | JVppCoreGlobalCallback |
- \--------------/                   \------------------------/
-
-
-
-Another useful utility of the JVpp is Callback facade. It is asynchronous API
-capable of calling specific callback instance (provided when performing a call)
-per call.
-
-
-Callback facade
-
-        /------------------\          /---------------\
-        | CallbackJVppCore |      +-->+ JVppRegistry* |
-        \-----+------------/      |   \---------------/
-              ^                   |
-              | implements        | uses
-              |                   |
-     /--------+---------------\   |    /--------------------------\
-     | CallbackJVppCoreFacade +---+--->+ CallbackJVppCoreCallback |
-     \---------+--------------/  uses  \-----+--------------------/
-               |                             |
----------------|-----------------------------|-------------------------------
-               | uses                        | implements
-JVpp Java      |                             |
-               |                             |
- /----------\  |                             |
- | JVppCore +<-+                             |
- \----+-----/                                |
-      ^                                      |
-      | implements                           v
- /----+---------\                   /----------+-------------\
- | JVppCoreImpl |                   | JVppCoreGlobalCallback |
- \--------------/                   \------------------------/
-
-
-== Package structure
-
-* *io.fd.vpp.jvpp* - top level package for generated JVpp interface+ implementation and hand-crafted
-VppConnection interface + implementation - packaged as jvpp-registry-version.jar
-
-* *io.fd.vpp.jvpp.[plugin]* - top level package for generated JVpp interface + implementation
-+ plugin's API tests - packaged as jvpp-[plugin]-version.jar
-
-** *dto* - package for DTOs generated from VPP API structures + base/marker hand-crafted interfaces
-(in case of jvpp-registry)
-** *callback* - package for low-level JVpp callbacks and a global callback interface implementing each of
-the low-level JVppcallbacks
-** *future* - package for future based facade on top of JVpp and callbacks
-** *callfacade* - package for callback based facade on top of JVpp and callbacks. Allowing
-users to provide callback per request
-** *test* - package for JVpp standalone tests. Can also serve as samples for JVpp.
-
-C code is structured into modules:
-
-* *jvpp_common* - shared library that provides jvpp_main_t reference used by jvpp_registry and plugins.
-
-* *jvpp_registry* - native library used by JVppRegistryImpl, responsible for:
-
-** VPP connection open/close
-** Rx thread to java thread attach
-** control ping message handling
-
-* *jvpp_core* - native library used by jvpp core plugin:
-** *jvpp_core.c* - contains hand crafted code for core plugin initialization
-** *jvpp_core_gen.h* - contains generated JNI compatible handlers for all requests and replies defined in vpe.api
-
-== Code generators
-All of the required code except the base/marker interfaces is generated using
-simple python2 code generators. The generators use __defs_vpp_papi.py__ input
-file produced by __vppapigen__ from vpe.api file.
-
-=== JNI compatible C code
-Produces __jvpp_[plugin]_gen.h__ file containing JNI compatible handlers for each VPP
-request and reply.
-
-[NOTE]
-====
-Source: jvpp_c_gen.py
-====
-
-=== Request/Reply DTOs
-For all the structures in __defs_vpp_papi.py__ a POJO DTO is produced. Logically,
-there are 4 types of DTOs:
-
-* Request - requests that can be sent to VPP and only a single response is expected
-* DumpRequest - requests that can be sent to VPP and a stream of responses is expected
-* Reply - reply to a simple request or a single response from dump triggered response stream
-* ReplyDump - collection of replies from a single dump request
-* Notifications/Events - Not implemented yet
-
-[NOTE]
-====
-Source: dto_gen.py
-====
-
-=== JVpp
-Produces __JVpp.java__ and __JVppImpl.java__. This is the layer right above JNI compatible C
-code.
-
-[NOTE]
-====
-Source: jvpp_impl_gen.py
-====
-
-=== Callbacks
-Produces callback interface for each VPP reply + a global callback interface called
-__JVpp[plugin]GlobalCallback.java__ aggregating all of the callback interfaces. The JNI
-compatible C code expects only a single instance of this global callback and calls
-it with every reply.
-
-[NOTE]
-====
-Source: callback_gen.py
-====
-
-=== Future facade
-Produces an asynchronous facade on top of JVpp and callbacks, which returns a Future that provides
-matching reply once VPP invocation finishes. Sources produced:
-__FutureJVpp[plugin].java, FutureJVpp[plugin]Facade.java and FutureJVpp[plugin]Callback.java__
-
-[NOTE]
-====
-Source: jvpp_future_facade_gen.py
-====
-
-=== Callback facade
-Similar to future facade, only this facade takes callback objects as part of the invocation
-and the callback is called with result once VPP invocation finishes. Sources produced:
-__CallbackJVpp[plugin].java, CallbackJVpp[plugin]Facade.java and CallbackJVpp[plugin]Callback.java__
-
-[NOTE]
-====
-Source: jvpp_callback_facade_gen.py
-====
diff --git a/vpp-api/java/configure.ac b/vpp-api/java/configure.ac
deleted file mode 100644
index 3082781e..00000000
--- a/vpp-api/java/configure.ac
+++ /dev/null
@@ -1,24 +0,0 @@
-AC_INIT(jvpp, 17.04)
-LT_INIT
-AC_CONFIG_MACRO_DIR([m4])
-AM_INIT_AUTOMAKE
-AM_SILENT_RULES([yes])
-
-AC_PROG_CC
-
-if test -f /usr/bin/lsb_release  && test `lsb_release -si` == "Ubuntu"  && test `lsb_release -sr` == "14.04"  && test -d /usr/lib/jvm/java-8-openjdk-amd64/ ; then
-    JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-    JAVAC=${JAVA_HOME}/bin/javac
-    PATH=${JAVA_HOME}/bin/:${PATH}
-    break
-fi
-
-AX_CHECK_JAVA_HOME
-AX_PROG_JAVAC
-AX_PROG_JAVAH
-AX_PROG_JAR
-AX_PROG_JAVADOC
-AX_PROG_JAVA
-
-AC_OUTPUT([Makefile])
-
diff --git a/vpp-api/java/jvpp-common/jvpp_common.c b/vpp-api/java/jvpp-common/jvpp_common.c
deleted file mode 100644
index a161c09c..00000000
--- a/vpp-api/java/jvpp-common/jvpp_common.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#define _GNU_SOURCE /* for strcasestr(3) */
-
-#include "jvpp_common.h"
-
-#ifndef JVPP_DEBUG
-#define JVPP_DEBUG 0
-#endif
-
-#if JVPP_DEBUG == 1
-#define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
-#else
-#define DEBUG_LOG(...)
-#endif
-
-/* shared jvpp main structure */
-jvpp_main_t jvpp_main __attribute__((aligned (64)));
-
-void call_on_error(const char* callName, int contextId, int retval,
-        jclass callbackClass, jobject callbackObject,
-        jclass callbackExceptionClass) {
-    DEBUG_LOG("\nCallOnError : callback=%s, retval=%d, context=%d\n", callName,
-            clib_net_to_host_u32(retval), clib_net_to_host_u32(context));
-    JNIEnv *env = jvpp_main.jenv;
-    if (!callbackClass) {
-        DEBUG_LOG("CallOnError : jm->callbackClass is null!\n");
-        return;
-    }
-    jmethodID excConstructor = (*env)->GetMethodID(env, callbackExceptionClass,
-            "", "(Ljava/lang/String;II)V");
-    if (!excConstructor) {
-        DEBUG_LOG("CallOnError : excConstructor is null!\n");
-        return;
-    }
-    jmethodID callbackExcMethod = (*env)->GetMethodID(env, callbackClass,
-            "onError", "(Lio/fd/vpp/jvpp/VppCallbackException;)V");
-    if (!callbackExcMethod) {
-        DEBUG_LOG("CallOnError : callbackExcMethod is null!\n");
-        return;
-    }
-
-    jobject excObject = (*env)->NewObject(env, callbackExceptionClass,
-            excConstructor, (*env)->NewStringUTF(env, callName),
-            clib_net_to_host_u32(contextId), clib_net_to_host_u32(retval));
-    if (!excObject) {
-        DEBUG_LOG("CallOnError : excObject is null!\n");
-        return;
-    }
-
-    (*env)->CallVoidMethod(env, callbackObject, callbackExcMethod, excObject);
-    DEBUG_LOG("CallOnError : Response sent\n");
-}
diff --git a/vpp-api/java/jvpp-common/jvpp_common.h b/vpp-api/java/jvpp-common/jvpp_common.h
deleted file mode 100644
index bbb203ed..00000000
--- a/vpp-api/java/jvpp-common/jvpp_common.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_jvpp_common_h__
-#define __included_jvpp_common_h__
-//
-#include 
-#include 
-#include 
-#include 
-
-typedef struct {
-    /* Unique identifier used for matching replays with requests  */
-    volatile u32 context_id;
-
-    /* Spinlock */
-    volatile u32 lock;
-    u32 tag;
-
-    /* JNI Native Method Interface pointer for message handlers */
-    JNIEnv *jenv;
-
-    /* JNI Invoke Interface pointer for attachment of rx thread to java thread */
-    JavaVM *jvm;
-
-    /* Convenience */
-    unix_shared_memory_queue_t * vl_input_queue;
-    u32 my_client_index;
-} jvpp_main_t;
-
-extern jvpp_main_t jvpp_main __attribute__((aligned (64)));
-
-static_always_inline u32 vppjni_get_context_id(jvpp_main_t * jm) {
-    return __sync_add_and_fetch(&jm->context_id, 1);
-}
-
-static_always_inline void vppjni_lock(jvpp_main_t * jm, u32 tag) {
-    while (__sync_lock_test_and_set(&jm->lock, 1))
-        ;
-    jm->tag = tag;
-}
-
-static_always_inline void vppjni_unlock(jvpp_main_t * jm) {
-    jm->tag = 0;
-    CLIB_MEMORY_BARRIER();
-    jm->lock = 0;
-}
-
-/**
- * Calls onError callback on callbackObject reference. Passes instance of callbackExceptionClass as parameter.
- */
-void call_on_error(const char* callName, int contextId, int retval,
-        jclass callbackClass, jobject callbackObject,
-        jclass callbackExceptionClass);
-
-#endif /* __included_jvpp_common_h__ */
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java
deleted file mode 100644
index 986993b8..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackApiTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.VppCallbackException;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.callback.GetNodeIndexCallback;
-import io.fd.vpp.jvpp.core.callback.ShowVersionCallback;
-import io.fd.vpp.jvpp.core.callback.SwInterfaceCallback;
-import io.fd.vpp.jvpp.core.dto.GetNodeIndex;
-import io.fd.vpp.jvpp.core.dto.GetNodeIndexReply;
-import io.fd.vpp.jvpp.core.dto.ShowVersion;
-import io.fd.vpp.jvpp.core.dto.ShowVersionReply;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDetails;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDump;
-
-public class CallbackApiTest {
-
-    public static void main(String[] args) throws Exception {
-        testCallbackApi();
-    }
-
-    private static void testCallbackApi() throws Exception {
-        System.out.println("Testing Java callback API with JVppRegistry");
-        try (final JVppRegistry registry = new JVppRegistryImpl("CallbackApiTest");
-             final JVpp jvpp = new JVppCoreImpl()) {
-            registry.register(jvpp, new TestCallback());
-
-            System.out.println("Sending ShowVersion request...");
-            final int result = jvpp.send(new ShowVersion());
-            System.out.printf("ShowVersion send result = %d%n", result);
-
-            System.out.println("Sending GetNodeIndex request...");
-            GetNodeIndex getNodeIndexRequest = new GetNodeIndex();
-            getNodeIndexRequest.nodeName = "non-existing-node".getBytes();
-            jvpp.send(getNodeIndexRequest);
-
-            System.out.println("Sending SwInterfaceDump request...");
-            SwInterfaceDump swInterfaceDumpRequest = new SwInterfaceDump();
-            swInterfaceDumpRequest.nameFilterValid = 0;
-            swInterfaceDumpRequest.nameFilter = "".getBytes();
-            jvpp.send(swInterfaceDumpRequest);
-
-            Thread.sleep(1000);
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    static class TestCallback implements GetNodeIndexCallback, ShowVersionCallback, SwInterfaceCallback {
-
-        @Override
-        public void onGetNodeIndexReply(final GetNodeIndexReply msg) {
-            System.out.printf("Received GetNodeIndexReply: %s%n", msg);
-        }
-
-        @Override
-        public void onShowVersionReply(final ShowVersionReply msg) {
-            System.out.printf("Received ShowVersionReply: context=%d, program=%s, version=%s, "
-                    + "buildDate=%s, buildDirectory=%s%n",
-                msg.context, new String(msg.program), new String(msg.version),
-                new String(msg.buildDate), new String(msg.buildDirectory));
-        }
-
-        @Override
-        public void onSwInterfaceDetails(final SwInterfaceDetails msg) {
-            System.out.printf("Received SwInterfaceDetails: interfaceName=%s, l2AddressLength=%d, adminUpDown=%d, "
-                    + "linkUpDown=%d, linkSpeed=%d, linkMtu=%d%n",
-                new String(msg.interfaceName), msg.l2AddressLength, msg.adminUpDown,
-                msg.linkUpDown, msg.linkSpeed, (int) msg.linkMtu);
-        }
-
-        @Override
-        public void onError(VppCallbackException ex) {
-            System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(),
-                ex.getCtxId(), ex.getErrorCode());
-        }
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java
deleted file mode 100644
index d84cb034..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeNotificationTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.VppCallbackException;
-import io.fd.vpp.jvpp.core.JVppCore;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.callback.WantInterfaceEventsCallback;
-import io.fd.vpp.jvpp.core.callfacade.CallbackJVppCoreFacade;
-import io.fd.vpp.jvpp.core.dto.WantInterfaceEventsReply;
-
-public class CallbackJVppFacadeNotificationTest {
-
-    private static void testCallbackFacade() throws Exception {
-        System.out.println("Testing CallbackJVppFacade for notifications");
-
-        try (final JVppRegistry registry = new JVppRegistryImpl("CallbackFacadeTest");
-             final JVppCore jvpp = new JVppCoreImpl()) {
-            final CallbackJVppCoreFacade jvppCallbackFacade = new CallbackJVppCoreFacade(registry, jvpp);
-            System.out.println("Successfully connected to VPP");
-
-            final AutoCloseable notificationListenerReg =
-                jvppCallbackFacade.getNotificationRegistry().registerSwInterfaceSetFlagsNotificationCallback(
-                    NotificationUtils::printNotification
-                );
-
-            jvppCallbackFacade.wantInterfaceEvents(NotificationUtils.getEnableInterfaceNotificationsReq(),
-                new WantInterfaceEventsCallback() {
-                    @Override
-                    public void onWantInterfaceEventsReply(final WantInterfaceEventsReply reply) {
-                        System.out.println("Interface events started");
-                    }
-
-                    @Override
-                    public void onError(final VppCallbackException ex) {
-                        System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n",
-                            ex.getMethodName(), ex.getCtxId(), ex.getErrorCode());
-                    }
-                });
-
-            System.out.println("Changing interface configuration");
-            NotificationUtils.getChangeInterfaceState().send(jvpp);
-
-            Thread.sleep(1000);
-
-            jvppCallbackFacade.wantInterfaceEvents(NotificationUtils.getDisableInterfaceNotificationsReq(),
-                new WantInterfaceEventsCallback() {
-                    @Override
-                    public void onWantInterfaceEventsReply(final WantInterfaceEventsReply reply) {
-                        System.out.println("Interface events stopped");
-                    }
-
-                    @Override
-                    public void onError(final VppCallbackException ex) {
-                        System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n",
-                            ex.getMethodName(), ex.getCtxId(), ex.getErrorCode());
-                    }
-                });
-
-            notificationListenerReg.close();
-
-            Thread.sleep(2000);
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    public static void main(String[] args) throws Exception {
-        testCallbackFacade();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java
deleted file mode 100644
index 9f7cb8de..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackJVppFacadeTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.VppCallbackException;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.callback.GetNodeIndexCallback;
-import io.fd.vpp.jvpp.core.callback.ShowVersionCallback;
-import io.fd.vpp.jvpp.core.callfacade.CallbackJVppCoreFacade;
-import io.fd.vpp.jvpp.core.dto.GetNodeIndex;
-import io.fd.vpp.jvpp.core.dto.GetNodeIndexReply;
-import io.fd.vpp.jvpp.core.dto.ShowVersionReply;
-
-/**
- * CallbackJVppFacade together with CallbackJVppFacadeCallback allow for setting different callback for each request.
- * This is more convenient than the approach shown in CallbackApiTest.
- */
-public class CallbackJVppFacadeTest {
-
-    private static ShowVersionCallback showVersionCallback1 = new ShowVersionCallback() {
-        @Override
-        public void onShowVersionReply(final ShowVersionReply msg) {
-            System.out.printf("ShowVersionCallback1 received ShowVersionReply: context=%d, program=%s,"
-                    + "version=%s, buildDate=%s, buildDirectory=%s%n", msg.context, new String(msg.program),
-                new String(msg.version), new String(msg.buildDate), new String(msg.buildDirectory));
-        }
-
-        @Override
-        public void onError(VppCallbackException ex) {
-            System.out.printf("Received onError exception in showVersionCallback1: call=%s, reply=%d, context=%d%n",
-                ex.getMethodName(), ex.getErrorCode(), ex.getCtxId());
-        }
-    };
-
-    private static ShowVersionCallback showVersionCallback2 = new ShowVersionCallback() {
-        @Override
-        public void onShowVersionReply(final ShowVersionReply msg) {
-            System.out.printf("ShowVersionCallback2 received ShowVersionReply: context=%d, program=%s,"
-                    + "version=%s, buildDate=%s, buildDirectory=%s%n", msg.context, new String(msg.program),
-                new String(msg.version), new String(msg.buildDate), new String(msg.buildDirectory));
-        }
-
-        @Override
-        public void onError(VppCallbackException ex) {
-            System.out.printf("Received onError exception in showVersionCallback2: call=%s, reply=%d, context=%d%n",
-                ex.getMethodName(), ex.getErrorCode(), ex.getCtxId());
-        }
-
-    };
-
-    private static GetNodeIndexCallback getNodeIndexCallback = new GetNodeIndexCallback() {
-        @Override
-        public void onGetNodeIndexReply(final GetNodeIndexReply msg) {
-            System.out.printf("Received GetNodeIndexReply: %s%n", msg);
-        }
-
-        @Override
-        public void onError(VppCallbackException ex) {
-            System.out.printf("Received onError exception in getNodeIndexCallback: call=%s, reply=%d, context=%d%n",
-                ex.getMethodName(), ex.getErrorCode(), ex.getCtxId());
-        }
-    };
-
-    private static void testCallbackFacade() throws Exception {
-        System.out.println("Testing CallbackJVppFacade");
-
-        try (final JVppRegistry registry = new JVppRegistryImpl("CallbackFacadeTest");
-             final CallbackJVppCoreFacade callbackFacade = new CallbackJVppCoreFacade(registry, new JVppCoreImpl())) {
-            System.out.println("Successfully connected to VPP");
-
-            callbackFacade.showVersion(showVersionCallback1);
-            callbackFacade.showVersion(showVersionCallback2);
-
-            GetNodeIndex getNodeIndexRequest = new GetNodeIndex();
-            getNodeIndexRequest.nodeName = "dummyNode".getBytes();
-            callbackFacade.getNodeIndex(getNodeIndexRequest, getNodeIndexCallback);
-
-            Thread.sleep(2000);
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    public static void main(String[] args) throws Exception {
-        testCallbackFacade();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java
deleted file mode 100644
index a9f71f11..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CallbackNotificationApiTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.getChangeInterfaceState;
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.getDisableInterfaceNotificationsReq;
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.getEnableInterfaceNotificationsReq;
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.printNotification;
-
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.VppCallbackException;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.callback.SwInterfaceSetFlagsCallback;
-import io.fd.vpp.jvpp.core.callback.SwInterfaceSetFlagsNotificationCallback;
-import io.fd.vpp.jvpp.core.callback.WantInterfaceEventsCallback;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlagsNotification;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlagsReply;
-import io.fd.vpp.jvpp.core.dto.WantInterfaceEventsReply;
-
-public class CallbackNotificationApiTest {
-
-    private static void testCallbackApi() throws Exception {
-        System.out.println("Testing Java callback API for notifications");
-        try (final JVppRegistry registry = new JVppRegistryImpl("CallbackNotificationTest");
-             final JVpp jvpp = new JVppCoreImpl()) {
-            registry.register(jvpp, new TestCallback());
-            System.out.println("Successfully connected to VPP");
-
-            getEnableInterfaceNotificationsReq().send(jvpp);
-            System.out.println("Interface notifications started");
-            // TODO test ifc dump which also triggers interface flags send
-
-            System.out.println("Changing interface configuration");
-            getChangeInterfaceState().send(jvpp);
-
-            // Notifications are received
-            Thread.sleep(500);
-
-            getDisableInterfaceNotificationsReq().send(jvpp);
-            System.out.println("Interface events stopped");
-
-            Thread.sleep(2000);
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    public static void main(String[] args) throws Exception {
-        testCallbackApi();
-    }
-
-    private static class TestCallback implements SwInterfaceSetFlagsNotificationCallback,
-        WantInterfaceEventsCallback, SwInterfaceSetFlagsCallback {
-
-        @Override
-        public void onSwInterfaceSetFlagsNotification(
-            final SwInterfaceSetFlagsNotification msg) {
-            printNotification(msg);
-        }
-
-        @Override
-        public void onWantInterfaceEventsReply(final WantInterfaceEventsReply wantInterfaceEventsReply) {
-            System.out.println("Interface notification stream updated");
-        }
-
-        @Override
-        public void onSwInterfaceSetFlagsReply(final SwInterfaceSetFlagsReply swInterfaceSetFlagsReply) {
-            System.out.println("Interface flags set successfully");
-        }
-
-        @Override
-        public void onError(VppCallbackException ex) {
-            System.out.printf("Received onError exception in getNodeIndexCallback: call=%s, reply=%d, context=%d%n",
-                ex.getMethodName(), ex.getErrorCode(), ex.getCtxId());
-
-        }
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java
deleted file mode 100644
index e97f4e3a..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/ControlPingTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.VppCallbackException;
-import io.fd.vpp.jvpp.callback.ControlPingCallback;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.dto.ControlPing;
-import io.fd.vpp.jvpp.dto.ControlPingReply;
-
-public class ControlPingTest {
-
-    private static void testControlPing() throws Exception {
-        System.out.println("Testing ControlPing using Java callback API");
-        try (JVppRegistry registry = new JVppRegistryImpl("ControlPingTest");
-             JVpp jvpp = new JVppCoreImpl()) {
-
-            registry.register(jvpp, new ControlPingCallback() {
-                @Override
-                public void onControlPingReply(final ControlPingReply reply) {
-                    System.out.printf("Received ControlPingReply: %s%n", reply);
-                }
-
-                @Override
-                public void onError(VppCallbackException ex) {
-                    System.out.printf("Received onError exception: call=%s, reply=%d, context=%d ", ex.getMethodName(),
-                        ex.getErrorCode(), ex.getCtxId());
-                }
-
-            });
-            System.out.println("Successfully connected to VPP");
-            Thread.sleep(1000);
-
-            System.out.println("Sending control ping using JVppRegistry");
-            registry.controlPing(jvpp.getClass());
-
-            Thread.sleep(2000);
-
-            System.out.println("Sending control ping using JVpp plugin");
-            jvpp.send(new ControlPing());
-
-            Thread.sleep(2000);
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    public static void main(String[] args) throws Exception {
-        testControlPing();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java
deleted file mode 100644
index a96258f4..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/CreateSubInterfaceTest.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import static java.util.Objects.requireNonNull;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.dto.CreateSubif;
-import io.fd.vpp.jvpp.core.dto.CreateSubifReply;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDetailsReplyDump;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDump;
-import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
-
-/**
- * 
Tests sub-interface creation.
 Equivalent to:

- *
- * 
{@code
- * vppctl create sub GigabitEthernet0/9/0 1 dot1q 100 inner-dot1q any
- * }
- * 

- *
- * To verify invoke:

- * 
{@code
- * vpp_api_test json
- * vat# sw_interface_dump
- * }
- */
-public class CreateSubInterfaceTest {
-
-    private static SwInterfaceDump createSwInterfaceDumpRequest(final String ifaceName) {
-        SwInterfaceDump request = new SwInterfaceDump();
-        request.nameFilter = ifaceName.getBytes();
-        request.nameFilterValid = 1;
-        return request;
-    }
-
-    private static void requireSingleIface(final SwInterfaceDetailsReplyDump response, final String ifaceName) {
-        if (response.swInterfaceDetails.size() != 1) {
-            throw new IllegalStateException(
-                String.format("Expected one interface matching filter %s but was %d", ifaceName,
-                    response.swInterfaceDetails.size()));
-        }
-    }
-
-    private static CreateSubif createSubifRequest(final int swIfIndex, final int subId) {
-        CreateSubif request = new CreateSubif();
-        request.swIfIndex = swIfIndex; // super interface id
-        request.subId = subId;
-        request.noTags = 0;
-        request.oneTag = 0;
-        request.twoTags = 1;
-        request.dot1Ad = 0;
-        request.exactMatch = 1;
-        request.defaultSub = 0;
-        request.outerVlanIdAny = 0;
-        request.innerVlanIdAny = 1;
-        request.outerVlanId = 100;
-        request.innerVlanId = 0;
-        return request;
-    }
-
-    private static void print(CreateSubifReply reply) {
-        System.out.printf("CreateSubifReply: %s%n", reply);
-    }
-
-    private static void testCreateSubInterface() throws Exception {
-        System.out.println("Testing sub-interface creation using Java callback API");
-        try (final JVppRegistry registry = new JVppRegistryImpl("CreateSubInterface");
-             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
-            System.out.println("Successfully connected to VPP");
-            Thread.sleep(1000);
-
-            final String ifaceName = "GigabitEthernet0/8/0";
-
-            final SwInterfaceDetailsReplyDump swInterfaceDetails =
-                jvppFacade.swInterfaceDump(createSwInterfaceDumpRequest(ifaceName)).toCompletableFuture().get();
-
-            requireNonNull(swInterfaceDetails, "swInterfaceDump returned null");
-            requireNonNull(swInterfaceDetails.swInterfaceDetails, "swInterfaceDetails is null");
-            requireSingleIface(swInterfaceDetails, ifaceName);
-
-            final int swIfIndex = swInterfaceDetails.swInterfaceDetails.get(0).swIfIndex;
-            final int subId = 1;
-
-            final CreateSubifReply createSubifReply =
-                jvppFacade.createSubif(createSubifRequest(swIfIndex, subId)).toCompletableFuture().get();
-            print(createSubifReply);
-
-            final String subIfaceName = "GigabitEthernet0/8/0." + subId;
-            final SwInterfaceDetailsReplyDump subIface =
-                jvppFacade.swInterfaceDump(createSwInterfaceDumpRequest(subIfaceName)).toCompletableFuture().get();
-            requireNonNull(swInterfaceDetails, "swInterfaceDump returned null");
-            requireNonNull(subIface.swInterfaceDetails, "swInterfaceDump returned null");
-            requireSingleIface(swInterfaceDetails, ifaceName);
-
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    public static void main(String[] args) throws Exception {
-        testCreateSubInterface();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java
deleted file mode 100644
index 9efeae19..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiNotificationTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.getChangeInterfaceState;
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.getDisableInterfaceNotificationsReq;
-import static io.fd.vpp.jvpp.core.test.NotificationUtils.getEnableInterfaceNotificationsReq;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
-
-public class FutureApiNotificationTest {
-
-    private static void testFutureApi() throws Exception {
-        System.out.println("Testing Java future API for notifications");
-        try (final JVppRegistry registry = new JVppRegistryImpl("FutureApiNotificationTest");
-             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl());
-             final AutoCloseable notificationListenerReg =
-                 jvppFacade.getNotificationRegistry()
-                     .registerSwInterfaceSetFlagsNotificationCallback(NotificationUtils::printNotification)) {
-            System.out.println("Successfully connected to VPP");
-            jvppFacade.wantInterfaceEvents(getEnableInterfaceNotificationsReq()).toCompletableFuture().get();
-            System.out.println("Interface events started");
-
-            System.out.println("Changing interface configuration");
-            jvppFacade.swInterfaceSetFlags(getChangeInterfaceState()).toCompletableFuture().get();
-
-            Thread.sleep(1000);
-
-            jvppFacade.wantInterfaceEvents(getDisableInterfaceNotificationsReq()).toCompletableFuture().get();
-            System.out.println("Interface events stopped");
-            System.out.println("Disconnecting...");
-        }
-    }
-
-    public static void main(String[] args) throws Exception {
-        testFutureApi();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java
deleted file mode 100644
index f478bab4..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/FutureApiTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.dto.BridgeDomainDetailsReplyDump;
-import io.fd.vpp.jvpp.core.dto.BridgeDomainDump;
-import io.fd.vpp.jvpp.core.dto.GetNodeIndex;
-import io.fd.vpp.jvpp.core.dto.GetNodeIndexReply;
-import io.fd.vpp.jvpp.core.dto.ShowVersion;
-import io.fd.vpp.jvpp.core.dto.ShowVersionReply;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDetails;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDetailsReplyDump;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceDump;
-import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class FutureApiTest {
-
-    private static final Logger LOG = Logger.getLogger(FutureApiTest.class.getName());
-
-    private static void testShowVersion(final FutureJVppCoreFacade jvpp) throws Exception {
-        LOG.info("Sending ShowVersion request...");
-        final Future replyFuture = jvpp.showVersion(new ShowVersion()).toCompletableFuture();
-        final ShowVersionReply reply = replyFuture.get();
-        LOG.info(
-            String.format(
-                "Received ShowVersionReply: context=%d, program=%s, version=%s, buildDate=%s, buildDirectory=%s%n",
-                reply.context, new String(reply.program), new String(reply.version), new String(reply.buildDate),
-                new String(reply.buildDirectory)));
-    }
-
-    private static void testEmptyBridgeDomainDump(final FutureJVppCoreFacade jvpp) throws Exception {
-        LOG.info("Sending ShowVersion request...");
-        final BridgeDomainDump request = new BridgeDomainDump();
-        request.bdId = -1; // dump call
-
-        final CompletableFuture
-            replyFuture = jvpp.bridgeDomainDump(request).toCompletableFuture();
-        final BridgeDomainDetailsReplyDump reply = replyFuture.get();
-
-        if (reply == null || reply.bridgeDomainDetails == null) {
-            LOG.severe("Received null response for empty dump: " + reply);
-        } else {
-            LOG.info(
-                String.format(
-                    "Received empty bridge-domain dump reply with list of bridge-domains: %s, %s",
-                    reply.bridgeDomainDetails, reply.bridgeDomainSwIfDetails));
-        }
-    }
-
-    private static void testGetNodeIndex(final FutureJVppCoreFacade jvpp) {
-        LOG.info("Sending GetNodeIndex request...");
-        final GetNodeIndex request = new GetNodeIndex();
-        request.nodeName = "non-existing-node".getBytes();
-        final Future replyFuture = jvpp.getNodeIndex(request).toCompletableFuture();
-        try {
-            final GetNodeIndexReply reply = replyFuture.get();
-            LOG.info(
-                String.format(
-                    "Received GetNodeIndexReply: context=%d, nodeIndex=%d%n", reply.context, reply.nodeIndex));
-        } catch (Exception e) {
-            LOG.log(Level.SEVERE, "GetNodeIndex request failed", e);
-        }
-    }
-
-    private static void testSwInterfaceDump(final FutureJVppCoreFacade jvpp) throws Exception {
-        LOG.info("Sending SwInterfaceDump request...");
-        final SwInterfaceDump request = new SwInterfaceDump();
-        request.nameFilterValid = 0;
-        request.nameFilter = "".getBytes();
-
-        final Future replyFuture = jvpp.swInterfaceDump(request).toCompletableFuture();
-        final SwInterfaceDetailsReplyDump reply = replyFuture.get();
-        for (SwInterfaceDetails details : reply.swInterfaceDetails) {
-            Objects.requireNonNull(details, "reply.swInterfaceDetails contains null element!");
-            LOG.info(
-                String.format("Received SwInterfaceDetails: interfaceName=%s, l2AddressLength=%d, adminUpDown=%d, "
-                        + "linkUpDown=%d, linkSpeed=%d, linkMtu=%d%n",
-                    new String(details.interfaceName), details.l2AddressLength, details.adminUpDown,
-                    details.linkUpDown, details.linkSpeed, (int) details.linkMtu));
-        }
-    }
-
-    private static void testFutureApi() throws Exception {
-        LOG.info("Testing Java future API");
-        try (final JVppRegistry registry = new JVppRegistryImpl("FutureApiTest");
-             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
-            LOG.info("Successfully connected to VPP");
-
-            testEmptyBridgeDomainDump(jvppFacade);
-            testShowVersion(jvppFacade);
-            testGetNodeIndex(jvppFacade);
-            testSwInterfaceDump(jvppFacade);
-
-            LOG.info("Disconnecting...");
-        }
-    }
-
-    public static void main(String[] args) throws Exception {
-        testFutureApi();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java
deleted file mode 100644
index 6b3fa993..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/L2AclTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.dto.ClassifyAddDelSession;
-import io.fd.vpp.jvpp.core.dto.ClassifyAddDelSessionReply;
-import io.fd.vpp.jvpp.core.dto.ClassifyAddDelTable;
-import io.fd.vpp.jvpp.core.dto.ClassifyAddDelTableReply;
-import io.fd.vpp.jvpp.core.dto.ClassifySessionDetailsReplyDump;
-import io.fd.vpp.jvpp.core.dto.ClassifySessionDump;
-import io.fd.vpp.jvpp.core.dto.ClassifyTableByInterface;
-import io.fd.vpp.jvpp.core.dto.ClassifyTableByInterfaceReply;
-import io.fd.vpp.jvpp.core.dto.ClassifyTableIds;
-import io.fd.vpp.jvpp.core.dto.ClassifyTableIdsReply;
-import io.fd.vpp.jvpp.core.dto.ClassifyTableInfo;
-import io.fd.vpp.jvpp.core.dto.ClassifyTableInfoReply;
-import io.fd.vpp.jvpp.core.dto.InputAclSetInterface;
-import io.fd.vpp.jvpp.core.dto.InputAclSetInterfaceReply;
-import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
-import javax.xml.bind.DatatypeConverter;
-
-/**
- * 
Tests L2 ACL creation and read.
 Equivalent to the following vppctl commands:

- *
- * 
{@code
- * vppctl classify table mask l2 src
- * vppctl classify session acl-hit-next deny opaque-index 0 table-index 0 match l2 src 01:02:03:04:05:06
- * vppctl set int input acl intfc local0 l2-table 0
- * vppctl sh class table verbose
- * }
- * 

- */
-public class L2AclTest {
-
-    private static final int LOCAL0_IFACE_ID = 0;
-
-    private static ClassifyAddDelTable createClassifyTable() {
-        ClassifyAddDelTable request = new ClassifyAddDelTable();
-        request.isAdd = 1;
-        request.tableIndex = ~0; // default
-        request.nbuckets = 2;
-        request.memorySize = 2 << 20;
-        request.nextTableIndex = ~0; // default
-        request.missNextIndex = ~0; // default
-        request.skipNVectors = 0;
-        request.matchNVectors = 1;
-        request.mask =
-            new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
-                (byte) 0xff, (byte) 0xff, 0x00, 0x00, 0x00, 0x00};
-        return request;
-    }
-
-    private static ClassifyTableInfo createClassifyTableInfoRequest(final int tableId) {
-        ClassifyTableInfo request = new ClassifyTableInfo();
-        request.tableId = tableId;
-        return request;
-    }
-
-    private static ClassifyAddDelSession createClassifySession(final int tableIndex) {
-        ClassifyAddDelSession request = new ClassifyAddDelSession();
-        request.isAdd = 1;
-        request.tableIndex = tableIndex;
-        request.hitNextIndex = 0; // deny
-        request.opaqueIndex = 0;
-        request.advance = 0; // default
-        // match 01:02:03:04:05:06 mac address
-        request.match =
-            new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04,
-                (byte) 0x05, (byte) 0x06, 0x00, 0x00, 0x00, 0x00};
-        return request;
-    }
-
-    private static ClassifySessionDump createClassifySessionDumpRequest(final int newTableIndex) {
-        ClassifySessionDump request = new ClassifySessionDump();
-        request.tableId = newTableIndex;
-        return request;
-    }
-
-    private static InputAclSetInterface aclSetInterface() {
-        InputAclSetInterface request = new InputAclSetInterface();
-        request.isAdd = 1;
-        request.swIfIndex = LOCAL0_IFACE_ID;
-        request.ip4TableIndex = ~0; // skip
-        request.ip6TableIndex = ~0; // skip
-        request.l2TableIndex = 0;
-        return request;
-    }
-
-    private static ClassifyTableByInterface createClassifyTableByInterfaceRequest() {
-        ClassifyTableByInterface request = new ClassifyTableByInterface();
-        request.swIfIndex = LOCAL0_IFACE_ID;
-        return request;
-    }
-
-    private static void print(ClassifyAddDelTableReply reply) {
-        System.out.printf("ClassifyAddDelTableReply: %s%n", reply);
-    }
-
-    private static void print(ClassifyTableIdsReply reply) {
-        System.out.printf("ClassifyTableIdsReply: %s%n", reply);
-    }
-
-    private static void print(final ClassifyTableInfoReply reply) {
-        System.out.println(reply);
-        if (reply != null) {
-            System.out.println("Mask hex: " + DatatypeConverter.printHexBinary(reply.mask));
-        }
-    }
-
-    private static void print(ClassifyAddDelSessionReply reply) {
-        System.out.printf("ClassifyAddDelSessionReply: context=%s%n", reply);
-    }
-
-    private static void print(final ClassifySessionDetailsReplyDump reply) {
-        System.out.println(reply);
-        reply.classifySessionDetails.forEach(detail -> {
-            System.out.println(detail);
-            System.out.println("Match hex: " + DatatypeConverter.printHexBinary(detail.match));
-        });
-    }
-
-    private static void print(final InputAclSetInterfaceReply reply) {
-        System.out.printf("InputAclSetInterfaceReply: context=%s%n", reply);
-    }
-
-    private static void print(final ClassifyTableByInterfaceReply reply) {
-        System.out.printf("ClassifyAddDelTableReply: %s%n", reply);
-    }
-
-    private static void testL2Acl() throws Exception {
-        System.out.println("Testing L2 ACLs using Java callback API");
-        try (final JVppRegistry registry = new JVppRegistryImpl("L2AclTest");
-             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
-
-            System.out.println("Successfully connected to VPP");
-            Thread.sleep(1000);
-
-            final ClassifyAddDelTableReply classifyAddDelTableReply =
-                jvppFacade.classifyAddDelTable(createClassifyTable()).toCompletableFuture().get();
-            print(classifyAddDelTableReply);
-
-            final ClassifyTableIdsReply classifyTableIdsReply =
-                jvppFacade.classifyTableIds(new ClassifyTableIds()).toCompletableFuture().get();
-            print(classifyTableIdsReply);
-
-            final ClassifyTableInfoReply classifyTableInfoReply =
-                jvppFacade.classifyTableInfo(createClassifyTableInfoRequest(classifyAddDelTableReply.newTableIndex))
-                    .toCompletableFuture().get();
-            print(classifyTableInfoReply);
-
-            final ClassifyAddDelSessionReply classifyAddDelSessionReply =
-                jvppFacade.classifyAddDelSession(createClassifySession(classifyAddDelTableReply.newTableIndex))
-                    .toCompletableFuture().get();
-            print(classifyAddDelSessionReply);
-
-            final ClassifySessionDetailsReplyDump classifySessionDetailsReplyDump =
-                jvppFacade.classifySessionDump(createClassifySessionDumpRequest(classifyAddDelTableReply.newTableIndex))
-                    .toCompletableFuture().get();
-            print(classifySessionDetailsReplyDump);
-
-            final InputAclSetInterfaceReply inputAclSetInterfaceReply =
-                jvppFacade.inputAclSetInterface(aclSetInterface()).toCompletableFuture().get();
-            print(inputAclSetInterfaceReply);
-
-            final ClassifyTableByInterfaceReply classifyTableByInterfaceReply =
-                jvppFacade.classifyTableByInterface(createClassifyTableByInterfaceRequest()).toCompletableFuture()
-                    .get();
-            print(classifyTableByInterfaceReply);
-
-            System.out.println("Disconnecting...");
-        }
-        Thread.sleep(1000);
-    }
-
-    public static void main(String[] args) throws Exception {
-        testL2Acl();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java
deleted file mode 100644
index d7f5039b..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/LispAdjacencyTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.core.JVppCoreImpl;
-import io.fd.vpp.jvpp.core.dto.LispAddDelAdjacency;
-import io.fd.vpp.jvpp.core.dto.LispAddDelLocalEid;
-import io.fd.vpp.jvpp.core.dto.LispAddDelLocatorSet;
-import io.fd.vpp.jvpp.core.dto.LispAddDelRemoteMapping;
-import io.fd.vpp.jvpp.core.dto.LispAdjacenciesGet;
-import io.fd.vpp.jvpp.core.dto.LispAdjacenciesGetReply;
-import io.fd.vpp.jvpp.core.dto.LispEnableDisable;
-import io.fd.vpp.jvpp.core.future.FutureJVppCoreFacade;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.ExecutionException;
-import java.util.logging.Logger;
-
-/**
- * Tests lisp adjacency creation and read (custom vpe.api type support showcase).
- */
-public class LispAdjacencyTest {
-
-    private static final Logger LOG = Logger.getLogger(LispAdjacencyTest.class.getName());
-
-    private static void enableLisp(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
-        final LispEnableDisable request = new LispEnableDisable();
-        request.isEn = 1;
-        jvpp.lispEnableDisable(request).toCompletableFuture().get();
-        LOG.info("Lisp enabled successfully");
-    }
-
-    private static void addLocatorSet(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
-        final LispAddDelLocatorSet request = new LispAddDelLocatorSet();
-        request.isAdd = 1;
-        request.locatorSetName = "ls1".getBytes(StandardCharsets.UTF_8);
-        jvpp.lispAddDelLocatorSet(request).toCompletableFuture().get();
-        LOG.info("Locator set created successfully:" + request.toString());
-    }
-
-    private static void addLocalEid(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
-        final LispAddDelLocalEid request = new LispAddDelLocalEid();
-        request.isAdd = 1;
-        request.locatorSetName = "ls1".getBytes(StandardCharsets.UTF_8);
-        request.eid = new byte[] {1, 2, 1, 10};
-        request.eidType = 0; // ip4
-        request.vni = 0;
-        request.prefixLen = 32;
-        jvpp.lispAddDelLocalEid(request).toCompletableFuture().get();
-        LOG.info("Local EID created successfully:" + request.toString());
-    }
-
-    private static void addRemoteMapping(final FutureJVppCoreFacade jvpp)
-        throws ExecutionException, InterruptedException {
-        final LispAddDelRemoteMapping request = new LispAddDelRemoteMapping();
-        request.isAdd = 1;
-        request.vni = 0;
-        request.eid = new byte[] {1, 2, 1, 20};
-        request.eidLen = 32;
-        request.rlocNum = 1;
-        request.rlocs = new byte[] {1, 1, 1, 1, 2, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-        jvpp.lispAddDelRemoteMapping(request).toCompletableFuture().get();
-        LOG.info("Remote mapping created successfully:" + request.toString());
-    }
-
-    private static void addAdjacency(final FutureJVppCoreFacade jvpp) throws ExecutionException, InterruptedException {
-        final LispAddDelAdjacency request = new LispAddDelAdjacency();
-        request.isAdd = 1;
-        request.leid = new byte[] {1, 2, 1, 10};
-        request.leidLen = 32;
-        request.reid = new byte[] {1, 2, 1, 20};
-        request.reidLen = 32;
-        request.eidType = 0; // ip4
-        request.vni = 0;
-        jvpp.lispAddDelAdjacency(request).toCompletableFuture().get();
-        LOG.info("Lisp adjacency created successfully:" + request.toString());
-    }
-
-    private static void showAdjacencies(final FutureJVppCoreFacade jvpp)
-        throws ExecutionException, InterruptedException {
-        final LispAdjacenciesGetReply reply =
-            jvpp.lispAdjacenciesGet(new LispAdjacenciesGet()).toCompletableFuture().get();
-        LOG.info("Lisp adjacency received successfully:" + reply.toString());
-    }
-
-    private static void testAdjacency(final FutureJVppCoreFacade jvpp) throws Exception {
-        enableLisp(jvpp);
-        addLocatorSet(jvpp);
-        addLocalEid(jvpp);
-        addRemoteMapping(jvpp);
-        addAdjacency(jvpp);
-        showAdjacencies(jvpp);
-    }
-
-    private static void testFutureApi() throws Exception {
-        LOG.info("Create lisp adjacency test");
-        try (final JVppRegistry registry = new JVppRegistryImpl("LispAdjacencyTest");
-             final FutureJVppCoreFacade jvppFacade = new FutureJVppCoreFacade(registry, new JVppCoreImpl())) {
-            LOG.info("Successfully connected to VPP");
-
-            testAdjacency(jvppFacade);
-            LOG.info("Disconnecting...");
-        }
-    }
-
-    public static void main(String[] args) throws Exception {
-        testFutureApi();
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java
deleted file mode 100644
index f82946c3..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/NotificationUtils.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.core.test;
-
-import java.io.PrintStream;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlags;
-import io.fd.vpp.jvpp.core.dto.SwInterfaceSetFlagsNotification;
-import io.fd.vpp.jvpp.core.dto.WantInterfaceEvents;
-
-final class NotificationUtils {
-
-    private NotificationUtils() {}
-
-    static PrintStream printNotification(final SwInterfaceSetFlagsNotification msg) {
-        return System.out.printf("Received interface notification: ifc: %s%n", msg);
-    }
-
-    static SwInterfaceSetFlags getChangeInterfaceState() {
-        final SwInterfaceSetFlags swInterfaceSetFlags = new SwInterfaceSetFlags();
-        swInterfaceSetFlags.swIfIndex = 0;
-        swInterfaceSetFlags.adminUpDown = 1;
-        swInterfaceSetFlags.deleted = 0;
-        return swInterfaceSetFlags;
-    }
-
-    static WantInterfaceEvents getEnableInterfaceNotificationsReq() {
-        WantInterfaceEvents wantInterfaceEvents = new WantInterfaceEvents();
-        wantInterfaceEvents.pid = 1;
-        wantInterfaceEvents.enableDisable = 1;
-        return wantInterfaceEvents;
-    }
-
-    static WantInterfaceEvents getDisableInterfaceNotificationsReq() {
-        WantInterfaceEvents wantInterfaceEvents = new WantInterfaceEvents();
-        wantInterfaceEvents.pid = 1;
-        wantInterfaceEvents.enableDisable = 0;
-        return wantInterfaceEvents;
-    }
-}
diff --git a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt b/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt
deleted file mode 100644
index 1344dc9e..00000000
--- a/vpp-api/java/jvpp-core/io/fd/vpp/jvpp/core/test/Readme.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-This package contains basic tests for jvpp. To run the tests:
-
-- Make sure VPP is running
-- From VPP's build-root/ folder execute:
-  - sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-17.01.jar:build-vpp-native/vpp-api/java/jvpp-core-17.01.jar io.fd.vpp.jvpp.core.test.[test name]
-
-Available tests:
-CallbackApiTest - Similar to ControlPingTest, invokes more complex calls (e.g. interface dump) using low level JVpp APIs
-CallbackJVppFacadeNotificationTest - Tests interface notifications using Callback based JVpp facade
-CallbackJVppFacadeTest - Execution of more complex calls using Callback based JVpp facade
-CallbackNotificationApiTest - Tests interface notifications using low level JVpp APIs
-ControlPingTest - Simple test executing a single control ping using low level JVpp APIs
-CreateSubInterfaceTest - Tests sub-interface creation
-FutureApiNotificationTest - Tests interface notifications using Future based JVpp facade
-FutureApiTest - Execution of more complex calls using Future based JVpp facade
-L2AclTest - Tests L2 ACL creation
-LispAdjacencyTest - Tests lisp adjacency creation and read (custom vpe.api type support showcase)
diff --git a/vpp-api/java/jvpp-core/jvpp_core.c b/vpp-api/java/jvpp-core/jvpp_core.c
deleted file mode 100644
index ef4cb8e3..00000000
--- a/vpp-api/java/jvpp-core/jvpp_core.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-
-#include 
-#define vl_typedefs             /* define message structures */
-#include 
-#undef vl_typedefs
-
-#define vl_endianfun
-#include 
-#undef vl_endianfun
-
-#define vl_print(handle, ...)
-#define vl_printfun
-#include 
-#undef vl_printfun
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-
-// TODO: generate jvpp_plugin_name.c files (or at least reuse plugin's main structure)
-typedef struct {
-    /* Base message index for the jvpp-core plugin */
-    u16 msg_id_base;
-
-    /* Pointer to shared memory queue */
-    unix_shared_memory_queue_t * vl_input_queue;
-
-    /* VPP api client index */
-    u32 my_client_index;
-
-    /* Callback object and class references enabling asynchronous Java calls */
-    jobject callbackObject;
-    jclass callbackClass;
-
-} core_main_t;
-
-core_main_t core_main __attribute__((aligned (64)));
-
-#include "io_fd_vpp_jvpp_core_JVppCoreImpl.h"
-#include "jvpp_core_gen.h"
-
-JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_core_JVppCoreImpl_init0
-(JNIEnv * env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
-    core_main_t * plugin_main = &core_main;
-    plugin_main->my_client_index = my_client_index;
-    plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
-
-    plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
-    plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
-
-    #define _(N,n)                                  \
-        vl_msg_api_set_handlers(VL_API_##N, #n,     \
-                vl_api_##n##_t_handler,             \
-                vl_noop_handler,                    \
-                vl_noop_handler,              \
-                vl_noop_handler,               \
-                sizeof(vl_api_##n##_t), 1);
-        foreach_api_reply_handler;
-    #undef _
-}
-
-JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_core_JVppCoreImpl_close0
-(JNIEnv *env, jclass clazz) {
-    core_main_t * plugin_main = &core_main;
-
-    // cleanup:
-    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
-    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
-
-    plugin_main->callbackClass = NULL;
-    plugin_main->callbackObject = NULL;
-}
-
-jint JNI_OnLoad(JavaVM *vm, void *reserved) {
-    JNIEnv* env;
-
-    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
-        return JNI_EVERSION;
-    }
-
-    if (cache_class_references(env) != 0) {
-        clib_warning ("Failed to cache class references\n");
-        return JNI_ERR;
-    }
-
-    return JNI_VERSION_1_8;
-}
-
-void JNI_OnUnload(JavaVM *vm, void *reserved) {
-    JNIEnv* env;
-    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
-        return;
-    }
-    delete_class_references(env);
-}
-
-
-
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java
deleted file mode 100644
index 55f25a7b..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVpp.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-import io.fd.vpp.jvpp.callback.JVppCallback;
-import io.fd.vpp.jvpp.dto.ControlPing;
-import io.fd.vpp.jvpp.dto.JVppRequest;
-
-/**
- * Base interface for plugin's Java API.
- */
-public interface JVpp extends AutoCloseable {
-
-    /**
-     * Sends request to vpp.
-     *
-     * @param request request to be sent
-     * @return unique identifer of message in message queue
-     * @throws VppInvocationException when message could not be sent
-     */
-    int send(final JVppRequest request) throws VppInvocationException;
-
-    /**
-     * Initializes plugin's Java API.
-     *
-     * @param registry     plugin registry
-     * @param callback     called by vpe.api message handlers
-     * @param queueAddress address of vpp shared memory queue
-     * @param clientIndex  vpp client identifier
-     */
-    void init(final JVppRegistry registry, final JVppCallback callback, final long queueAddress,
-              final int clientIndex);
-
-    /**
-     * Sends control_ping message.
-     *
-     * @param controlPing request DTO
-     * @return unique identifer of message in message queue
-     * @throws VppInvocationException when message could not be sent
-     */
-    int controlPing(final ControlPing controlPing) throws VppInvocationException;
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java
deleted file mode 100644
index 6535db02..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistry.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-import io.fd.vpp.jvpp.callback.JVppCallback;
-
-/**
- * Manages VPP connection and stores plugin callbacks.
- */
-public interface JVppRegistry extends AutoCloseable {
-
-    /**
-     * Vpp connection managed by the registry.
-     *
-     * @return representation of vpp connection
-     */
-    VppConnection getConnection();
-
-    /**
-     * Registers callback and initializes Java API for given plugin.
-     *
-     * @param jvpp     plugin name
-     * @param callback callback provided by the plugin
-     * @throws NullPointerException     if name or callback is null
-     * @throws IllegalArgumentException if plugin was already registered
-     */
-    void register(final JVpp jvpp, final JVppCallback callback);
-
-    /**
-     * Unregisters callback for the given plugin.
-     *
-     * @param name plugin name
-     * @throws NullPointerException     if name is null
-     * @throws IllegalArgumentException if plugin was not registered
-     */
-    void unregister(final String name);
-
-    /**
-     * Returns callback registered for the plugin.
-     *
-     * @param name plugin name
-     * @return callback provided by the plugin
-     * @throws NullPointerException     if name is null
-     * @throws IllegalArgumentException if plugin was not registered
-     */
-    JVppCallback get(final String name);
-
-    /**
-     * Sends control ping. Reply handler calls callback registered for give plugin.
-     *
-     * Control ping is used for initial RX thread to Java thread attachment
-     * that takes place in the plugin's JNI lib
-     * and to wrap dump message replies in one list.
-     *
-     * VPP plugins don't have to provide special control ping, therefore
-     * it is necessary to providing control ping support in JVppRegistry.
-
-     * @param clazz identifies plugin that should receive ping callback
-     * @return unique identifier of message in message queue
-     */
-    int controlPing(final Class clazz) throws VppInvocationException;
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java
deleted file mode 100644
index 98ef1c15..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/JVppRegistryImpl.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-import static java.util.Objects.requireNonNull;
-
-import io.fd.vpp.jvpp.callback.ControlPingCallback;
-import io.fd.vpp.jvpp.callback.JVppCallback;
-import io.fd.vpp.jvpp.dto.ControlPingReply;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Default implementation of JVppRegistry.
- */
-public final class JVppRegistryImpl implements JVppRegistry, ControlPingCallback {
-
-    private static final Logger LOG = Logger.getLogger(JVppRegistryImpl.class.getName());
-
-    private final VppJNIConnection connection;
-    // Unguarded concurrent map, no race conditions expected on top of that
-    private final Map pluginRegistry;
-    // Guarded by self
-    private final Map pingCalls;
-
-    public JVppRegistryImpl(final String clientName) throws IOException {
-        connection = new VppJNIConnection(clientName);
-        connection.connect();
-        pluginRegistry = new ConcurrentHashMap<>();
-        pingCalls = new HashMap<>();
-    }
-
-    @Override
-    public VppConnection getConnection() {
-        return connection;
-    }
-
-    @Override
-    public void register(final JVpp jvpp, final JVppCallback callback) {
-        requireNonNull(jvpp, "jvpp should not be null");
-        requireNonNull(callback, "Callback should not be null");
-        final String name = jvpp.getClass().getName();
-        if (pluginRegistry.containsKey(name)) {
-            throw new IllegalArgumentException(
-                String.format("Callback for plugin %s was already registered", name));
-        }
-        jvpp.init(this, callback, connection.getConnectionInfo().queueAddress,
-            connection.getConnectionInfo().clientIndex);
-        pluginRegistry.put(name, callback);
-    }
-
-    @Override
-    public void unregister(final String name) {
-        requireNonNull(name, "Plugin name should not be null");
-        final JVppCallback previous = pluginRegistry.remove(name);
-        assertPluginWasRegistered(name, previous);
-    }
-
-    @Override
-    public JVppCallback get(final String name) {
-        requireNonNull(name, "Plugin name should not be null");
-        JVppCallback value = pluginRegistry.get(name);
-        assertPluginWasRegistered(name, value);
-        return value;
-    }
-
-    private native int controlPing0() throws VppInvocationException;
-
-    @Override
-    public int controlPing(final Class clazz) throws VppInvocationException {
-        connection.checkActive();
-        final String name = clazz.getName();
-
-        final ControlPingCallback callback = (ControlPingCallback) pluginRegistry.get(clazz.getName());
-        assertPluginWasRegistered(name, callback);
-
-        synchronized (pingCalls) {
-            int context = controlPing0();
-            if (context < 0) {
-                throw new VppInvocationException("controlPing", context);
-            }
-
-            pingCalls.put(context, callback);
-            return context;
-        }
-    }
-
-    @Override
-    public void onControlPingReply(final ControlPingReply reply) {
-        final ControlPingCallback callback;
-        synchronized (pingCalls) {
-            callback = pingCalls.remove(reply.context);
-            if (callback == null) {
-                LOG.log(Level.WARNING, "No callback was registered for reply context=" + reply.context + " Contexts waiting="
-                    + pingCalls.keySet());
-                return;
-            }
-        }
-        // pass the reply to the callback registered by the ping caller
-        callback.onControlPingReply(reply);
-    }
-
-    @Override
-    public void onError(final VppCallbackException ex) {
-        final int ctxId = ex.getCtxId();
-        final ControlPingCallback callback;
-
-        synchronized (pingCalls) {
-            callback = pingCalls.get(ctxId);
-        }
-        if (callback == null) {
-            LOG.log(Level.WARNING, "No callback was registered for reply id={0} ", ctxId);
-            return;
-        }
-        // pass the error to the callback registered by the ping caller
-        callback.onError(ex);
-    }
-
-    private static void assertPluginWasRegistered(final String name, final JVppCallback value) {
-        if (value == null) {
-            throw new IllegalArgumentException(String.format("Callback for plugin %s is not registered", name));
-        }
-    }
-
-    @Override
-    public void close() throws Exception {
-        connection.close();
-    }
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java
deleted file mode 100644
index ce6d1bfc..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/NativeLibraryLoader.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Utility class for loading JNI libraries.
- */
-public final class NativeLibraryLoader {
-
-    private static final Logger LOG = Logger.getLogger(NativeLibraryLoader.class.getName());
-
-    private NativeLibraryLoader() {
-        throw new UnsupportedOperationException("This utility class cannot be instantiated.");
-    }
-
-    /**
-     * Loads JNI library using class loader of the given class.
-     *
-     * @param libName name of the library to be loaded
-     */
-    public static void loadLibrary(final String libName, final Class clazz) throws IOException {
-        java.util.Objects.requireNonNull(libName, "libName should not be null");
-        java.util.Objects.requireNonNull(clazz, "clazz should not be null");
-        try (final InputStream is = clazz.getResourceAsStream('/' + libName)) {
-            if (is == null) {
-                throw new IOException("Failed to open library resource " + libName);
-            }
-            loadStream(libName, is);
-        }
-    }
-
-    private static void loadStream(final String libName, final InputStream is) throws IOException {
-        final Set perms = PosixFilePermissions.fromString("rwxr-x---");
-        final Path p = Files.createTempFile(libName, null, PosixFilePermissions.asFileAttribute(perms));
-        try {
-            Files.copy(is, p, StandardCopyOption.REPLACE_EXISTING);
-            Runtime.getRuntime().load(p.toString());
-        } catch (Exception e) {
-            throw new IOException("Failed to load library " + p, e);
-        } finally {
-            try {
-                Files.deleteIfExists(p);
-            } catch (IOException e) {
-                LOG.log(Level.WARNING, String.format("Failed to delete temporary file %s.", p), e);
-            }
-        }
-    }
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java
deleted file mode 100644
index d71e3055..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppBaseCallException.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-/**
- * Base exception representing failed operation of JVpp request call
- */
-public abstract class VppBaseCallException extends Exception {
-    private final String methodName;
-    private final int errorCode;
-
-    /**
-     * Constructs an VppCallbackException with the specified api method name and error code.
-     *
-     * @param methodName name of a method, which invocation or execution failed
-     * @param errorCode  negative error code value associated with this failure
-     * @throws NullPointerException     if apiMethodName is null
-     */
-    public VppBaseCallException(final String methodName, final int errorCode) {
-        super(String.format("vppApi.%s failed with error code: %d", methodName, errorCode));
-        this.methodName = java.util.Objects.requireNonNull(methodName, "apiMethodName is null!");
-        this.errorCode = errorCode;
-        if(errorCode >= 0) {
-            throw new IllegalArgumentException("Error code must be < 0. Was " + errorCode +
-                    " for " + methodName );
-        }
-    }
-
-    /**
-     * Returns  name of a method, which invocation failed.
-     *
-     * @return method name
-     */
-    public String getMethodName() {
-        return methodName;
-    }
-
-    /**
-     * Returns the error code associated with this failure.
-     *
-     * @return a negative integer error code
-     */
-    public int getErrorCode() {
-        return errorCode;
-    }
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java
deleted file mode 100644
index ccfcbd3c..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppCallbackException.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-/**
- * Callback Exception representing failed operation of JVpp request call
- */
-public class VppCallbackException extends VppBaseCallException {
-    private final int ctxId;
-
-    /**
-     * Constructs an VppCallbackException with the specified api method name and error code.
-     *
-     * @param methodName  name of a method, which invocation failed.
-     * @param ctxId      api request context identifier
-     * @param errorCode  negative error code value associated with this failure
-     * @throws NullPointerException     if apiMethodName is null
-     */
-    public VppCallbackException(final String methodName, final int ctxId, final int errorCode ){
-        super(methodName, errorCode);
-        this.ctxId = ctxId;
-    }
-
-    /**
-     * Returns api request context identifier.
-     *
-     * @return value of context identifier
-     */
-    public int getCtxId() {
-        return ctxId;
-    }
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java
deleted file mode 100644
index e6fd3bdb..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppConnection.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-import java.io.IOException;
-
-/**
- * Representation of a management connection to VPP.
- */
-public interface VppConnection extends AutoCloseable {
-
-    /**
-     * Opens VppConnection for communication with VPP.
-     *
-     * @throws IOException if connection is not established
-     */
-    void connect() throws IOException;
-
-    /**
-     * Checks if this instance connection is active.
-     *
-     * @throws IllegalStateException if this instance was disconnected.
-     */
-    void checkActive();
-
-    /**
-     * Closes Vpp connection.
-     */
-    @Override
-    void close();
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java
deleted file mode 100644
index a7ccb197..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppInvocationException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-/**
- * Exception thrown when Vpp jAPI method invocation failed.
- */
-public class VppInvocationException extends VppBaseCallException {
-    /**
-     * Constructs an VppApiInvocationFailedException with the specified api method name and error code.
-     *
-     * @param methodName  name of a method, which invocation failed.
-     * @param errorCode  negative error code value associated with this failure
-     * @throws NullPointerException     if apiMethodName is null
-     */
-    public VppInvocationException(final String methodName, final int errorCode) {
-        super(methodName, errorCode);
-    }
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java
deleted file mode 100644
index 7178bcf7..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/VppJNIConnection.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp;
-
-import static io.fd.vpp.jvpp.NativeLibraryLoader.loadLibrary;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * JNI based representation of a management connection to VPP.
- */
-public final class VppJNIConnection implements VppConnection {
-    private static final Logger LOG = Logger.getLogger(VppJNIConnection.class.getName());
-
-    static {
-        final String libName = "libjvpp_registry.so.0.0.0";
-        try {
-            loadLibrary(libName, VppJNIConnection.class);
-        } catch (IOException e) {
-            LOG.log(Level.SEVERE, String.format("Can't find vpp jni library: %s", libName), e);
-            throw new ExceptionInInitializerError(e);
-        }
-    }
-
-    private ConnectionInfo connectionInfo;
-
-    private final String clientName;
-    private volatile boolean disconnected = false;
-
-    /**
-     * Create VPPJNIConnection instance for client connecting to VPP.
-     *
-     * @param clientName client name instance to be used for communication. Single connection per clientName is
-     *                   allowed.
-     */
-    public VppJNIConnection(final String clientName) {
-        this.clientName = Objects.requireNonNull(clientName, "Null clientName");
-    }
-
-    /**
-     * Guarded by VppJNIConnection.class
-     */
-    private static final Map connections = new HashMap<>();
-
-    /**
-     * Initiate VPP connection for current instance
-     *
-     * Multiple instances are allowed since this class is not a singleton (VPP allows multiple management connections).
-     *
-     * However only a single connection per clientName is allowed.
-     *
-     * @throws IOException in case the connection could not be established
-     */
-
-    @Override
-    public void connect() throws IOException {
-        _connect();
-    }
-
-    private void _connect() throws IOException {
-        synchronized (VppJNIConnection.class) {
-            if (connections.containsKey(clientName)) {
-                throw new IOException("Client " + clientName + " already connected");
-            }
-
-            connectionInfo = clientConnect(clientName);
-            if (connectionInfo.status != 0) {
-                throw new IOException("Connection returned error " + connectionInfo.status);
-            }
-            connections.put(clientName, this);
-        }
-    }
-
-    @Override
-    public final void checkActive() {
-        if (disconnected) {
-            throw new IllegalStateException("Disconnected client " + clientName);
-        }
-    }
-
-    @Override
-    public final synchronized void close() {
-        if (!disconnected) {
-            disconnected = true;
-            try {
-                clientDisconnect();
-            } finally {
-                synchronized (VppJNIConnection.class) {
-                    connections.remove(clientName);
-                }
-            }
-        }
-    }
-
-    public ConnectionInfo getConnectionInfo() {
-        return connectionInfo;
-    }
-
-    /**
-     * VPP connection information used by plugins to reuse the connection.
-     */
-    public static final class ConnectionInfo {
-        public final long queueAddress;
-        public final int clientIndex;
-        public final int status; // FIXME throw exception instead
-
-        public ConnectionInfo(long queueAddress, int clientIndex, int status) {
-            this.queueAddress = queueAddress;
-            this.clientIndex = clientIndex;
-            this.status = status;
-        }
-    }
-
-    private static native ConnectionInfo clientConnect(String clientName);
-
-    private static native void clientDisconnect();
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java
deleted file mode 100644
index efddfdbb..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/ControlPingCallback.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.callback;
-
-import io.fd.vpp.jvpp.dto.ControlPingReply;
-
-/**
- * Represents callback for control_ping message.
- */
-public interface ControlPingCallback extends JVppCallback {
-
-    void onControlPingReply(ControlPingReply reply);
-
-}
-
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java
deleted file mode 100644
index ae02063b..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppCallback.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.callback;
-import io.fd.vpp.jvpp.VppCallbackException;
-
-/**
- * Base JVppCallback interface
- */
-public interface JVppCallback {
-    /**
-     * onError callback handler used to report failing operation
-     * @param ex VppCallbackException object containing details about failing operation
-     */
-    void onError(VppCallbackException ex);
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java
deleted file mode 100644
index 8ab0cb21..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/callback/JVppNotificationCallback.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.callback;
-
-/**
-* Notification callback
-*/
-public interface JVppNotificationCallback {
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java
deleted file mode 100644
index 984e1674..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPing.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.VppInvocationException;
-
-/**
- * Represents request DTO for control_ping message.
- */
-public final class ControlPing implements JVppRequest {
-
-    @Override
-    public int send(final JVpp jvpp) throws VppInvocationException {
-        return jvpp.controlPing(this);
-    }
-
-}
-
-
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java
deleted file mode 100644
index 61e4d0e4..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/ControlPingReply.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-import java.util.Objects;
-
-/**
- * Represents reply DTO for control_ping message.
- */
-public final class ControlPingReply implements JVppReply {
-
-    public int context;
-    public int clientIndex;
-    public int vpePid;
-
-    @Override
-    public boolean equals(final Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        final ControlPingReply that = (ControlPingReply) o;
-        return context == that.context &&
-                clientIndex == that.clientIndex &&
-                vpePid == that.vpePid;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(context, clientIndex, vpePid);
-    }
-
-    @Override
-    public String toString() {
-        return "ControlPingReply{" +
-                "context=" + context +
-                ", clientIndex=" + clientIndex +
-                ", vpePid=" + vpePid +
-                '}';
-    }
-}
-
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java
deleted file mode 100644
index 60b98984..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppDump.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-/**
-* Base interface for all dump requests
-*/
-public interface JVppDump extends JVppRequest {
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java
deleted file mode 100644
index 5554f501..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppNotification.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-/**
-* Base interface for all notification DTOs
-*/
-public interface JVppNotification {
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java
deleted file mode 100644
index 73f512d4..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReply.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-/**
-* Base interface for all reply DTOs
-*/
-public interface JVppReply {
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java
deleted file mode 100644
index 15111395..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppReplyDump.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-/**
-* Base interface for all dump replies
-*/
-public interface JVppReplyDump>
-    extends JVppReply {
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java
deleted file mode 100644
index 9b301da2..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/dto/JVppRequest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.dto;
-
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.VppInvocationException;
-
-/**
-* Base interface for all request DTOs
-*/
-public interface JVppRequest {
-
-    /**
-     * Invoke current operation asynchronously on VPP
-     *
-     * @return context id of this request. Can be used to track incoming response
-     */
-    int send(JVpp jvpp) throws VppInvocationException;
-
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java
deleted file mode 100644
index e7df528a..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/AbstractFutureJVppInvoker.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.future;
-
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.VppInvocationException;
-import io.fd.vpp.jvpp.dto.JVppDump;
-import io.fd.vpp.jvpp.dto.JVppReply;
-import io.fd.vpp.jvpp.dto.JVppReplyDump;
-import io.fd.vpp.jvpp.dto.JVppRequest;
-
-/**
- * Future facade on top of JVpp
- */
-public abstract class AbstractFutureJVppInvoker implements FutureJVppInvoker {
-
-    private final JVpp jvpp;
-    private final JVppRegistry registry;
-
-    /**
-     * Guarded by self
-     */
-    private final Map>> requests;
-
-    protected AbstractFutureJVppInvoker(final JVpp jvpp, final JVppRegistry registry,
-                                        final Map>> requestMap) {
-        this.jvpp =  Objects.requireNonNull(jvpp, "jvpp should not be null");
-        this.registry =  Objects.requireNonNull(registry, "registry should not be null");
-        // Request map represents the shared state between this facade and it's callback
-        // where facade puts futures in and callback completes + removes them
-        this.requests = Objects.requireNonNull(requestMap, "Null requestMap");
-    }
-
-    protected final Map>> getRequests() {
-        synchronized (requests) {
-            return requests;
-        }
-    }
-
-    // TODO use Optional in Future, java8
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public > CompletionStage send(REQ req) {
-        synchronized(requests) {
-            try {
-                final CompletableFuture replyCompletableFuture;
-                final int contextId = jvpp.send(req);
-
-                if(req instanceof JVppDump) {
-                    throw new IllegalArgumentException("Send with empty reply dump has to be used in case of dump calls");
-                }
-                replyCompletableFuture = new CompletableFuture<>();
-                requests.put(contextId, replyCompletableFuture);
-
-                // TODO in case of timeouts/missing replies, requests from the map are not removed
-                // consider adding cancel method, that would remove requests from the map and cancel
-                // associated replyCompletableFuture
-
-                return replyCompletableFuture;
-            } catch (VppInvocationException ex) {
-                final CompletableFuture replyCompletableFuture = new CompletableFuture<>();
-                replyCompletableFuture.completeExceptionally(ex);
-                return replyCompletableFuture;
-            }
-        }
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public , DUMP extends JVppReplyDump> CompletionStage send(
-            REQ req, DUMP emptyReplyDump) {
-        synchronized(requests) {
-            try {
-                final CompletableDumpFuture replyCompletableFuture;
-                final int contextId = jvpp.send(req);
-
-                if(!(req instanceof JVppDump)) {
-                    throw new IllegalArgumentException("Send without empty reply dump has to be used in case of regular calls");
-                }
-                replyCompletableFuture = new CompletableDumpFuture<>(contextId, emptyReplyDump);
-
-                requests.put(contextId, replyCompletableFuture);
-                requests.put(registry.controlPing(jvpp.getClass()), replyCompletableFuture);
-
-                // TODO in case of timeouts/missing replies, requests from the map are not removed
-                // consider adding cancel method, that would remove requests from the map and cancel
-                // associated replyCompletableFuture
-
-                return replyCompletableFuture;
-            } catch (VppInvocationException ex) {
-                final CompletableFuture replyCompletableFuture = new CompletableFuture<>();
-                replyCompletableFuture.completeExceptionally(ex);
-                return replyCompletableFuture;
-            }
-        }
-    }
-
-    public static final class CompletableDumpFuture> extends CompletableFuture {
-        private final T replyDump;
-        private final int contextId;
-
-        public CompletableDumpFuture(final int contextId, final T emptyDump) {
-            this.contextId = contextId;
-            this.replyDump = emptyDump;
-        }
-
-        public int getContextId() {
-            return contextId;
-        }
-
-        public T getReplyDump() {
-            return replyDump;
-        }
-    }
-
-    @Override
-    public void close() throws Exception {
-        jvpp.close();
-    }
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java
deleted file mode 100644
index 7a48e418..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/future/FutureJVppInvoker.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.future;
-
-
-import io.fd.vpp.jvpp.dto.JVppReply;
-import io.fd.vpp.jvpp.dto.JVppReplyDump;
-import io.fd.vpp.jvpp.dto.JVppRequest;
-
-import java.util.concurrent.CompletionStage;
-import io.fd.vpp.jvpp.notification.NotificationRegistryProvider;
-
-/**
-* Future facade on top of JVpp
-*/
-public interface FutureJVppInvoker extends NotificationRegistryProvider, AutoCloseable {
-
-    /**
-     * Invoke asynchronous operation on VPP
-     *
-     * @return CompletionStage with future result of an async VPP call
-     * @throws io.fd.vpp.jvpp.VppInvocationException when send request failed with details
-     */
-    > CompletionStage send(REQ req);
-
-
-    /**
-     * Invoke asynchronous dump operation on VPP
-     *
-     * @return CompletionStage with aggregated future result of an async VPP dump call
-     * @throws io.fd.vpp.jvpp.VppInvocationException when send request failed with details
-     */
-    , DUMP extends JVppReplyDump> CompletionStage send(
-            REQ req, DUMP emptyReplyDump);
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java
deleted file mode 100644
index 3c72ff79..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistry.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.notification;
-
-/**
- * Base registry for notification callbacks.
- */
-public interface NotificationRegistry extends AutoCloseable {
-
-    void close();
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java
deleted file mode 100644
index 4a6e06b7..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/notification/NotificationRegistryProvider.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.notification;
-
-/**
- * Provides notification registry
- */
-public interface NotificationRegistryProvider {
-
-    /**
-     * Get current notification registry instance
-     */
-    NotificationRegistry getNotificationRegistry();
-}
diff --git a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java b/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java
deleted file mode 100644
index 27b4d29f..00000000
--- a/vpp-api/java/jvpp-registry/io/fd/vpp/jvpp/test/ConnectionTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.test;
-
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-
-/**
- * Run using:
- * sudo java -cp build-vpp-native/vpp-api/java/jvpp-registry-16.09.jar io.fd.vpp.jvpp.test.ConnectionTest
- */
-public class ConnectionTest {
-
-    private static void testConnect() throws Exception {
-        System.out.println("Testing JNI connection with JVppRegistry");
-        final JVppRegistry registry = new JVppRegistryImpl("ConnectionTest");
-        try {
-            System.out.println("Successfully connected to vpp");
-            Thread.sleep(5000);
-            System.out.println("Disconnecting...");
-            Thread.sleep(1000);
-        } finally {
-            registry.close();
-        }
-    }
-
-    public static void main(String[] args) throws Exception {
-        testConnect();
-    }
-}
diff --git a/vpp-api/java/jvpp-registry/jvpp_registry.c b/vpp-api/java/jvpp-registry/jvpp_registry.c
deleted file mode 100644
index cbd5e0ab..00000000
--- a/vpp-api/java/jvpp-registry/jvpp_registry.c
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#define _GNU_SOURCE /* for strcasestr(3) */
-#include 
-
-#define vl_api_version(n,v) static u32 vpe_api_version = (v);
-#include 
-#undef vl_api_version
-
-
-#include 
-#include 
-#include "io_fd_vpp_jvpp_VppJNIConnection.h"
-#include "io_fd_vpp_jvpp_JVppRegistryImpl.h"
-
-#include 
-#define vl_typedefs             /* define message structures */
-#include 
-#undef vl_typedefs
-
-#define vl_endianfun
-#include 
-#undef vl_endianfun
-
-/* instantiate all the print functions we know about */
-#define vl_print(handle, ...)
-#define vl_printfun
-#include 
-#undef vl_printfun
-
-vlib_main_t vlib_global_main;
-vlib_main_t **vlib_mains;
-
-/*
- * The Java runtime isn't compile w/ -fstack-protector,
- * so we have to supply missing external references for the
- * regular vpp libraries.
- */
-void __stack_chk_guard(void) __attribute__((weak));
-void __stack_chk_guard(void) {
-}
-
-typedef struct {
-    /* UThread attachment */
-    volatile u32 control_ping_result_ready;
-    volatile i32 control_ping_retval;
-
-    /* Control poing callback */
-    jobject registryObject;
-    jclass registryClass;
-    jclass controlPingReplyClass;
-    jclass callbackExceptionClass;
-
-    /* Thread cleanup */
-    pthread_key_t cleanup_rx_thread_key;
-
-    /* Connected indication */
-    volatile u8 is_connected;
-} jvpp_registry_main_t;
-
-jvpp_registry_main_t jvpp_registry_main __attribute__((aligned (64)));
-
-void vl_client_add_api_signatures(vl_api_memclnt_create_t *mp) {
-    /*
-     * Send the main API signature in slot 0. This bit of code must
-     * match the checks in ../vpe/api/api.c: vl_msg_api_version_check().
-     */
-    mp->api_versions[0] = clib_host_to_net_u32(vpe_api_version);
-}
-
-/* cleanup handler for RX thread */
-static_always_inline void cleanup_rx_thread(void *arg) {
-    jvpp_main_t * jm = &jvpp_main;
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-
-    vppjni_lock(jm, 99);
-
-    int getEnvStat = (*jm->jvm)->GetEnv(jm->jvm, (void **) &(jm->jenv),
-    JNI_VERSION_1_8);
-    if (getEnvStat == JNI_EVERSION) {
-        clib_warning("Unsupported JNI version\n");
-        rm->control_ping_retval = VNET_API_ERROR_UNSUPPORTED_JNI_VERSION;
-        goto out;
-    } else if (getEnvStat != JNI_EDETACHED) {
-        (*jm->jvm)->DetachCurrentThread(jm->jvm);
-    }
-    out: vppjni_unlock(jm);
-}
-
-static void vl_api_control_ping_reply_t_handler(
-        vl_api_control_ping_reply_t * mp) {
-    jvpp_main_t * jm = &jvpp_main;
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-    char was_thread_connected = 0;
-
-    // attach to java thread if not attached
-    int getEnvStat = (*jm->jvm)->GetEnv(jm->jvm, (void **) &(jm->jenv),
-            JNI_VERSION_1_8);
-    if (getEnvStat == JNI_EDETACHED) {
-        if ((*jm->jvm)->AttachCurrentThread(jm->jvm, (void **) &(jm->jenv),
-                NULL) != 0) {
-            clib_warning("Failed to attach thread\n");
-            rm->control_ping_retval =
-                    VNET_API_ERROR_FAILED_TO_ATTACH_TO_JAVA_THREAD;
-            goto out;
-        }
-
-        // workaround as we can't use pthread_cleanup_push
-        pthread_key_create(&rm->cleanup_rx_thread_key, cleanup_rx_thread);
-        // destructor is only called if the value of key is non null
-        pthread_setspecific(rm->cleanup_rx_thread_key, (void *) 1);
-        was_thread_connected = 1;
-    } else if (getEnvStat == JNI_EVERSION) {
-        clib_warning("Unsupported JNI version\n");
-        rm->control_ping_retval = VNET_API_ERROR_UNSUPPORTED_JNI_VERSION;
-        goto out;
-    }
-
-    if (was_thread_connected == 0) {
-        JNIEnv *env = jm->jenv;
-        if (mp->retval < 0) {
-            call_on_error("controlPing", mp->context, mp->retval,
-                    rm->registryClass, rm->registryObject,
-                    rm->callbackExceptionClass);
-        } else {
-            jmethodID constructor = (*env)->GetMethodID(env,
-                    rm->controlPingReplyClass, "", "()V");
-            jmethodID callbackMethod = (*env)->GetMethodID(env,
-                    rm->registryClass, "onControlPingReply",
-                    "(Lio/fd/vpp/jvpp/dto/ControlPingReply;)V");
-
-            jobject dto = (*env)->NewObject(env, rm->controlPingReplyClass,
-                    constructor);
-
-            jfieldID contextFieldId = (*env)->GetFieldID(env,
-                    rm->controlPingReplyClass, "context", "I");
-            (*env)->SetIntField(env, dto, contextFieldId,
-                    clib_net_to_host_u32(mp->context));
-
-            jfieldID clientIndexFieldId = (*env)->GetFieldID(env,
-                    rm->controlPingReplyClass, "clientIndex", "I");
-            (*env)->SetIntField(env, dto, clientIndexFieldId,
-                    clib_net_to_host_u32(mp->client_index));
-
-            jfieldID vpePidFieldId = (*env)->GetFieldID(env,
-                    rm->controlPingReplyClass, "vpePid", "I");
-            (*env)->SetIntField(env, dto, vpePidFieldId,
-                    clib_net_to_host_u32(mp->vpe_pid));
-
-            (*env)->CallVoidMethod(env, rm->registryObject, callbackMethod,
-                    dto);
-            (*env)->DeleteLocalRef(env, dto);
-        }
-    }
-
-    out: rm->control_ping_result_ready = 1;
-}
-
-static int send_initial_control_ping() {
-    f64 timeout;
-    clib_time_t clib_time;
-    vl_api_control_ping_t * mp;
-    jvpp_main_t * jm = &jvpp_main;
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-
-    clib_time_init(&clib_time);
-
-    rm->control_ping_result_ready = 0;
-    mp = vl_msg_api_alloc(sizeof(*mp));
-    memset(mp, 0, sizeof(*mp));
-    mp->_vl_msg_id = ntohs(VL_API_CONTROL_PING);
-    mp->client_index = jm->my_client_index;
-
-    // send message:
-    vl_msg_api_send_shmem(jm->vl_input_queue, (u8 *) &mp);
-
-    // wait for results: Current time + 10 seconds is the timeout
-    timeout = clib_time_now(&clib_time) + 10.0;
-    int rv = VNET_API_ERROR_RESPONSE_NOT_READY;
-    while (clib_time_now(&clib_time) < timeout) {
-        if (rm->control_ping_result_ready == 1) {
-            rv = rm->control_ping_retval;
-            break;
-        }
-    }
-
-    if (rv != 0) {
-        clib_warning("common: first control ping failed: %d", rv);
-    }
-
-    return rv;
-}
-
-static int connect_to_vpe(char *name) {
-    jvpp_main_t * jm = &jvpp_main;
-    api_main_t * am = &api_main;
-
-    if (vl_client_connect_to_vlib("/vpe-api", name, 32) < 0)
-        return -1;
-
-    jm->my_client_index = am->my_client_index;
-
-    jm->vl_input_queue = am->shmem_hdr->vl_input_queue;
-
-    vl_msg_api_set_handlers(VL_API_CONTROL_PING_REPLY, "control_ping_reply",
-            vl_api_control_ping_reply_t_handler, vl_noop_handler,
-            vl_api_control_ping_reply_t_endian,
-            vl_api_control_ping_reply_t_print,
-            sizeof(vl_api_control_ping_reply_t), 1);
-
-    return send_initial_control_ping();
-}
-
-JNIEXPORT jobject JNICALL Java_io_fd_vpp_jvpp_VppJNIConnection_clientConnect(
-        JNIEnv *env, jclass obj, jstring clientName) {
-    int rv;
-    const char *client_name;
-    void vl_msg_reply_handler_hookup(void);
-    jvpp_main_t * jm = &jvpp_main;
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-
-    jclass connectionInfoClass = (*env)->FindClass(env,
-            "io/fd/vpp/jvpp/VppJNIConnection$ConnectionInfo");
-    jmethodID connectionInfoConstructor = (*env)->GetMethodID(env,
-            connectionInfoClass, "", "(JII)V");
-
-    /*
-     * Bail out now if we're not running as root
-     */
-    if (geteuid() != 0) {
-        return (*env)->NewObject(env, connectionInfoClass,
-                connectionInfoConstructor, 0, 0,
-                VNET_API_ERROR_NOT_RUNNING_AS_ROOT);
-    }
-
-    if (rm->is_connected) {
-        return (*env)->NewObject(env, connectionInfoClass,
-                connectionInfoConstructor, 0, 0,
-                VNET_API_ERROR_ALREADY_CONNECTED);
-    }
-
-    client_name = (*env)->GetStringUTFChars(env, clientName, 0);
-    if (!client_name) {
-        return (*env)->NewObject(env, connectionInfoClass,
-                connectionInfoConstructor, 0, 0, VNET_API_ERROR_INVALID_VALUE);
-    }
-
-    rv = connect_to_vpe((char *) client_name);
-
-    if (rv < 0)
-        clib_warning("connection failed, rv %d", rv);
-
-    (*env)->ReleaseStringUTFChars(env, clientName, client_name);
-
-    return (*env)->NewObject(env, connectionInfoClass,
-            connectionInfoConstructor, (jlong) jm->vl_input_queue,
-            (jint) jm->my_client_index, (jint) rv);
-}
-
-JNIEXPORT jint JNICALL Java_io_fd_vpp_jvpp_JVppRegistryImpl_controlPing0(
-        JNIEnv *env, jobject regstryObject) {
-    jvpp_main_t * jm = &jvpp_main;
-    vl_api_control_ping_t * mp;
-    u32 my_context_id = vppjni_get_context_id(&jvpp_main);
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-
-    if (rm->registryObject == 0) {
-        rm->registryObject = (*env)->NewGlobalRef(env, regstryObject);
-    }
-    if (rm->registryClass == 0) {
-        rm->registryClass = (jclass) (*env)->NewGlobalRef(env,
-                (*env)->GetObjectClass(env, regstryObject));
-    }
-
-    mp = vl_msg_api_alloc(sizeof(*mp));
-    memset(mp, 0, sizeof(*mp));
-    mp->_vl_msg_id = ntohs(VL_API_CONTROL_PING);
-    mp->client_index = jm->my_client_index;
-    mp->context = clib_host_to_net_u32(my_context_id);
-
-    // send message:
-    vl_msg_api_send_shmem(jm->vl_input_queue, (u8 *) &mp);
-    return my_context_id;
-}
-
-JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_VppJNIConnection_clientDisconnect(
-        JNIEnv *env, jclass clazz) {
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-    rm->is_connected = 0; // TODO make thread safe
-    vl_client_disconnect_from_vlib();
-
-    // cleanup:
-    if (rm->registryObject) {
-        (*env)->DeleteGlobalRef(env, rm->registryObject);
-        rm->registryObject = 0;
-    }
-    if (rm->registryClass) {
-        (*env)->DeleteGlobalRef(env, rm->registryClass);
-        rm->registryClass = 0;
-    }
-}
-
-jint JNI_OnLoad(JavaVM *vm, void *reserved) {
-    jvpp_main_t * jm = &jvpp_main;
-    jvpp_registry_main_t * rm = &jvpp_registry_main;
-    JNIEnv* env;
-
-    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
-        return JNI_EVERSION;
-    }
-
-    rm->controlPingReplyClass = (jclass) (*env)->NewGlobalRef(env,
-            (*env)->FindClass(env, "io/fd/vpp/jvpp/dto/ControlPingReply"));
-    if ((*env)->ExceptionCheck(env)) {
-        (*env)->ExceptionDescribe(env);
-        clib_warning("Failed to cache class references\n");
-        return JNI_ERR;
-    }
-
-    rm->callbackExceptionClass = (jclass) (*env)->NewGlobalRef(env,
-            (*env)->FindClass(env, "io/fd/vpp/jvpp/VppCallbackException"));
-    if ((*env)->ExceptionCheck(env)) {
-        (*env)->ExceptionDescribe(env);
-        return JNI_ERR;
-    }
-
-    jm->jvm = vm;
-    return JNI_VERSION_1_8;
-}
-
-void JNI_OnUnload(JavaVM *vm, void *reserved) {
-    jvpp_main_t * jm = &jvpp_main;
-    JNIEnv* env;
-    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
-        return;
-    }
-
-    jm->jenv = NULL;
-    jm->jvm = NULL;
-}
diff --git a/vpp-api/java/jvpp/gen/jvpp_gen.py b/vpp-api/java/jvpp/gen/jvpp_gen.py
deleted file mode 100755
index 5f8df2a9..00000000
--- a/vpp-api/java/jvpp/gen/jvpp_gen.py
+++ /dev/null
@@ -1,171 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# l
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import argparse
-import importlib
-import sys
-import os
-import json
-
-from jvppgen import types_gen
-from jvppgen import callback_gen
-from jvppgen import notification_gen
-from jvppgen import dto_gen
-from jvppgen import jvpp_callback_facade_gen
-from jvppgen import jvpp_future_facade_gen
-from jvppgen import jvpp_impl_gen
-from jvppgen import jvpp_c_gen
-from jvppgen import util
-
-blacklist = [ "memclnt.api", "flowperpkt.api" ]
-
-# Invocation:
-# ~/Projects/vpp/vpp-api/jvpp/gen$ mkdir -p java/io/fd/vpp/jvpp && cd java/io/fd/vpp/jvpp
-# ~/Projects/vpp/vpp-api/jvpp/gen/java/io/fd/vpp/jvpp$ ../../../../jvpp_gen.py -idefs_api_vpp_papi.py
-#
-# Compilation:
-# ~/Projects/vpp/vpp-api/jvpp/gen/java/io/fd/vpp/jvpp$ javac *.java dto/*.java callback/*.java
-#
-# where
-# defs_api_vpp_papi.py - vpe.api in python format (generated by vppapigen)
-
-parser = argparse.ArgumentParser(description='VPP Java API generator')
-parser.add_argument('-i', action="store", dest="inputfiles", nargs='+')
-parser.add_argument('--plugin_name', action="store", dest="plugin_name")
-args = parser.parse_args()
-
-sys.path.append(".")
-
-print "Generating Java API for %s" % args.inputfiles
-print "inputfiles %s" % args.inputfiles
-plugin_name = args.plugin_name
-print "plugin_name %s" % plugin_name
-
-cfg = {}
-
-for inputfile in args.inputfiles:
-    if any(substring in inputfile for substring in blacklist):
-        print "WARNING: Imput file %s blacklisted" % inputfile
-        continue
-    _cfg = json.load(open(inputfile, 'r'))
-    if 'types' in cfg:
-        cfg['types'].extend(_cfg['types'])
-    else:
-        cfg['types'] = _cfg['types']
-    if 'messages' in cfg:
-        cfg['messages'].extend(_cfg['messages'])
-    else:
-        cfg['messages'] = _cfg['messages']
-
-
-def is_request_field(field_name):
-    return field_name not in {'_vl_msg_id', 'client_index', 'context'}
-
-
-def is_response_field(field_name):
-    return field_name not in {'_vl_msg_id'}
-
-
-def get_args(t, filter):
-    arg_list = []
-    for i in t:
-        if is_crc(i):
-            continue
-        if not filter(i[1]):
-            continue
-        arg_list.append(i[1])
-    return arg_list
-
-
-def get_types(t, filter):
-    types_list = []
-    lengths_list = []
-    crc = None
-    for i in t:
-        if is_crc(i):
-            crc = ('crc', i['crc'][2:])
-            continue
-        if not filter(i[1]):
-            continue
-        if len(i) is 3:  # array type
-            types_list.append(i[0] + '[]')
-            lengths_list.append((i[2], False))
-        elif len(i) is 4:  # variable length array type
-            types_list.append(i[0] + '[]')
-            lengths_list.append((i[3], True))
-        else:  # primitive type
-            types_list.append(i[0])
-            lengths_list.append((0, False))
-    return types_list, lengths_list, crc
-
-
-def is_crc(arg):
-    """ Check whether the argument inside message definition is just crc """
-    return 'crc' in arg
-
-
-def get_definitions(defs):
-    # Pass 1
-    func_list = []
-    func_name = {}
-    for a in defs:
-        java_name = util.underscore_to_camelcase(a[0])
-
-        # For replies include all the arguments except message_id
-        if util.is_reply(java_name):
-            types, lengths, crc = get_types(a[1:], is_response_field)
-            func_name[a[0]] = dict(
-                [('name', a[0]), ('java_name', java_name),
-                 ('args', get_args(a[1:], is_response_field)), ('full_args', get_args(a[1:], lambda x: True)),
-                 ('types', types), ('lengths', lengths), crc])
-        # For requests skip message_id, client_id and context
-        else:
-            types, lengths, crc = get_types(a[1:], is_request_field)
-            func_name[a[0]] = dict(
-                [('name', a[0]), ('java_name', java_name),
-                 ('args', get_args(a[1:], is_request_field)), ('full_args', get_args(a[1:], lambda x: True)),
-                 ('types', types), ('lengths', lengths), crc])
-
-        # Indexed by name
-        func_list.append(func_name[a[0]])
-    return func_list, func_name
-
-
-base_package = 'io.fd.vpp.jvpp'
-plugin_package = base_package + '.' + plugin_name
-types_package = 'types'
-dto_package = 'dto'
-callback_package = 'callback'
-notification_package = 'notification'
-future_package = 'future'
-# TODO find better package name
-callback_facade_package = 'callfacade'
-
-types_list, types_name = get_definitions(cfg['types'])
-
-types_gen.generate_types(types_list, plugin_package, types_package, args.inputfiles)
-
-func_list, func_name = get_definitions(cfg['messages'])
-
-dto_gen.generate_dtos(func_list, base_package, plugin_package, plugin_name.title(), dto_package, args.inputfiles)
-jvpp_impl_gen.generate_jvpp(func_list, base_package, plugin_package, plugin_name, dto_package, args.inputfiles)
-callback_gen.generate_callbacks(func_list, base_package, plugin_package, plugin_name.title(), callback_package, dto_package, args.inputfiles)
-notification_gen.generate_notification_registry(func_list, base_package, plugin_package, plugin_name.title(), notification_package, callback_package, dto_package, args.inputfiles)
-jvpp_c_gen.generate_jvpp(func_list, plugin_name, args.inputfiles)
-jvpp_future_facade_gen.generate_jvpp(func_list, base_package, plugin_package, plugin_name.title(), dto_package, callback_package, notification_package, future_package, args.inputfiles)
-jvpp_callback_facade_gen.generate_jvpp(func_list, base_package, plugin_package, plugin_name.title(), dto_package, callback_package, notification_package, callback_facade_package, args.inputfiles)
-
-print "Java API for %s generated successfully" % args.inputfiles
diff --git a/vpp-api/java/jvpp/gen/jvppgen/__init__.py b/vpp-api/java/jvpp/gen/jvppgen/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py b/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py
deleted file mode 100644
index 68f70126..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/callback_gen.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import util
-from string import Template
-
-from util import remove_suffix
-
-callback_suffix = "Callback"
-
-callback_template = Template("""
-package $plugin_package.$callback_package;
-
-/**
- * 
Represents callback for plugin's api file message.
- * 
It was generated by callback_gen.py based on $inputfile preparsed data:
- * 
-$docs
- * 

- */
-public interface $cls_name extends $base_package.$callback_package.$callback_type {
-
-    $callback_method
-
-}
-""")
-
-global_callback_template = Template("""
-package $plugin_package.$callback_package;
-
-/**
- * 
Global aggregated callback interface.
- * 
It was generated by callback_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface JVpp${plugin_name}GlobalCallback extends $base_package.$callback_package.ControlPingCallback, $callbacks {
-}
-""")
-
-
-def generate_callbacks(func_list, base_package, plugin_package, plugin_name, callback_package, dto_package, inputfile):
-    """ Generates callback interfaces """
-    print "Generating Callback interfaces"
-
-    if not os.path.exists(callback_package):
-        raise Exception("%s folder is missing" % callback_package)
-
-    callbacks = []
-    for func in func_list:
-
-        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
-
-        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_name_with_suffix):
-            continue
-        if not util.is_reply(camel_case_name_with_suffix) and not util.is_notification(func['name']):
-            continue
-
-        if util.is_reply(camel_case_name_with_suffix):
-            camel_case_name = util.remove_reply_suffix(camel_case_name_with_suffix)
-            callback_type = "JVppCallback"
-        else:
-            camel_case_name_with_suffix = util.add_notification_suffix(camel_case_name_with_suffix)
-            camel_case_name = camel_case_name_with_suffix
-            callback_type = "JVppNotificationCallback"
-
-        callbacks.append("{0}.{1}.{2}".format(plugin_package, callback_package, camel_case_name + callback_suffix))
-        callback_path = os.path.join(callback_package, camel_case_name + callback_suffix + ".java")
-        callback_file = open(callback_path, 'w')
-
-        reply_type = "%s.%s.%s" % (plugin_package, dto_package, camel_case_name_with_suffix)
-        method = "void on{0}({1} reply);".format(camel_case_name_with_suffix, reply_type)
-        callback_file.write(
-            callback_template.substitute(inputfile=inputfile,
-                                         docs=util.api_message_to_javadoc(func),
-                                         cls_name=camel_case_name + callback_suffix,
-                                         callback_method=method,
-                                         base_package=base_package,
-                                         plugin_package=plugin_package,
-                                         callback_package=callback_package,
-                                         callback_type=callback_type))
-        callback_file.flush()
-        callback_file.close()
-
-    callback_file = open(os.path.join(callback_package, "JVpp%sGlobalCallback.java" % plugin_name), 'w')
-    callback_file.write(global_callback_template.substitute(inputfile=inputfile,
-                                                            callbacks=", ".join(callbacks),
-                                                            base_package=base_package,
-                                                            plugin_package=plugin_package,
-                                                            plugin_name=plugin_name,
-                                                            callback_package=callback_package))
-    callback_file.flush()
-    callback_file.close()
diff --git a/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py b/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
deleted file mode 100644
index a043c945..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
+++ /dev/null
@@ -1,308 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from string import Template
-
-import util
-
-dto_template = Template("""
-package $plugin_package.$dto_package;
-
-/**
- * 
This class represents $description.
- * 
It was generated by dto_gen.py based on $inputfile preparsed data:
- * 
-$docs
- * 

- */
-public final class $cls_name implements $base_package.$dto_package.$base_type {
-
-$fields
-$methods
-}
-""")
-
-field_template = Template("""    public $type $name;\n""")
-
-send_template = Template("""    @Override
-    public int send(final $base_package.JVpp jvpp) throws io.fd.vpp.jvpp.VppInvocationException {
-        return (($plugin_package.JVpp${plugin_name})jvpp).$method_name($args);
-    }""")
-
-
-def generate_dtos(func_list, base_package, plugin_package, plugin_name, dto_package, inputfile):
-    """ Generates dto objects in a dedicated package """
-    print "Generating DTOs"
-
-    if not os.path.exists(dto_package):
-        raise Exception("%s folder is missing" % dto_package)
-
-    for func in func_list:
-        camel_case_dto_name = util.underscore_to_camelcase_upper(func['name'])
-        camel_case_method_name = util.underscore_to_camelcase(func['name'])
-        dto_path = os.path.join(dto_package, camel_case_dto_name + ".java")
-
-        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_dto_name):
-            continue
-
-        fields = generate_dto_fields(camel_case_dto_name, func)
-        methods = generate_dto_base_methods(camel_case_dto_name, func)
-        base_type = ""
-
-        # Generate request/reply or dump/dumpReply even if structure can be used as notification
-        if not util.is_just_notification(func["name"]):
-            if util.is_reply(camel_case_dto_name):
-                description = "reply DTO"
-                request_dto_name = get_request_name(camel_case_dto_name, func['name'])
-                if util.is_details(camel_case_dto_name):
-                    # FIXME assumption that dump calls end with "Dump" suffix. Not enforced in vpe.api
-                    base_type += "JVppReply<%s.%s.%s>" % (plugin_package, dto_package, request_dto_name + "Dump")
-                    generate_dump_reply_dto(request_dto_name, base_package, plugin_package, dto_package,
-                                            camel_case_dto_name, camel_case_method_name, func)
-                else:
-                    base_type += "JVppReply<%s.%s.%s>" % (plugin_package, dto_package, request_dto_name)
-            else:
-                args = "" if fields is "" else "this"
-                methods += send_template.substitute(method_name=camel_case_method_name,
-                                                    base_package=base_package,
-                                                    plugin_package=plugin_package,
-                                                    plugin_name=plugin_name,
-                                                    args=args)
-                if util.is_dump(camel_case_dto_name):
-                    base_type += "JVppDump"
-                    description = "dump request DTO"
-                else:
-                    base_type += "JVppRequest"
-                    description = "request DTO"
-
-            write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package,
-                           dto_path, fields, func, inputfile, methods)
-
-        # for structures that are also used as notifications, generate dedicated notification DTO
-        if util.is_notification(func["name"]):
-            base_type = "JVppNotification"
-            description = "notification DTO"
-            camel_case_dto_name = util.add_notification_suffix(camel_case_dto_name)
-            dto_path = os.path.join(dto_package, camel_case_dto_name + ".java")
-            methods = generate_dto_base_methods(camel_case_dto_name, func)
-            write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package,
-                           dto_path, fields, func, inputfile, methods)
-
-    flush_dump_reply_dtos(inputfile)
-
-
-def generate_dto_base_methods(camel_case_dto_name, func):
-    methods = generate_dto_hash(func)
-    methods += generate_dto_equals(camel_case_dto_name, func)
-    methods += generate_dto_tostring(camel_case_dto_name, func)
-    return methods
-
-
-def generate_dto_fields(camel_case_dto_name, func):
-    fields = ""
-    for t in zip(func['types'], func['args']):
-        # for retval don't generate dto field in Reply
-        field_name = util.underscore_to_camelcase(t[1])
-        if util.is_reply(camel_case_dto_name) and util.is_retval_field(field_name):
-            continue
-        fields += field_template.substitute(type=util.jni_2_java_type_mapping[t[0]],
-                                            name=field_name)
-    return fields
-
-
-tostring_field_template = Template("""                \"$field_name=\" + $field_name + ", " +\n""")
-tostring_array_field_template = Template("""                \"$field_name=\" + java.util.Arrays.toString($field_name) + ", " +\n""")
-tostring_template = Template("""    @Override
-    public String toString() {
-        return "$cls_name{" +
-$fields_tostring "}";
-    }\n\n""")
-
-
-def generate_dto_tostring(camel_case_dto_name, func):
-    tostring_fields = ""
-    for t in zip(func['types'], func['args']):
-
-        field_name = util.underscore_to_camelcase(t[1])
-        # for retval don't generate dto field in Reply
-        if util.is_retval_field(field_name):
-            continue
-
-        # handle array types
-        if util.is_array(util.jni_2_java_type_mapping[t[0]]):
-            tostring_fields += tostring_array_field_template.substitute(field_name=field_name)
-        else:
-            tostring_fields += tostring_field_template.substitute(field_name=field_name)
-
-    return tostring_template.substitute(cls_name=camel_case_dto_name,
-                                        fields_tostring=tostring_fields[:-8])
-
-equals_other_template = Template("""
-        final $cls_name other = ($cls_name) o;
-\n""")
-equals_field_template = Template("""        if (!java.util.Objects.equals(this.$field_name, other.$field_name)) {
-            return false;
-        }\n""")
-equals_array_field_template = Template("""        if (!java.util.Arrays.equals(this.$field_name, other.$field_name)) {
-            return false;
-        }\n""")
-equals_template = Template("""    @Override
-    public boolean equals(final Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-$comparisons
-        return true;
-    }\n\n""")
-
-
-def generate_dto_equals(camel_case_dto_name, func):
-    equals_fields = ""
-    for t in zip(func['types'], func['args']):
-        field_name = util.underscore_to_camelcase(t[1])
-        # for retval don't generate dto field in Reply
-        if util.is_retval_field(field_name):
-            continue
-
-        # handle array types
-        if util.is_array(util.jni_2_java_type_mapping[t[0]]):
-            equals_fields += equals_array_field_template.substitute(field_name=field_name)
-        else:
-            equals_fields += equals_field_template.substitute(field_name=field_name)
-
-    if equals_fields != "":
-        equals_fields = equals_other_template.substitute(cls_name=camel_case_dto_name) + equals_fields
-
-    return equals_template.substitute(comparisons=equals_fields)
-
-
-hash_template = Template("""    @Override
-    public int hashCode() {
-        return java.util.Objects.hash($fields);
-    }\n\n""")
-hash_single_array_type_template = Template("""    @Override
-    public int hashCode() {
-        return java.util.Arrays.hashCode($fields);
-    }\n\n""")
-
-
-def generate_dto_hash(func):
-    hash_fields = ""
-
-    # Special handling for hashCode in case just a single array field is present. Cannot use Objects.equals since the
-    # array is mistaken for a varargs parameter. Instead use Arrays.hashCode in such case.
-    if len(func['args']) == 1:
-        single_type = func['types'][0]
-        single_type_name = func['args'][0]
-        if util.is_array(util.jni_2_java_type_mapping[single_type]):
-            return hash_single_array_type_template.substitute(fields=util.underscore_to_camelcase(single_type_name))
-
-    for t in zip(func['types'], func['args']):
-        field_name = util.underscore_to_camelcase(t[1])
-        # for retval don't generate dto field in Reply
-        if util.is_retval_field(field_name):
-            continue
-
-        hash_fields += field_name + ", "
-
-    return hash_template.substitute(fields=hash_fields[:-2])
-
-
-def write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package, dto_path,
-                   fields, func, inputfile, methods):
-    dto_file = open(dto_path, 'w')
-    dto_file.write(dto_template.substitute(inputfile=inputfile,
-                                           description=description,
-                                           docs=util.api_message_to_javadoc(func),
-                                           cls_name=camel_case_dto_name,
-                                           fields=fields,
-                                           methods=methods,
-                                           base_package=base_package,
-                                           plugin_package=plugin_package,
-                                           base_type=base_type,
-                                           dto_package=dto_package))
-    dto_file.flush()
-    dto_file.close()
-
-
-dump_dto_suffix = "ReplyDump"
-dump_reply_artificial_dtos = {}
-
-
-# Returns request name or special one from unconventional_naming_rep_req map
-def get_request_name(camel_case_dto_name, func_name):
-    return util.underscore_to_camelcase_upper(
-        util.unconventional_naming_rep_req[func_name]) if func_name in util.unconventional_naming_rep_req \
-        else util.remove_reply_suffix(camel_case_dto_name)
-
-
-def flush_dump_reply_dtos(inputfile):
-    for dump_reply_artificial_dto in dump_reply_artificial_dtos.values():
-        dto_path = os.path.join(dump_reply_artificial_dto['dto_package'],
-                                dump_reply_artificial_dto['cls_name'] + ".java")
-        dto_file = open(dto_path, 'w')
-        dto_file.write(dto_template.substitute(inputfile=inputfile,
-                                               description="dump reply wrapper",
-                                               docs=dump_reply_artificial_dto['docs'],
-                                               cls_name=dump_reply_artificial_dto['cls_name'],
-                                               fields=dump_reply_artificial_dto['fields'],
-                                               methods=dump_reply_artificial_dto['methods'],
-                                               plugin_package=dump_reply_artificial_dto['plugin_package'],
-                                               base_package=dump_reply_artificial_dto['base_package'],
-                                               base_type=dump_reply_artificial_dto['base_type'],
-                                               dto_package=dump_reply_artificial_dto['dto_package']))
-        dto_file.flush()
-        dto_file.close()
-
-
-def generate_dump_reply_dto(request_dto_name, base_package, plugin_package, dto_package, camel_case_dto_name,
-                            camel_case_method_name, func):
-    base_type = "JVppReplyDump<%s.%s.%s, %s.%s.%s>" % (
-        plugin_package, dto_package, util.remove_reply_suffix(camel_case_dto_name) + "Dump",
-        plugin_package, dto_package, camel_case_dto_name)
-    fields = "    public java.util.List<%s> %s = new java.util.ArrayList<>();" % (camel_case_dto_name, camel_case_method_name)
-    cls_name = camel_case_dto_name + dump_dto_suffix
-    # using artificial type for fields, just to bypass the is_array check in base methods generators
-    # the type is not really used
-    artificial_type = 'u8'
-
-    # In case of already existing artificial reply dump DTO, just update it
-    # Used for sub-dump dtos
-    if request_dto_name in dump_reply_artificial_dtos.keys():
-        dump_reply_artificial_dtos[request_dto_name]['fields'] += '\n' + fields
-        dump_reply_artificial_dtos[request_dto_name]['field_names'].append(func['name'])
-        dump_reply_artificial_dtos[request_dto_name]['field_types'].append(artificial_type)
-        methods = '\n' + generate_dto_base_methods(dump_reply_artificial_dtos[request_dto_name]['cls_name'],
-                                            {'args': dump_reply_artificial_dtos[request_dto_name]['field_names'],
-                                             'types': dump_reply_artificial_dtos[request_dto_name]['field_types']})
-        dump_reply_artificial_dtos[request_dto_name]['methods'] = methods
-    else:
-        methods = '\n' + generate_dto_base_methods(cls_name, {'args': [func['name']],
-                                                              'types': [artificial_type]})
-        dump_reply_artificial_dtos[request_dto_name] = ({'docs': util.api_message_to_javadoc(func),
-                                                         'cls_name': cls_name,
-                                                         'fields': fields,
-                                                         'field_names': [func['name']],
-                                                         'field_types': [artificial_type],
-                                                         # strip too many newlines at the end of base method block
-                                                         'methods': methods,
-                                                         'plugin_package': plugin_package,
-                                                         'base_package': base_package,
-                                                         'base_type': base_type,
-                                                         'dto_package': dto_package})
diff --git a/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py b/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
deleted file mode 100644
index 328cc8d3..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
+++ /dev/null
@@ -1,295 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from string import Template
-
-import util
-
-variable_length_array_value_template = Template("""mp->${length_var_name}""")
-variable_length_array_template = Template("""clib_net_to_host_${length_field_type}(${value})""")
-
-dto_field_id_template = Template("""
-    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${class_ref_name}Class, "${field_name}", "${jni_signature}");""")
-
-default_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, mp->${c_name});
-""")
-
-variable_length_array_value_template = Template("""mp->${length_var_name}""")
-variable_length_array_template = Template("""clib_net_to_host_${length_field_type}(${value})""")
-
-u16_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u16(mp->${c_name}));
-""")
-
-u32_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u32(mp->${c_name}));
-""")
-
-u64_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u64(mp->${c_name}));
-""")
-
-u8_array_dto_field_setter_template = Template("""
-    jbyteArray ${field_reference_name} = (*env)->NewByteArray(env, ${field_length});
-    (*env)->SetByteArrayRegion(env, ${field_reference_name}, 0, ${field_length}, (const jbyte*)mp->${c_name});
-    (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-    (*env)->DeleteLocalRef(env, ${field_reference_name});
-""")
-
-u16_array_dto_field_setter_template = Template("""
-    {
-        jshortArray ${field_reference_name} = (*env)->NewShortArray(env, ${field_length});
-        jshort * ${field_reference_name}ArrayElements = (*env)->GetShortArrayElements(env, ${field_reference_name}, NULL);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u16(mp->${c_name}[_i]);
-        }
-
-        (*env)->ReleaseShortArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-        (*env)->DeleteLocalRef(env, ${field_reference_name});
-    }
-""")
-
-u32_array_dto_field_setter_template = Template("""
-    {
-        jintArray ${field_reference_name} = (*env)->NewIntArray(env, ${field_length});
-        jint * ${field_reference_name}ArrayElements = (*env)->GetIntArrayElements(env, ${field_reference_name}, NULL);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u32(mp->${c_name}[_i]);
-        }
-
-        (*env)->ReleaseIntArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-        (*env)->DeleteLocalRef(env, ${field_reference_name});
-    }
-""")
-
-# For each u64 array we get its elements. Then we convert values to host byte order.
-# All changes to  jlong* buffer are written to jlongArray (isCopy is set to NULL)
-u64_array_dto_field_setter_template = Template("""
-    {
-        jlongArray ${field_reference_name} = (*env)->NewLongArray(env, ${field_length});
-        jlong * ${field_reference_name}ArrayElements = (*env)->GetLongArrayElements(env, ${field_reference_name}, NULL);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u64(mp->${c_name}[_i]);
-        }
-
-        (*env)->ReleaseLongArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-        (*env)->DeleteLocalRef(env, ${field_reference_name});
-    }
-""")
-
-dto_field_setter_templates = {'u8': default_dto_field_setter_template,
-                              'u16': u16_dto_field_setter_template,
-                              'u32': u32_dto_field_setter_template,
-                              'i32': u32_dto_field_setter_template,
-                              'u64': u64_dto_field_setter_template,
-                              'f64': default_dto_field_setter_template,  # fixme
-                              'u8[]': u8_array_dto_field_setter_template,
-                              'u16[]': u16_array_dto_field_setter_template,
-                              'u32[]': u32_array_dto_field_setter_template,
-                              'u64[]': u64_array_dto_field_setter_template
-                              }
-
-
-def jni_reply_handler_for_type(handler_name, ref_name, field_type, c_name, field_reference_name,
-                               field_name, field_length, is_variable_len_array, length_field_type,
-                               object_name="dto"):
-    """
-    Generates jni code that initializes a field of java object (dto or custom type).
-    To be used in reply message handlers.
-    :param field_type: type of the field to be initialized (as defined in vpe.api)
-    :param c_name: name of the message struct member that stores initialization value
-    :param field_reference_name: name of the field reference in generated code
-    :param field_name: name of the field (camelcase)
-    :param field_length: integer or name of variable that stores field length
-    :param object_name: name of the object to be initialized
-    """
-
-    # todo move validation to vppapigen
-    if field_type.endswith('[]') and field_length == '0':
-        raise Exception('Variable array \'%s\' defined in \'%s\' '
-                        'should have defined length (e.g. \'%s[%s_length]\''
-                        % (c_name, handler_name, c_name, c_name))
-
-    if is_variable_len_array:
-        length_var_name = field_length
-        field_length = variable_length_array_value_template.substitute(length_var_name=length_var_name)
-        if length_field_type != 'u8':  # we need net to host conversion:
-            field_length = variable_length_array_template.substitute(
-                    length_field_type=length_field_type, value=field_length)
-
-    # for retval don't generate setters
-    if util.is_retval_field(c_name):
-        return ""
-
-    jni_signature = util.jni_2_signature_mapping[field_type]
-    jni_setter = util.jni_field_accessors[field_type]
-
-    result = dto_field_id_template.substitute(
-            field_reference_name=field_reference_name,
-            field_name=field_name,
-            class_ref_name=ref_name,
-            jni_signature=jni_signature)
-
-    dto_setter_template = dto_field_setter_templates[field_type]
-
-    result += dto_setter_template.substitute(
-            jni_signature=jni_signature,
-            object_name=object_name,
-            field_reference_name=field_reference_name,
-            c_name=c_name,
-            jni_setter=jni_setter,
-            field_length=field_length)
-    return result
-
-
-request_field_identifier_template = Template("""
-    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${object_name}Class, "${field_name}", "${jni_signature}");
-    ${jni_type} ${field_reference_name} = (*env)->Get${jni_getter}(env, ${object_name}, ${field_reference_name}FieldId);
-    """)
-
-array_length_enforcement_template = Template("""
-        size_t max_size = ${field_length};
-        if (cnt > max_size) cnt = max_size;""")
-
-u8_struct_setter_template = Template("""
-    mp->${c_name} = ${field_reference_name};""")
-
-u16_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_u16(${field_reference_name});""")
-
-u32_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_u32(${field_reference_name});""")
-
-i32_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_i32(${field_reference_name});!""")
-
-u64_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_u64(${field_reference_name});""")
-
-array_length_enforcement_template = Template("""
-        size_t max_size = ${field_length};
-        if (cnt > max_size) cnt = max_size;""")
-
-u8_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        (*env)->GetByteArrayRegion(env, ${field_reference_name}, 0, cnt, (jbyte *)mp->${c_name});
-    }
-""")
-
-u16_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jshort * ${field_reference_name}ArrayElements = (*env)->GetShortArrayElements(env, ${field_reference_name}, NULL);
-        size_t _i;
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        for (_i = 0; _i < cnt; _i++) {
-            mp->${c_name}[_i] = clib_host_to_net_u16(${field_reference_name}ArrayElements[_i]);
-        }
-        (*env)->ReleaseShortArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-    }
-    """)
-
-u32_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jint * ${field_reference_name}ArrayElements = (*env)->GetIntArrayElements(env, ${field_reference_name}, NULL);
-        size_t _i;
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        for (_i = 0; _i < cnt; _i++) {
-            mp->${c_name}[_i] = clib_host_to_net_u32(${field_reference_name}ArrayElements[_i]);
-        }
-        (*env)->ReleaseIntArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-    }
-    """)
-
-u64_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jlong * ${field_reference_name}ArrayElements = (*env)->GetLongArrayElements(env, ${field_reference_name}, NULL);
-        size_t _i;
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        for (_i = 0; _i < cnt; _i++) {
-            mp->${c_name}[_i] = clib_host_to_net_u64(${field_reference_name}ArrayElements[_i]);
-        }
-        (*env)->ReleaseLongArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-    }
-    """)
-
-struct_setter_templates = {'u8': u8_struct_setter_template,
-                           'u16': u16_struct_setter_template,
-                           'u32': u32_struct_setter_template,
-                           'i32': u32_struct_setter_template,
-                           'u64': u64_struct_setter_template,
-                           'u8[]': u8_array_struct_setter_template,
-                           'u16[]': u16_array_struct_setter_template,
-                           'u32[]': u32_array_struct_setter_template,
-                           'u64[]': u64_array_struct_setter_template
-                           }
-
-
-def jni_request_binding_for_type(field_type, c_name, field_reference_name, field_name, field_length,
-                                 is_variable_len_array, object_name="request"):
-    """
-    Generates jni code that initializes C structure that corresponds to a field of java object
-    (dto or custom type). To be used in request message handlers.
-    :param field_type: type of the field to be initialized (as defined in vpe.api)
-    :param c_name: name of the message struct member to be initialized
-    :param field_reference_name: name of the field reference in generated code
-    :param field_name: name of the field (camelcase)
-    :param field_length: integer or name of variable that stores field length
-    :param object_name: name of the object to be initialized
-    """
-    # field identifiers
-    jni_type = util.vpp_2_jni_type_mapping[field_type]
-    jni_signature = util.jni_2_signature_mapping[field_type]
-    jni_getter = util.jni_field_accessors[field_type]
-
-    # field identifier
-    msg_initialization = request_field_identifier_template.substitute(
-            jni_type=jni_type,
-            field_reference_name=field_reference_name,
-            field_name=field_name,
-            jni_signature=jni_signature,
-            jni_getter=jni_getter,
-            object_name=object_name)
-
-    # field setter
-    field_length_check = ""
-
-    # check if we are processing variable length array:
-    if is_variable_len_array:
-        field_length = util.underscore_to_camelcase(field_length)
-
-    # enforce max length if array has fixed length or uses variable length syntax
-    if str(field_length) != "0":
-        field_length_check = array_length_enforcement_template.substitute(field_length=field_length)
-
-    struct_setter_template = struct_setter_templates[field_type]
-
-    msg_initialization += struct_setter_template.substitute(
-            c_name=c_name,
-            field_reference_name=field_reference_name,
-            field_length_check=field_length_check)
-
-    return msg_initialization
diff --git a/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py b/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py
deleted file mode 100644
index 5fc84c7b..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/jvpp_c_gen.py
+++ /dev/null
@@ -1,343 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-# l
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, util
-from string import Template
-
-import jni_gen
-
-
-def is_manually_generated(f_name, plugin_name):
-    return f_name in {'control_ping_reply'}
-
-
-class_reference_template = Template("""jclass ${ref_name}Class;
-""")
-
-find_class_invocation_template = Template("""
-    ${ref_name}Class = (jclass)(*env)->NewGlobalRef(env, (*env)->FindClass(env, "io/fd/vpp/jvpp/${plugin_name}/dto/${class_name}"));
-    if ((*env)->ExceptionCheck(env)) {
-        (*env)->ExceptionDescribe(env);
-        return JNI_ERR;
-    }""")
-
-find_class_template = Template("""
-    ${ref_name}Class = (jclass)(*env)->NewGlobalRef(env, (*env)->FindClass(env, "${class_name}"));
-    if ((*env)->ExceptionCheck(env)) {
-        (*env)->ExceptionDescribe(env);
-        return JNI_ERR;
-    }""")
-
-delete_class_invocation_template = Template("""
-    if (${ref_name}Class) {
-        (*env)->DeleteGlobalRef(env, ${ref_name}Class);
-    }""")
-
-class_cache_template = Template("""
-$class_references
-static int cache_class_references(JNIEnv* env) {
-    $find_class_invocations
-    return 0;
-}
-
-static void delete_class_references(JNIEnv* env) {
-    $delete_class_invocations
-}""")
-
-
-def generate_class_cache(func_list, plugin_name):
-    class_references = []
-    find_class_invocations = []
-    delete_class_invocations = []
-    for f in func_list:
-        c_name = f['name']
-        class_name = util.underscore_to_camelcase_upper(c_name)
-        ref_name = util.underscore_to_camelcase(c_name)
-
-        if util.is_ignored(c_name) or util.is_control_ping(class_name):
-            continue
-
-        if util.is_reply(class_name):
-            class_references.append(class_reference_template.substitute(
-                ref_name=ref_name))
-            find_class_invocations.append(find_class_invocation_template.substitute(
-                plugin_name=plugin_name,
-                ref_name=ref_name,
-                class_name=class_name))
-            delete_class_invocations.append(delete_class_invocation_template.substitute(ref_name=ref_name))
-        elif util.is_notification(c_name):
-            class_references.append(class_reference_template.substitute(
-                ref_name=util.add_notification_suffix(ref_name)))
-            find_class_invocations.append(find_class_invocation_template.substitute(
-                plugin_name=plugin_name,
-                ref_name=util.add_notification_suffix(ref_name),
-                class_name=util.add_notification_suffix(class_name)))
-            delete_class_invocations.append(delete_class_invocation_template.substitute(
-                ref_name=util.add_notification_suffix(ref_name)))
-
-    # add exception class to class cache
-    ref_name = 'callbackException'
-    class_name = 'io/fd/vpp/jvpp/VppCallbackException'
-    class_references.append(class_reference_template.substitute(
-            ref_name=ref_name))
-    find_class_invocations.append(find_class_template.substitute(
-            ref_name=ref_name,
-            class_name=class_name))
-    delete_class_invocations.append(delete_class_invocation_template.substitute(ref_name=ref_name))
-
-    return class_cache_template.substitute(
-        class_references="".join(class_references), find_class_invocations="".join(find_class_invocations),
-        delete_class_invocations="".join(delete_class_invocations))
-
-
-# TODO: cache method and field identifiers to achieve better performance
-# https://jira.fd.io/browse/HONEYCOMB-42
-request_class_template = Template("""
-    jclass requestClass = (*env)->FindClass(env, "io/fd/vpp/jvpp/${plugin_name}/dto/${java_name_upper}");""")
-
-request_field_identifier_template = Template("""
-    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${object_name}Class, "${field_name}", "${jni_signature}");
-    ${jni_type} ${field_reference_name} = (*env)->Get${jni_getter}(env, ${object_name}, ${field_reference_name}FieldId);
-    """)
-
-
-jni_impl_template = Template("""
-/**
- * JNI binding for sending ${c_name} message.
- * Generated based on $inputfile preparsed data:
-$api_data
- */
-JNIEXPORT jint JNICALL Java_io_fd_vpp_jvpp_${plugin_name}_JVpp${java_plugin_name}Impl_${field_name}0
-(JNIEnv * env, jclass clazz$args) {
-    ${plugin_name}_main_t *plugin_main = &${plugin_name}_main;
-    vl_api_${c_name}_t * mp;
-    u32 my_context_id = vppjni_get_context_id (&jvpp_main);
-    $request_class
-
-    // create message:
-    mp = vl_msg_api_alloc(sizeof(*mp));
-    memset (mp, 0, sizeof (*mp));
-    mp->_vl_msg_id = ntohs (VL_API_${c_name_uppercase} + plugin_main->msg_id_base);
-    mp->client_index = plugin_main->my_client_index;
-    mp->context = clib_host_to_net_u32 (my_context_id);
-
-    $msg_initialization
-
-    // send message:
-    vl_msg_api_send_shmem (plugin_main->vl_input_queue, (u8 *)&mp);
-    if ((*env)->ExceptionCheck(env)) {
-        return JNI_ERR;
-    }
-    return my_context_id;
-}""")
-
-def generate_jni_impl(func_list, plugin_name, inputfile):
-    jni_impl = []
-    for f in func_list:
-        f_name = f['name']
-        camel_case_function_name = util.underscore_to_camelcase(f_name)
-        if is_manually_generated(f_name, plugin_name) or util.is_reply(camel_case_function_name) \
-                or util.is_ignored(f_name) or util.is_just_notification(f_name):
-            continue
-
-        arguments = ''
-        request_class = ''
-        msg_initialization = ''
-        f_name_uppercase = f_name.upper()
-
-        if f['args']:
-            arguments = ', jobject request'
-            camel_case_function_name_upper = util.underscore_to_camelcase_upper(f_name)
-
-            request_class = request_class_template.substitute(
-                    java_name_upper=camel_case_function_name_upper,
-                    plugin_name=plugin_name)
-
-            for t in zip(f['types'], f['args'], f['lengths']):
-                field_name = util.underscore_to_camelcase(t[1])
-                msg_initialization += jni_gen.jni_request_binding_for_type(field_type=t[0], c_name=t[1],
-                                                                           field_reference_name=field_name,
-                                                                           field_name=field_name,
-                                                                           field_length=t[2][0],
-                                                                           is_variable_len_array=t[2][1])
-
-        jni_impl.append(jni_impl_template.substitute(
-                inputfile=inputfile,
-                api_data=util.api_message_to_javadoc(f),
-                field_reference_name=camel_case_function_name,
-                field_name=camel_case_function_name,
-                c_name_uppercase=f_name_uppercase,
-                c_name=f_name,
-                plugin_name=plugin_name,
-                java_plugin_name=plugin_name.title(),
-                request_class=request_class,
-                msg_initialization=msg_initialization,
-                args=arguments))
-
-    return "\n".join(jni_impl)
-
-# code fragment for checking result of the operation before sending request reply
-callback_err_handler_template = Template("""
-    // for negative result don't send callback message but send error callback
-    if (mp->retval<0) {
-        call_on_error("${handler_name}", mp->context, mp->retval, plugin_main->callbackClass, plugin_main->callbackObject, callbackExceptionClass);
-        return;
-    }
-    if (mp->retval == VNET_API_ERROR_IN_PROGRESS) {
-        clib_warning("Result in progress");
-        return;
-    }
-""")
-
-msg_handler_template = Template("""
-/**
- * Handler for ${handler_name} message.
- * Generated based on $inputfile preparsed data:
-$api_data
- */
-static void vl_api_${handler_name}_t_handler (vl_api_${handler_name}_t * mp)
-{
-    ${plugin_name}_main_t *plugin_main = &${plugin_name}_main;
-    JNIEnv *env = jvpp_main.jenv;
-
-    $err_handler
-
-    jmethodID constructor = (*env)->GetMethodID(env, ${class_ref_name}Class, "", "()V");
-    jmethodID callbackMethod = (*env)->GetMethodID(env, plugin_main->callbackClass, "on${dto_name}", "(Lio/fd/vpp/jvpp/${plugin_name}/dto/${dto_name};)V");
-
-    jobject dto = (*env)->NewObject(env, ${class_ref_name}Class, constructor);
-    $dto_setters
-
-    (*env)->CallVoidMethod(env, plugin_main->callbackObject, callbackMethod, dto);
-    // free DTO as per http://stackoverflow.com/questions/1340938/memory-leak-when-calling-java-code-from-c-using-jni
-    (*env)->DeleteLocalRef(env, dto);
-}""")
-
-
-def generate_msg_handlers(func_list, plugin_name, inputfile):
-    handlers = []
-    for f in func_list:
-        handler_name = f['name']
-        dto_name = util.underscore_to_camelcase_upper(handler_name)
-        ref_name = util.underscore_to_camelcase(handler_name)
-
-        if is_manually_generated(handler_name, plugin_name) or util.is_ignored(handler_name):
-            continue
-
-        if not util.is_reply(dto_name) and not util.is_notification(handler_name):
-            continue
-
-        if util.is_notification(handler_name):
-            dto_name = util.add_notification_suffix(dto_name)
-            ref_name = util.add_notification_suffix(ref_name)
-
-        dto_setters = ''
-        err_handler = ''
-        # dto setters
-        for t in zip(f['types'], f['args'], f['lengths']):
-            c_name = t[1]
-            java_name = util.underscore_to_camelcase(c_name)
-            field_length = t[2][0]
-            is_variable_len_array = t[2][1]
-            length_field_type = None
-            if is_variable_len_array:
-                length_field_type = f['types'][f['args'].index(field_length)]
-            dto_setters += jni_gen.jni_reply_handler_for_type(handler_name=handler_name, ref_name=ref_name,
-                                                              field_type=t[0], c_name=t[1],
-                                                              field_reference_name=java_name,
-                                                              field_name=java_name, field_length=field_length,
-                                                              is_variable_len_array=is_variable_len_array,
-                                                              length_field_type=length_field_type)
-
-            # for retval don't generate setters and generate retval check
-            if util.is_retval_field(c_name):
-                err_handler = callback_err_handler_template.substitute(
-                    handler_name=handler_name
-                )
-                continue
-
-        handlers.append(msg_handler_template.substitute(
-            inputfile=inputfile,
-            api_data=util.api_message_to_javadoc(f),
-            handler_name=handler_name,
-            plugin_name=plugin_name,
-            dto_name=dto_name,
-            class_ref_name=ref_name,
-            dto_setters=dto_setters,
-            err_handler=err_handler))
-
-    return "\n".join(handlers)
-
-
-handler_registration_template = Template("""_(${upercase_name}, ${name}) \\
-""")
-
-
-def generate_handler_registration(func_list):
-    handler_registration = ["#define foreach_api_reply_handler \\\n"]
-    for f in func_list:
-        name = f['name']
-        camelcase_name = util.underscore_to_camelcase(f['name'])
-
-        if (not util.is_reply(camelcase_name) and not util.is_notification(name)) or util.is_ignored(name) \
-                or util.is_control_ping(camelcase_name):
-            continue
-
-        handler_registration.append(handler_registration_template.substitute(
-            name=name,
-            upercase_name=name.upper()))
-
-    return "".join(handler_registration)
-
-
-jvpp_c_template = Template("""/**
- * This file contains JNI bindings for jvpp Java API.
- * It was generated by jvpp_c_gen.py based on $inputfile
- * (python representation of api file generated by vppapigen).
- */
-
-// JAVA class reference cache
-$class_cache
-
-// JNI bindings
-$jni_implementations
-
-// Message handlers
-$msg_handlers
-
-// Registration of message handlers in vlib
-$handler_registration
-""")
-
-def generate_jvpp(func_list, plugin_name, inputfile):
-    """ Generates jvpp C file """
-    print "Generating jvpp C"
-
-    class_cache = generate_class_cache(func_list, plugin_name)
-    jni_impl = generate_jni_impl(func_list, plugin_name, inputfile)
-    msg_handlers = generate_msg_handlers(func_list, plugin_name, inputfile)
-    handler_registration = generate_handler_registration(func_list)
-
-    jvpp_c_file = open("jvpp_%s_gen.h" % plugin_name, 'w')
-    jvpp_c_file.write(jvpp_c_template.substitute(
-            inputfile=inputfile,
-            class_cache=class_cache,
-            jni_implementations=jni_impl,
-            msg_handlers=msg_handlers,
-            handler_registration=handler_registration))
-    jvpp_c_file.flush()
-    jvpp_c_file.close()
-
diff --git a/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py b/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py
deleted file mode 100644
index ac096a71..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/jvpp_callback_facade_gen.py
+++ /dev/null
@@ -1,324 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os, util
-from string import Template
-
-import callback_gen
-import dto_gen
-
-jvpp_ifc_template = Template("""
-package $plugin_package.$callback_facade_package;
-
-/**
- * 
Callback Java API representation of $plugin_package plugin.
- * 
It was generated by jvpp_callback_facade_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface CallbackJVpp${plugin_name} extends $base_package.$notification_package.NotificationRegistryProvider, java.lang.AutoCloseable {
-
-    // TODO add send
-
-$methods
-}
-""")
-
-jvpp_impl_template = Template("""
-package $plugin_package.$callback_facade_package;
-
-/**
- * 
Default implementation of Callback${plugin_name}JVpp interface.
- * 
It was generated by jvpp_callback_facade_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public final class CallbackJVpp${plugin_name}Facade implements CallbackJVpp${plugin_name} {
-
-    private final $plugin_package.JVpp${plugin_name} jvpp;
-    private final java.util.Map callbacks;
-    private final $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl notificationRegistry = new $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl();
-    /**
-     * 
Create CallbackJVpp${plugin_name}Facade object for provided JVpp instance.
-     * Constructor internally creates CallbackJVppFacadeCallback class for processing callbacks
-     * and then connects to provided JVpp instance
-     *
-     * @param jvpp provided $base_package.JVpp instance
-     *
-     * @throws java.io.IOException in case instance cannot connect to JVPP
-     */
-    public CallbackJVpp${plugin_name}Facade(final $base_package.JVppRegistry registry, final $plugin_package.JVpp${plugin_name} jvpp) throws java.io.IOException {
-        this.jvpp = java.util.Objects.requireNonNull(jvpp,"jvpp is null");
-        this.callbacks = new java.util.HashMap<>();
-        java.util.Objects.requireNonNull(registry, "JVppRegistry should not be null");
-        registry.register(jvpp, new CallbackJVpp${plugin_name}FacadeCallback(this.callbacks, notificationRegistry));
-    }
-
-    @Override
-    public $plugin_package.$notification_package.${plugin_name}NotificationRegistry getNotificationRegistry() {
-        return notificationRegistry;
-    }
-
-    @Override
-    public void close() throws Exception {
-        jvpp.close();
-    }
-
-    // TODO add send()
-
-$methods
-}
-""")
-
-method_template = Template(
-    """    void $name($plugin_package.$dto_package.$request request, $plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException;""")
-
-method_impl_template = Template("""    public final void $name($plugin_package.$dto_package.$request request, $plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException {
-        synchronized (callbacks) {
-            callbacks.put(jvpp.$name(request), callback);
-        }
-    }
-""")
-
-no_arg_method_template = Template("""    void $name($plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException;""")
-no_arg_method_impl_template = Template("""    public final void $name($plugin_package.$callback_package.$callback callback) throws $base_package.VppInvocationException {
-        synchronized (callbacks) {
-            callbacks.put(jvpp.$name(), callback);
-        }
-    }
-""")
-
-
-def generate_jvpp(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, callback_facade_package, inputfile):
-    """ Generates callback facade """
-    print "Generating JVpp callback facade"
-
-    if os.path.exists(callback_facade_package):
-        util.remove_folder(callback_facade_package)
-
-    os.mkdir(callback_facade_package)
-
-    methods = []
-    methods_impl = []
-    for func in func_list:
-
-        if util.is_notification(func['name']) or util.is_ignored(func['name']):
-            continue
-
-        camel_case_name = util.underscore_to_camelcase(func['name'])
-        camel_case_name_upper = util.underscore_to_camelcase_upper(func['name'])
-        if util.is_reply(camel_case_name) or util.is_control_ping(camel_case_name):
-            continue
-
-        # Strip suffix for dump calls
-        callback_type = get_request_name(camel_case_name_upper, func['name']) + callback_gen.callback_suffix
-
-        if len(func['args']) == 0:
-            methods.append(no_arg_method_template.substitute(name=camel_case_name,
-                                                             base_package=base_package,
-                                                             plugin_package=plugin_package,
-                                                             dto_package=dto_package,
-                                                             callback_package=callback_package,
-                                                             callback=callback_type))
-            methods_impl.append(no_arg_method_impl_template.substitute(name=camel_case_name,
-                                                                       base_package=base_package,
-                                                                       plugin_package=plugin_package,
-                                                                       dto_package=dto_package,
-                                                                       callback_package=callback_package,
-                                                                       callback=callback_type))
-        else:
-            methods.append(method_template.substitute(name=camel_case_name,
-                                                      request=camel_case_name_upper,
-                                                      base_package=base_package,
-                                                      plugin_package=plugin_package,
-                                                      dto_package=dto_package,
-                                                      callback_package=callback_package,
-                                                      callback=callback_type))
-            methods_impl.append(method_impl_template.substitute(name=camel_case_name,
-                                                                request=camel_case_name_upper,
-                                                                base_package=base_package,
-                                                                plugin_package=plugin_package,
-                                                                dto_package=dto_package,
-                                                                callback_package=callback_package,
-                                                                callback=callback_type))
-
-    join = os.path.join(callback_facade_package, "CallbackJVpp%s.java" % plugin_name)
-    jvpp_file = open(join, 'w')
-    jvpp_file.write(
-        jvpp_ifc_template.substitute(inputfile=inputfile,
-                                     methods="\n".join(methods),
-                                     base_package=base_package,
-                                     plugin_package=plugin_package,
-                                     plugin_name=plugin_name,
-                                     dto_package=dto_package,
-                                     notification_package=notification_package,
-                                     callback_facade_package=callback_facade_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-    jvpp_file = open(os.path.join(callback_facade_package, "CallbackJVpp%sFacade.java" % plugin_name), 'w')
-    jvpp_file.write(jvpp_impl_template.substitute(inputfile=inputfile,
-                                                  methods="\n".join(methods_impl),
-                                                  base_package=base_package,
-                                                  plugin_package=plugin_package,
-                                                  plugin_name=plugin_name,
-                                                  dto_package=dto_package,
-                                                  notification_package=notification_package,
-                                                  callback_package=callback_package,
-                                                  callback_facade_package=callback_facade_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-    generate_callback(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, callback_facade_package, inputfile)
-
-
-jvpp_facade_callback_template = Template("""
-package $plugin_package.$callback_facade_package;
-
-/**
- * 
Implementation of JVppGlobalCallback interface for Java Callback API.
- * 
It was generated by jvpp_callback_facade_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public final class CallbackJVpp${plugin_name}FacadeCallback implements $plugin_package.$callback_package.JVpp${plugin_name}GlobalCallback {
-
-    private final java.util.Map requests;
-    private final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback;
-    private static final java.util.logging.Logger LOG = java.util.logging.Logger.getLogger(CallbackJVpp${plugin_name}FacadeCallback.class.getName());
-
-    public CallbackJVpp${plugin_name}FacadeCallback(final java.util.Map requestMap,
-                                      final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback) {
-        this.requests = requestMap;
-        this.notificationCallback = notificationCallback;
-    }
-
-    @Override
-    public void onError($base_package.VppCallbackException reply) {
-
-        $base_package.$callback_package.JVppCallback failedCall;
-        synchronized(requests) {
-            failedCall = requests.remove(reply.getCtxId());
-        }
-
-        if(failedCall != null) {
-            try {
-                failedCall.onError(reply);
-            } catch(RuntimeException ex) {
-                ex.addSuppressed(reply);
-                LOG.log(java.util.logging.Level.WARNING, String.format("Callback: %s failed while handling exception: %s", failedCall, reply), ex);
-            }
-        }
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public void onControlPingReply($base_package.$dto_package.ControlPingReply reply) {
-
-        $base_package.$callback_package.ControlPingCallback callback;
-        synchronized(requests) {
-            callback = ($base_package.$callback_package.ControlPingCallback) requests.remove(reply.context);
-        }
-
-        if(callback != null) {
-            callback.onControlPingReply(reply);
-        }
-    }
-
-$methods
-}
-""")
-
-jvpp_facade_callback_method_template = Template("""
-    @Override
-    @SuppressWarnings("unchecked")
-    public void on$callback_dto($plugin_package.$dto_package.$callback_dto reply) {
-
-        $plugin_package.$callback_package.$callback callback;
-        synchronized(requests) {
-            callback = ($plugin_package.$callback_package.$callback) requests.remove(reply.context);
-        }
-
-        if(callback != null) {
-            callback.on$callback_dto(reply);
-        }
-    }
-""")
-
-jvpp_facade_callback_notification_method_template = Template("""
-    @Override
-    @SuppressWarnings("unchecked")
-    public void on$callback_dto($plugin_package.$dto_package.$callback_dto notification) {
-        notificationCallback.on$callback_dto(notification);
-    }
-""")
-
-
-def generate_callback(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, callback_facade_package, inputfile):
-    callbacks = []
-    for func in func_list:
-
-        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
-
-        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_name_with_suffix):
-            continue
-
-        if util.is_reply(camel_case_name_with_suffix):
-            callbacks.append(jvpp_facade_callback_method_template.substitute(plugin_package=plugin_package,
-                                                                             dto_package=dto_package,
-                                                                             callback_package=callback_package,
-                                                                             callback=util.remove_reply_suffix(camel_case_name_with_suffix) + callback_gen.callback_suffix,
-                                                                             callback_dto=camel_case_name_with_suffix))
-
-        if util.is_notification(func["name"]):
-            with_notification_suffix = util.add_notification_suffix(camel_case_name_with_suffix)
-            callbacks.append(jvpp_facade_callback_notification_method_template.substitute(plugin_package=plugin_package,
-                                                                             dto_package=dto_package,
-                                                                             callback_package=callback_package,
-                                                                             callback=with_notification_suffix + callback_gen.callback_suffix,
-                                                                             callback_dto=with_notification_suffix))
-
-    jvpp_file = open(os.path.join(callback_facade_package, "CallbackJVpp%sFacadeCallback.java" % plugin_name), 'w')
-    jvpp_file.write(jvpp_facade_callback_template.substitute(inputfile=inputfile,
-                                                             base_package=base_package,
-                                                             plugin_package=plugin_package,
-                                                             plugin_name=plugin_name,
-                                                             dto_package=dto_package,
-                                                             notification_package=notification_package,
-                                                             callback_package=callback_package,
-                                                             methods="".join(callbacks),
-                                                             callback_facade_package=callback_facade_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-
-# Returns request name or special one from unconventional_naming_rep_req map
-def get_request_name(camel_case_dto_name, func_name):
-    if func_name in reverse_dict(util.unconventional_naming_rep_req):
-        request_name = util.underscore_to_camelcase_upper(reverse_dict(util.unconventional_naming_rep_req)[func_name])
-    else:
-        request_name = camel_case_dto_name
-    return remove_suffix(request_name)
-
-
-def reverse_dict(map):
-    return dict((v, k) for k, v in map.iteritems())
-
-
-def remove_suffix(name):
-    if util.is_reply(name):
-        return util.remove_reply_suffix(name)
-    else:
-        if util.is_dump(name):
-            return util.remove_suffix(name, util.dump_suffix)
-        else:
-            return name
diff --git a/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py b/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py
deleted file mode 100644
index ebb840f7..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/jvpp_future_facade_gen.py
+++ /dev/null
@@ -1,331 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from string import Template
-
-import dto_gen
-import util
-
-jvpp_facade_callback_template = Template("""
-package $plugin_package.$future_package;
-
-/**
- * 
Async facade callback setting values to future objects
- * 
It was generated by jvpp_future_facade_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public final class FutureJVpp${plugin_name}FacadeCallback implements $plugin_package.$callback_package.JVpp${plugin_name}GlobalCallback {
-
-    private final java.util.Map>> requests;
-    private final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback;
-
-    public FutureJVpp${plugin_name}FacadeCallback(
-        final java.util.Map>> requestMap,
-        final $plugin_package.$notification_package.Global${plugin_name}NotificationCallback notificationCallback) {
-        this.requests = requestMap;
-        this.notificationCallback = notificationCallback;
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public void onError($base_package.VppCallbackException reply) {
-        final java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply> completableFuture;
-
-        synchronized(requests) {
-            completableFuture = (java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply>) requests.get(reply.getCtxId());
-        }
-
-        if(completableFuture != null) {
-            completableFuture.completeExceptionally(reply);
-
-            synchronized(requests) {
-                requests.remove(reply.getCtxId());
-            }
-        }
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public void onControlPingReply($base_package.$dto_package.ControlPingReply reply) {
-        final java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply> completableFuture;
-
-        synchronized(requests) {
-            completableFuture = (java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply>) requests.get(reply.context);
-        }
-
-        if(completableFuture != null) {
-            // Finish dump call
-            if (completableFuture instanceof $base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture) {
-                completableFuture.complete((($base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture) completableFuture).getReplyDump());
-                // Remove future mapped to dump call context id
-                synchronized(requests) {
-                    requests.remove((($base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture) completableFuture).getContextId());
-                }
-            } else {
-                completableFuture.complete(reply);
-            }
-
-            synchronized(requests) {
-                requests.remove(reply.context);
-            }
-        }
-    }
-
-$methods
-}
-""")
-
-jvpp_facade_callback_method_template = Template("""
-    @Override
-    @SuppressWarnings("unchecked")
-    public void on$callback_dto($plugin_package.$dto_package.$callback_dto reply) {
-        final java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply> completableFuture;
-
-        synchronized(requests) {
-            completableFuture = (java.util.concurrent.CompletableFuture<$base_package.$dto_package.JVppReply>) requests.get(reply.context);
-        }
-
-        if(completableFuture != null) {
-            completableFuture.complete(reply);
-
-            synchronized(requests) {
-                requests.remove(reply.context);
-            }
-        }
-    }
-""")
-
-jvpp_facade_callback_notification_method_template = Template("""
-    @Override
-    public void on$callback_dto($plugin_package.$dto_package.$callback_dto notification) {
-        notificationCallback.on$callback_dto(notification);
-    }
-""")
-
-jvpp_facade_details_callback_method_template = Template("""
-    @Override
-    @SuppressWarnings("unchecked")
-    public void on$callback_dto($plugin_package.$dto_package.$callback_dto reply) {
-        final $base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture<$plugin_package.$dto_package.$callback_dto_reply_dump> completableFuture;
-
-        synchronized(requests) {
-            completableFuture = ($base_package.$future_package.AbstractFutureJVppInvoker.CompletableDumpFuture<$plugin_package.$dto_package.$callback_dto_reply_dump>) requests.get(reply.context);
-        }
-
-        if(completableFuture != null) {
-            completableFuture.getReplyDump().$callback_dto_field.add(reply);
-        }
-    }
-""")
-
-
-def generate_jvpp(func_list, base_package, plugin_package, plugin_name, dto_package, callback_package, notification_package, future_facade_package, inputfile):
-    """ Generates JVpp interface and JNI implementation """
-    print "Generating JVpp future facade"
-
-    if not os.path.exists(future_facade_package):
-        raise Exception("%s folder is missing" % future_facade_package)
-
-    methods = []
-    methods_impl = []
-    callbacks = []
-    for func in func_list:
-        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
-
-        if util.is_ignored(func['name']) or util.is_control_ping(camel_case_name_with_suffix):
-            continue
-
-        if not util.is_reply(camel_case_name_with_suffix) and not util.is_notification(func['name']):
-            continue
-
-        camel_case_method_name = util.underscore_to_camelcase(func['name'])
-
-        if not util.is_notification(func["name"]):
-            camel_case_request_method_name = util.remove_reply_suffix(util.underscore_to_camelcase(func['name']))
-            if util.is_details(camel_case_name_with_suffix):
-                camel_case_reply_name = get_standard_dump_reply_name(util.underscore_to_camelcase_upper(func['name']),
-                                                                     func['name'])
-                callbacks.append(jvpp_facade_details_callback_method_template.substitute(base_package=base_package,
-                                                                                         plugin_package=plugin_package,
-                                                                                         dto_package=dto_package,
-                                                                                         callback_dto=camel_case_name_with_suffix,
-                                                                                         callback_dto_field=camel_case_method_name,
-                                                                                         callback_dto_reply_dump=camel_case_reply_name + dto_gen.dump_dto_suffix,
-                                                                                         future_package=future_facade_package))
-
-                methods.append(future_jvpp_method_template.substitute(plugin_package=plugin_package,
-                                                                      dto_package=dto_package,
-                                                                      method_name=camel_case_request_method_name +
-                                                                                  util.underscore_to_camelcase_upper(util.dump_suffix),
-                                                                      reply_name=camel_case_reply_name + dto_gen.dump_dto_suffix,
-                                                                      request_name=util.remove_reply_suffix(camel_case_reply_name) +
-                                                                                   util.underscore_to_camelcase_upper(util.dump_suffix)))
-                methods_impl.append(future_jvpp_dump_method_impl_template.substitute(plugin_package=plugin_package,
-                                                                                     dto_package=dto_package,
-                                                                                     method_name=camel_case_request_method_name +
-                                                                                                 util.underscore_to_camelcase_upper(util.dump_suffix),
-                                                                                     reply_name=camel_case_reply_name + dto_gen.dump_dto_suffix,
-                                                                                     request_name=util.remove_reply_suffix(camel_case_reply_name) +
-                                                                                                  util.underscore_to_camelcase_upper(util.dump_suffix)))
-            else:
-                request_name = util.underscore_to_camelcase_upper(util.unconventional_naming_rep_req[func['name']]) \
-                    if func['name'] in util.unconventional_naming_rep_req else util.remove_reply_suffix(camel_case_name_with_suffix)
-
-                methods.append(future_jvpp_method_template.substitute(plugin_package=plugin_package,
-                                                                      dto_package=dto_package,
-                                                                      method_name=camel_case_request_method_name,
-                                                                      reply_name=camel_case_name_with_suffix,
-                                                                      request_name=request_name))
-                methods_impl.append(future_jvpp_method_impl_template.substitute(plugin_package=plugin_package,
-                                                                                dto_package=dto_package,
-                                                                                method_name=camel_case_request_method_name,
-                                                                                reply_name=camel_case_name_with_suffix,
-                                                                                request_name=request_name))
-
-                callbacks.append(jvpp_facade_callback_method_template.substitute(base_package=base_package,
-                                                                                 plugin_package=plugin_package,
-                                                                                 dto_package=dto_package,
-                                                                                 callback_dto=camel_case_name_with_suffix))
-
-        if util.is_notification(func["name"]):
-            callbacks.append(jvpp_facade_callback_notification_method_template.substitute(plugin_package=plugin_package,
-                                                                                          dto_package=dto_package,
-                                                                                          callback_dto=util.add_notification_suffix(camel_case_name_with_suffix)))
-
-    jvpp_file = open(os.path.join(future_facade_package, "FutureJVpp%sFacadeCallback.java" % plugin_name), 'w')
-    jvpp_file.write(jvpp_facade_callback_template.substitute(inputfile=inputfile,
-                                                             base_package=base_package,
-                                                             plugin_package=plugin_package,
-                                                             plugin_name=plugin_name,
-                                                             dto_package=dto_package,
-                                                             notification_package=notification_package,
-                                                             callback_package=callback_package,
-                                                             methods="".join(callbacks),
-                                                             future_package=future_facade_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-    jvpp_file = open(os.path.join(future_facade_package, "FutureJVpp%s.java" % plugin_name), 'w')
-    jvpp_file.write(future_jvpp_template.substitute(inputfile=inputfile,
-                                                    base_package=base_package,
-                                                    plugin_package=plugin_package,
-                                                    plugin_name=plugin_name,
-                                                    notification_package=notification_package,
-                                                    methods="".join(methods),
-                                                    future_package=future_facade_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-    jvpp_file = open(os.path.join(future_facade_package, "FutureJVpp%sFacade.java" % plugin_name), 'w')
-    jvpp_file.write(future_jvpp_facade_template.substitute(inputfile=inputfile,
-                                                           base_package=base_package,
-                                                           plugin_package=plugin_package,
-                                                           plugin_name=plugin_name,
-                                                           dto_package=dto_package,
-                                                           notification_package=notification_package,
-                                                           methods="".join(methods_impl),
-                                                           future_package=future_facade_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-
-future_jvpp_template = Template('''
-package $plugin_package.$future_package;
-
-/**
- * 
Async facade extension adding specific methods for each request invocation
- * 
It was generated by jvpp_future_facade_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface FutureJVpp${plugin_name} extends $base_package.$future_package.FutureJVppInvoker {
-$methods
-
-    @Override
-    public $plugin_package.$notification_package.${plugin_name}NotificationRegistry getNotificationRegistry();
-
-}
-''')
-
-future_jvpp_method_template = Template('''
-    java.util.concurrent.CompletionStage<$plugin_package.$dto_package.$reply_name> $method_name($plugin_package.$dto_package.$request_name request);
-''')
-
-
-future_jvpp_facade_template = Template('''
-package $plugin_package.$future_package;
-
-/**
- * 
Implementation of FutureJVpp based on AbstractFutureJVppInvoker
- * 
It was generated by jvpp_future_facade_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public class FutureJVpp${plugin_name}Facade extends $base_package.$future_package.AbstractFutureJVppInvoker implements FutureJVpp${plugin_name} {
-
-    private final $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl notificationRegistry = new $plugin_package.$notification_package.${plugin_name}NotificationRegistryImpl();
-
-    /**
-     * 
Create FutureJVpp${plugin_name}Facade object for provided JVpp instance.
-     * Constructor internally creates FutureJVppFacadeCallback class for processing callbacks
-     * and then connects to provided JVpp instance
-     *
-     * @param jvpp provided $base_package.JVpp instance
-     *
-     * @throws java.io.IOException in case instance cannot connect to JVPP
-     */
-    public FutureJVpp${plugin_name}Facade(final $base_package.JVppRegistry registry, final $base_package.JVpp jvpp) throws java.io.IOException {
-        super(jvpp, registry, new java.util.HashMap<>());
-        java.util.Objects.requireNonNull(registry, "JVppRegistry should not be null");
-        registry.register(jvpp, new FutureJVpp${plugin_name}FacadeCallback(getRequests(), notificationRegistry));
-    }
-
-    @Override
-    public $plugin_package.$notification_package.${plugin_name}NotificationRegistry getNotificationRegistry() {
-        return notificationRegistry;
-    }
-
-$methods
-}
-''')
-
-future_jvpp_method_impl_template = Template('''
-    @Override
-    public java.util.concurrent.CompletionStage<$plugin_package.$dto_package.$reply_name> $method_name($plugin_package.$dto_package.$request_name request) {
-        return send(request);
-    }
-''')
-
-future_jvpp_dump_method_impl_template = Template('''
-    @Override
-    public java.util.concurrent.CompletionStage<$plugin_package.$dto_package.$reply_name> $method_name($plugin_package.$dto_package.$request_name request) {
-        return send(request, new $plugin_package.$dto_package.$reply_name());
-    }
-''')
-
-
-# Returns request name or special one from unconventional_naming_rep_req map
-def get_standard_dump_reply_name(camel_case_dto_name, func_name):
-    # FIXME this is a hotfix for sub-details callbacks
-    # FIXME also for L2FibTableEntry
-    # It's all because unclear mapping between
-    #  request -> reply,
-    #  dump -> reply, details,
-    #  notification_start -> reply, notifications
-
-    # vpe.api needs to be "standardized" so we can parse the information and create maps before generating java code
-    suffix = func_name.split("_")[-1]
-    return util.underscore_to_camelcase_upper(
-        util.unconventional_naming_rep_req[func_name]) + util.underscore_to_camelcase_upper(suffix) if func_name in util.unconventional_naming_rep_req \
-        else camel_case_dto_name
diff --git a/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py b/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py
deleted file mode 100644
index 4146c141..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/jvpp_impl_gen.py
+++ /dev/null
@@ -1,219 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os, util
-from string import Template
-
-jvpp_ifc_template = Template("""
-package $plugin_package;
-
-/**
- * 
Java representation of plugin's api file.
- * 
It was generated by jvpp_impl_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface JVpp${plugin_name} extends $base_package.JVpp {
-
-    /**
-     * Generic dispatch method for sending requests to VPP
-     *
-     * @throws io.fd.vpp.jvpp.VppInvocationException if send request had failed
-     */
-    int send($base_package.$dto_package.JVppRequest request) throws io.fd.vpp.jvpp.VppInvocationException;
-
-$methods
-}
-""")
-
-jvpp_impl_template = Template("""
-package $plugin_package;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Set;
-import java.util.logging.Logger;
-import $base_package.callback.JVppCallback;
-import $base_package.VppConnection;
-import $base_package.JVppRegistry;
-
-/**
- * 
Default implementation of JVpp interface.
- * 
It was generated by jvpp_impl_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public final class JVpp${plugin_name}Impl implements $plugin_package.JVpp${plugin_name} {
-
-    private final static Logger LOG = Logger.getLogger(JVpp${plugin_name}Impl.class.getName());
-    private static final String LIBNAME = "libjvpp_${plugin_name_underscore}.so.0.0.0";
-
-    // FIXME using NativeLibraryLoader makes load fail could not find (WantInterfaceEventsReply).
-    static {
-        try {
-            loadLibrary();
-        } catch (Exception e) {
-            LOG.severe("Can't find jvpp jni library: " + LIBNAME);
-            throw new ExceptionInInitializerError(e);
-        }
-    }
-
-    private static void loadStream(final InputStream is) throws IOException {
-        final Set perms = PosixFilePermissions.fromString("rwxr-x---");
-        final Path p = Files.createTempFile(LIBNAME, null, PosixFilePermissions.asFileAttribute(perms));
-        try {
-            Files.copy(is, p, StandardCopyOption.REPLACE_EXISTING);
-
-            try {
-                Runtime.getRuntime().load(p.toString());
-            } catch (UnsatisfiedLinkError e) {
-                throw new IOException("Failed to load library " + p, e);
-            }
-        } finally {
-            try {
-                Files.deleteIfExists(p);
-            } catch (IOException e) {
-            }
-        }
-    }
-
-    private static void loadLibrary() throws IOException {
-        try (final InputStream is = JVpp${plugin_name}Impl.class.getResourceAsStream('/' + LIBNAME)) {
-            if (is == null) {
-                throw new IOException("Failed to open library resource " + LIBNAME);
-            }
-            loadStream(is);
-        }
-    }
-
-    private VppConnection connection;
-    private JVppRegistry registry;
-
-    private static native void init0(final JVppCallback callback, final long queueAddress, final int clientIndex);
-    @Override
-    public void init(final JVppRegistry registry, final JVppCallback callback, final long queueAddress, final int clientIndex) {
-        this.registry = java.util.Objects.requireNonNull(registry, "registry should not be null");
-        this.connection = java.util.Objects.requireNonNull(registry.getConnection(), "connection should not be null");
-        connection.checkActive();
-        init0(callback, queueAddress, clientIndex);
-    }
-
-    private static native void close0();
-    @Override
-    public void close() {
-        close0();
-    }
-
-    @Override
-    public int send($base_package.$dto_package.JVppRequest request) throws io.fd.vpp.jvpp.VppInvocationException {
-        return request.send(this);
-    }
-
-    @Override
-    public final int controlPing(final io.fd.vpp.jvpp.dto.ControlPing controlPing) throws io.fd.vpp.jvpp.VppInvocationException {
-        return registry.controlPing(JVpp${plugin_name}Impl.class);
-    }
-
-$methods
-}
-""")
-
-method_template = Template("""    int $name($plugin_package.$dto_package.$request request) throws io.fd.vpp.jvpp.VppInvocationException;""")
-method_native_template = Template(
-    """    private static native int ${name}0($plugin_package.$dto_package.$request request);""")
-method_impl_template = Template("""    public final int $name($plugin_package.$dto_package.$request request) throws io.fd.vpp.jvpp.VppInvocationException {
-        java.util.Objects.requireNonNull(request,"Null request object");
-        connection.checkActive();
-        int result=${name}0(request);
-        if(result<0){
-            throw new io.fd.vpp.jvpp.VppInvocationException("${name}",result);
-        }
-        return result;
-    }
-""")
-
-no_arg_method_template = Template("""    int $name() throws io.fd.vpp.jvpp.VppInvocationException;""")
-no_arg_method_native_template = Template("""    private static native int ${name}0() throws io.fd.vpp.jvpp.VppInvocationException;""")
-no_arg_method_impl_template = Template("""    public final int $name() throws io.fd.vpp.jvpp.VppInvocationException {
-        connection.checkActive();
-        int result=${name}0();
-        if(result<0){
-            throw new io.fd.vpp.jvpp.VppInvocationException("${name}",result);
-        }
-        return result;
-    }
-""")
-
-
-def generate_jvpp(func_list, base_package, plugin_package, plugin_name_underscore, dto_package, inputfile):
-    """ Generates JVpp interface and JNI implementation """
-    print "Generating JVpp"
-    plugin_name = util.underscore_to_camelcase_upper(plugin_name_underscore)
-
-    methods = []
-    methods_impl = []
-    for func in func_list:
-
-        # Skip structures that are used only as notifications
-        if util.is_just_notification(func['name']) or util.is_ignored(func['name']):
-            continue
-
-        camel_case_name = util.underscore_to_camelcase(func['name'])
-        camel_case_name_upper = util.underscore_to_camelcase_upper(func['name'])
-        if util.is_reply(camel_case_name):
-            continue
-
-        if len(func['args']) == 0:
-            methods.append(no_arg_method_template.substitute(name=camel_case_name))
-            methods_impl.append(no_arg_method_native_template.substitute(name=camel_case_name))
-            methods_impl.append(no_arg_method_impl_template.substitute(name=camel_case_name))
-        else:
-            methods.append(method_template.substitute(name=camel_case_name,
-                                                      request=camel_case_name_upper,
-                                                      plugin_package=plugin_package,
-                                                      dto_package=dto_package))
-            methods_impl.append(method_native_template.substitute(name=camel_case_name,
-                                                                  request=camel_case_name_upper,
-                                                                  plugin_package=plugin_package,
-                                                                  dto_package=dto_package))
-            methods_impl.append(method_impl_template.substitute(name=camel_case_name,
-                                                                request=camel_case_name_upper,
-                                                                plugin_package=plugin_package,
-                                                                dto_package=dto_package))
-
-    jvpp_file = open("JVpp%s.java" % plugin_name, 'w')
-    jvpp_file.write(
-        jvpp_ifc_template.substitute(inputfile=inputfile,
-                                     methods="\n".join(methods),
-                                     base_package=base_package,
-                                     plugin_package=plugin_package,
-                                     plugin_name=plugin_name,
-                                     dto_package=dto_package))
-    jvpp_file.flush()
-    jvpp_file.close()
-
-    jvpp_file = open("JVpp%sImpl.java" % plugin_name, 'w')
-    jvpp_file.write(jvpp_impl_template.substitute(inputfile=inputfile,
-                                                  methods="\n".join(methods_impl),
-                                                  base_package=base_package,
-                                                  plugin_package=plugin_package,
-                                                  plugin_name=plugin_name,
-                                                  plugin_name_underscore=plugin_name_underscore,
-                                                  dto_package=dto_package))
-    jvpp_file.flush()
-    jvpp_file.close()
diff --git a/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py b/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py
deleted file mode 100644
index eb380fc8..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/notification_gen.py
+++ /dev/null
@@ -1,199 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import callback_gen
-import util
-from string import Template
-
-notification_registry_template = Template("""
-package $plugin_package.$notification_package;
-
-/**
- * 
Registry for notification callbacks defined in ${plugin_name}.
- * 
It was generated by notification_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface ${plugin_name}NotificationRegistry extends $base_package.$notification_package.NotificationRegistry {
-
-    $register_callback_methods
-
-    @Override
-    void close();
-}
-""")
-
-global_notification_callback_template = Template("""
-package $plugin_package.$notification_package;
-
-/**
- * 
Aggregated callback interface for notifications only.
- * 
It was generated by notification_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface Global${plugin_name}NotificationCallback$callbacks {
-
-}
-""")
-
-notification_registry_impl_template = Template("""
-package $plugin_package.$notification_package;
-
-/**
- * 
Notification registry delegating notification processing to registered callbacks.
- * 
It was generated by notification_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public final class ${plugin_name}NotificationRegistryImpl implements ${plugin_name}NotificationRegistry, Global${plugin_name}NotificationCallback {
-
-    // TODO add a special NotificationCallback interface and only allow those to be registered
-    private final java.util.concurrent.ConcurrentMap, $base_package.$callback_package.JVppNotificationCallback> registeredCallbacks =
-        new java.util.concurrent.ConcurrentHashMap<>();
-
-    $register_callback_methods
-    $handler_methods
-
-    @Override
-    public void close() {
-        registeredCallbacks.clear();
-    }
-}
-""")
-
-register_callback_impl_template = Template("""
-    public java.lang.AutoCloseable register$callback(final $plugin_package.$callback_package.$callback callback){
-        if(null != registeredCallbacks.putIfAbsent($plugin_package.$dto_package.$notification.class, callback)){
-            throw new IllegalArgumentException("Callback for " + $plugin_package.$dto_package.$notification.class +
-                "notification already registered");
-        }
-        return () -> registeredCallbacks.remove($plugin_package.$dto_package.$notification.class);
-    }
-""")
-
-handler_impl_template = Template("""
-    @Override
-    public void on$notification(
-        final $plugin_package.$dto_package.$notification notification) {
-        final $base_package.$callback_package.JVppNotificationCallback jVppNotificationCallback = registeredCallbacks.get($plugin_package.$dto_package.$notification.class);
-        if (null != jVppNotificationCallback) {
-            (($plugin_package.$callback_package.$callback) registeredCallbacks
-                .get($plugin_package.$dto_package.$notification.class))
-                .on$notification(notification);
-        }
-    }
-""")
-
-notification_provider_template = Template("""
-package $plugin_package.$notification_package;
-
- /**
- * Provides ${plugin_name}NotificationRegistry.
- * 
The file was generated by notification_gen.py based on $inputfile
- * 
(python representation of api file generated by vppapigen).
- */
-public interface ${plugin_name}NotificationRegistryProvider extends $base_package.$notification_package.NotificationRegistryProvider {
-
-    @Override
-    public ${plugin_name}NotificationRegistry getNotificationRegistry();
-}
-""")
-
-
-def generate_notification_registry(func_list, base_package, plugin_package, plugin_name, notification_package, callback_package, dto_package, inputfile):
-    """ Generates notification registry interface and implementation """
-    print "Generating Notification interfaces and implementation"
-
-    if not os.path.exists(notification_package):
-        raise Exception("%s folder is missing" % notification_package)
-
-    callbacks = []
-    register_callback_methods = []
-    register_callback_methods_impl = []
-    handler_methods = []
-    for func in func_list:
-
-        if not util.is_notification(func['name']):
-            continue
-
-        camel_case_name_with_suffix = util.underscore_to_camelcase_upper(func['name'])
-        notification_dto = util.add_notification_suffix(camel_case_name_with_suffix)
-        callback_ifc = notification_dto + callback_gen.callback_suffix
-        fully_qualified_callback_ifc = "{0}.{1}.{2}".format(plugin_package, callback_package, callback_ifc)
-        callbacks.append(fully_qualified_callback_ifc)
-
-        # TODO create NotificationListenerRegistration and return that instead of AutoCloseable to better indicate
-        # that the registration should be closed
-        register_callback_methods.append("java.lang.AutoCloseable register{0}({1} callback);"
-                                         .format(callback_ifc, fully_qualified_callback_ifc))
-        register_callback_methods_impl.append(register_callback_impl_template.substitute(plugin_package=plugin_package,
-                                                                                         callback_package=callback_package,
-                                                                                         dto_package=dto_package,
-                                                                                         notification=notification_dto,
-                                                                                         callback=callback_ifc))
-        handler_methods.append(handler_impl_template.substitute(base_package=base_package,
-                                                                plugin_package=plugin_package,
-                                                                callback_package=callback_package,
-                                                                dto_package=dto_package,
-                                                                notification=notification_dto,
-                                                                callback=callback_ifc))
-
-
-    callback_file = open(os.path.join(notification_package, "%sNotificationRegistry.java" % plugin_name), 'w')
-    callback_file.write(notification_registry_template.substitute(inputfile=inputfile,
-                                                                register_callback_methods="\n    ".join(register_callback_methods),
-                                                                base_package=base_package,
-                                                                plugin_package=plugin_package,
-                                                                plugin_name=plugin_name,
-                                                                notification_package=notification_package))
-    callback_file.flush()
-    callback_file.close()
-
-    callback_file = open(os.path.join(notification_package, "Global%sNotificationCallback.java" % plugin_name), 'w')
-
-    global_notification_callback_callbacks = ""
-    if (callbacks):
-        global_notification_callback_callbacks = " extends " + ", ".join(callbacks)
-
-    callback_file.write(global_notification_callback_template.substitute(inputfile=inputfile,
-                                                                       callbacks=global_notification_callback_callbacks,
-                                                                       plugin_package=plugin_package,
-                                                                       plugin_name=plugin_name,
-                                                                       notification_package=notification_package))
-    callback_file.flush()
-    callback_file.close()
-
-    callback_file = open(os.path.join(notification_package, "%sNotificationRegistryImpl.java" % plugin_name), 'w')
-    callback_file.write(notification_registry_impl_template.substitute(inputfile=inputfile,
-                                                                     callback_package=callback_package,
-                                                                     dto_package=dto_package,
-                                                                     register_callback_methods="".join(register_callback_methods_impl),
-                                                                     handler_methods="".join(handler_methods),
-                                                                     base_package=base_package,
-                                                                     plugin_package=plugin_package,
-                                                                     plugin_name=plugin_name,
-                                                                     notification_package=notification_package))
-    callback_file.flush()
-    callback_file.close()
-
-    callback_file = open(os.path.join(notification_package, "%sNotificationRegistryProvider.java" % plugin_name), 'w')
-    callback_file.write(notification_provider_template.substitute(inputfile=inputfile,
-                                                                     base_package=base_package,
-                                                                     plugin_package=plugin_package,
-                                                                     plugin_name=plugin_name,
-                                                                     notification_package=notification_package))
-    callback_file.flush()
-    callback_file.close()
-
diff --git a/vpp-api/java/jvpp/gen/jvppgen/types_gen.py b/vpp-api/java/jvpp/gen/jvppgen/types_gen.py
deleted file mode 100644
index d12fb3d7..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/types_gen.py
+++ /dev/null
@@ -1,227 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from string import Template
-
-import util
-import jni_gen
-import dto_gen
-
-type_template = Template("""
-package $plugin_package.$type_package;
-
-/**
- * 
This class represents $c_type_name type definition.
- * 
It was generated by types_gen.py based on $inputfile preparsed data:
- * 
-$docs
- * 

- */
-public final class $java_type_name {
-$fields
-$methods
-}
-""")
-
-field_template = Template("""    public $type $name;\n""")
-
-
-def generate_type_fields(type_definition):
-    """
-    Generates fields for class representing typeonly definition
-    :param type_definition: python representation of typeonly definition
-    :return: string representing class fields
-    """
-    fields = ""
-    for t in zip(type_definition['types'], type_definition['args']):
-        field_name = util.underscore_to_camelcase(t[1])
-        fields += field_template.substitute(type=util.jni_2_java_type_mapping[t[0]],
-                                            name=field_name)
-    return fields
-
-object_struct_setter_template = Template("""
-    {
-        jclass ${field_reference_name}Class = (*env)->FindClass(env, "${class_FQN}");
-        memset (&(mp->${c_name}), 0, sizeof (mp->${c_name}));
-        ${struct_initialization}
-    }
-""")
-
-object_array_struct_setter_template = Template("""
-    {
-        jclass ${field_reference_name}ArrayElementClass = (*env)->FindClass(env, "${class_FQN}");
-        if (${field_reference_name}) {
-            size_t _i;
-            jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-            ${field_length_check}
-            for (_i = 0; _i < cnt; _i++) {
-                jobject ${field_reference_name}ArrayElement = (*env)->GetObjectArrayElement(env, ${field_reference_name}, _i);
-                memset (&(mp->${c_name}[_i]), 0, sizeof (mp->${c_name}[_i]));
-                ${struct_initialization}
-            }
-        }
-    }
-""")
-
-object_dto_field_setter_template = Template("""
-    {
-        jclass ${field_reference_name}Class = (*env)->FindClass(env, "${class_FQN}");
-        jmethodID ${field_reference_name}Constructor = (*env)->GetMethodID(env, ${field_reference_name}Class, "", "()V");
-        jobject ${field_reference_name} = (*env)->NewObject(env, ${field_reference_name}Class,  ${field_reference_name}Constructor);
-        ${type_initialization}
-        (*env)->SetObjectField(env, dto, ${field_reference_name}FieldId, ${field_reference_name});
-    }
-""")
-
-object_array_dto_field_setter_template = Template("""
-    {
-        jclass ${field_reference_name}Class = (*env)->FindClass(env, "${class_FQN}");
-        jobjectArray ${field_reference_name} = (*env)->NewObjectArray(env, ${field_length}, ${field_reference_name}Class, 0);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            jmethodID ${field_reference_name}Constructor = (*env)->GetMethodID(env, ${field_reference_name}Class, "", "()V");
-            jobject ${field_reference_name}ArrayElement = (*env)->NewObject(env, ${field_reference_name}Class,  ${field_reference_name}Constructor);
-            ${type_initialization}
-            (*env)->SetObjectArrayElement(env, ${field_reference_name}, _i, ${field_reference_name}ArrayElement);
-        }
-        (*env)->SetObjectField(env, dto, ${field_reference_name}FieldId, ${field_reference_name});
-    }
-""")
-
-
-def generate_struct_initialization(type_def, c_name_prefix, object_name, indent):
-    struct_initialization = ""
-    # field identifiers
-    for t in zip(type_def['types'], type_def['args'], type_def['lengths']):
-        field_reference_name = "${c_name}" + util.underscore_to_camelcase_upper(t[1])
-        field_name = util.underscore_to_camelcase(t[1])
-        struct_initialization += jni_gen.jni_request_binding_for_type(field_type=t[0], c_name=c_name_prefix + t[1],
-                                                                     field_reference_name=field_reference_name,
-                                                                     field_name=field_name,
-                                                                     field_length=t[2][0],
-                                                                     is_variable_len_array=t[2][1],
-                                                                     object_name=object_name)
-    return indent + struct_initialization.replace('\n', '\n' + indent)
-
-
-def generate_type_setter(handler_name, type_def, c_name_prefix, object_name, indent):
-    type_initialization = ""
-    for t in zip(type_def['types'], type_def['args'], type_def['lengths']):
-        field_length = t[2][0]
-        is_variable_len_array = t[2][1]
-        length_field_type = None
-        if is_variable_len_array:
-            length_field_type = type_def['types'][type_def['args'].index(field_length)]
-        type_initialization += jni_gen.jni_reply_handler_for_type(handler_name=handler_name,
-                                                                  ref_name="${field_reference_name}",
-                                                                  field_type=t[0], c_name=c_name_prefix + t[1],
-                                                                  field_reference_name="${c_name}" + util.underscore_to_camelcase_upper(t[1]),
-                                                                  field_name=util.underscore_to_camelcase(t[1]),
-                                                                  field_length=field_length,
-                                                                  is_variable_len_array=is_variable_len_array,
-                                                                  length_field_type=length_field_type,
-                                                                  object_name=object_name)
-    return indent + type_initialization.replace('\n', '\n' + indent)
-
-
-def generate_types(types_list, plugin_package, types_package, inputfile):
-    """
-    Generates Java representation of custom types defined in api file.
-    """
-
-    #
-    if not types_list:
-        print "Skipping custom types generation (%s does not define custom types)." % inputfile
-        return
-
-    print "Generating custom types"
-
-    if not os.path.exists(types_package):
-        raise Exception("%s folder is missing" % types_package)
-
-    for type in types_list:
-        c_type_name = type['name']
-        java_type_name = util.underscore_to_camelcase_upper(type['name'])
-        dto_path = os.path.join(types_package, java_type_name + ".java")
-
-        fields = generate_type_fields(type)
-
-        dto_file = open(dto_path, 'w')
-        dto_file.write(type_template.substitute(plugin_package=plugin_package,
-                                                type_package=types_package,
-                                                c_type_name=c_type_name,
-                                                inputfile=inputfile,
-                                                docs=util.api_message_to_javadoc(type),
-                                                java_type_name=java_type_name,
-                                                fields=fields,
-                                                methods=dto_gen.generate_dto_base_methods(java_type_name, type)
-                                                ))
-
-        # update type mappings:
-        # todo fix vpe.api to use type_name instead of vl_api_type_name_t
-        type_name = "vl_api_" + c_type_name + "_t"
-        java_fqn = "%s.%s.%s" % (plugin_package, types_package, java_type_name)
-        util.vpp_2_jni_type_mapping[type_name] = "jobject"
-        util.vpp_2_jni_type_mapping[type_name + "[]"] = "jobjectArray"
-        util.jni_2_java_type_mapping[type_name] = java_fqn
-        util.jni_2_java_type_mapping[type_name + "[]"] = java_fqn + "[]"
-        jni_name = java_fqn.replace('.', "/")
-        jni_signature = "L" + jni_name + ";"
-        util.jni_2_signature_mapping[type_name] = "L" + jni_name + ";"
-        util.jni_2_signature_mapping[type_name + "[]"] = "[" + jni_signature
-        util.jni_field_accessors[type_name] = "ObjectField"
-        util.jni_field_accessors[type_name + "[]"] = "ObjectField"
-
-        jni_gen.struct_setter_templates[type_name] = Template(
-                object_struct_setter_template.substitute(
-                        c_name="${c_name}",
-                        field_reference_name="${field_reference_name}",
-                        class_FQN=jni_name,
-                        struct_initialization=generate_struct_initialization(type, "${c_name}.",
-                                                                           "${field_reference_name}", ' ' * 4))
-        )
-
-        jni_gen.struct_setter_templates[type_name+ "[]"] = Template(
-                object_array_struct_setter_template.substitute(
-                        c_name="${c_name}",
-                        field_reference_name="${field_reference_name}",
-                        field_length_check="${field_length_check}",
-                        class_FQN=jni_name,
-                        struct_initialization=generate_struct_initialization(type, "${c_name}[_i].",
-                                                                           "${field_reference_name}ArrayElement", ' ' * 8))
-        )
-
-        jni_gen.dto_field_setter_templates[type_name] = Template(
-                object_dto_field_setter_template.substitute(
-                        field_reference_name="${field_reference_name}",
-                        field_length="${field_length}",
-                        class_FQN=jni_name,
-                        type_initialization=generate_type_setter(c_type_name, type, "${c_name}.",
-                                                                 "${field_reference_name}", ' ' * 4))
-        )
-
-        jni_gen.dto_field_setter_templates[type_name + "[]"] = Template(
-                object_array_dto_field_setter_template.substitute(
-                        field_reference_name="${field_reference_name}",
-                        field_length="${field_length}",
-                        class_FQN=jni_name,
-                        type_initialization=generate_type_setter(c_type_name, type, "${c_name}[_i].",
-                                                                 "${field_reference_name}ArrayElement", ' ' * 8))
-        )
-
-        dto_file.flush()
-        dto_file.close()
-
diff --git a/vpp-api/java/jvpp/gen/jvppgen/util.py b/vpp-api/java/jvpp/gen/jvppgen/util.py
deleted file mode 100644
index fc971c17..00000000
--- a/vpp-api/java/jvpp/gen/jvppgen/util.py
+++ /dev/null
@@ -1,220 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os, pprint
-from os import removedirs
-
-
-def underscore_to_camelcase(name):
-    name = name.title().replace("_", "")
-    return name[0].lower() + name[1:]
-
-
-def underscore_to_camelcase_upper(name):
-    name = name.title().replace("_", "")
-    return name[0].upper() + name[1:]
-
-
-def remove_folder(folder):
-    """ Remove folder with all its files """
-    for root, dirs, files in os.walk(folder, topdown=False):
-        for name in files:
-            os.remove(os.path.join(root, name))
-        removedirs(folder)
-
-
-reply_suffixes = ("reply", "details", "l2fibtableentry")
-
-
-def is_reply(name):
-    return name.lower().endswith(reply_suffixes)
-
-
-def is_details(name):
-    return name.lower().endswith(reply_suffixes[1]) or name.lower().endswith(reply_suffixes[2])
-
-
-def is_retval_field(name):
-    return name == 'retval'
-
-dump_suffix = "dump"
-
-
-def is_dump(name):
-    return name.lower().endswith(dump_suffix)
-
-
-def get_reply_suffix(name):
-    for reply_suffix in reply_suffixes:
-        if name.lower().endswith(reply_suffix):
-            if reply_suffix == reply_suffixes[2]:
-                # FIXME workaround for l2_fib_table_entry
-                return 'entry'
-            else:
-                return reply_suffix
-
-# Mapping according to:
-# http://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/types.html
-#
-# Unsigned types are converted to signed java types that have the same size.
-# It is the API user responsibility to interpret them correctly.
-jni_2_java_type_mapping = {'u8': 'byte',
-                           'u8[]': 'byte[]',
-                           'i8': 'byte',
-                           'i8[]': 'byte[]',
-                           'u16': 'short',
-                           'u16[]': 'short[]',
-                           'i16': 'short',
-                           'i16[]': 'short[]',
-                           'u32': 'int',
-                           'u32[]': 'int[]',
-                           'i32': 'int',
-                           'i32[]': 'int[]',
-                           'u64': 'long',
-                           'u64[]': 'long[]',
-                           'i64': 'long',
-                           'i64[]': 'long[]',
-                           'f64': 'double',
-                           'f64[]': 'double[]'
-                           }
-
-vpp_2_jni_type_mapping = {'u8': 'jbyte',
-                          'u8[]': 'jbyteArray',
-                          'i8': 'jbyte',
-                          'u8[]': 'jbyteArray',
-                          'u16': 'jshort',
-                          'u16[]': 'jshortArray',
-                          'i16': 'jshort',
-                          'i16[]': 'jshortArray',
-                          'u32': 'jint',
-                          'u32[]': 'jintArray',
-                          'i32': 'jint',
-                          'i32[]': 'jintArray',
-                          'u64': 'jlong',
-                          'u64[]': 'jlongArray',
-                          'i64': 'jlong',
-                          'i64[]': 'jlongArray',
-                          'f64': 'jdouble',
-                          'f64[]': 'jdoubleArray'
-                          }
-
-# https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/types.html#type_signatures
-jni_2_signature_mapping = {'u8': 'B',
-                           'u8[]': '[B',
-                           'i8': 'B',
-                           'i8[]': '[B',
-                           'u16': 'S',
-                           'u16[]': '[S',
-                           'i16': 'S',
-                           'i16[]': '[S',
-                           'u32': 'I',
-                           'u32[]': '[I',
-                           'i32': 'I',
-                           'i32[]': '[I',
-                           'u64': 'J',
-                           'u64[]': '[J',
-                           'i64': 'J',
-                           'i64[]': '[J',
-                           'f64': 'D',
-                           'f64[]': '[D'
-                           }
-
-# https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#Get_type_Field_routines
-jni_field_accessors =  {'u8': 'ByteField',
-                        'u8[]': 'ObjectField',
-                        'i8': 'ByteField',
-                        'i8[]': 'ObjectField',
-                        'u16': 'ShortField',
-                        'u16[]': 'ObjectField',
-                        'i16': 'ShortField',
-                        'i16[]': 'ObjectField',
-                        'u32': 'IntField',
-                        'u32[]': 'ObjectField',
-                        'i32': 'IntField',
-                        'i32[]': 'ObjectField',
-                        'u64': 'LongField',
-                        'u64[]': 'ObjectField',
-                        'i64': 'LongField',
-                        'i64[]': 'ObjectField',
-                        'f64': 'DoubleField',
-                        'f64[]': 'ObjectField'
-                        }
-
-
-# vpe.api calls that do not follow naming conventions and have to be handled exceptionally when finding reply -> request mapping
-# FIXME in vpe.api
-unconventional_naming_rep_req = {
-                                 'cli_reply': 'cli_request',
-                                 'vnet_summary_stats_reply': 'vnet_get_summary_stats',
-                                 # This below is actually a sub-details callback. We cannot derive the mapping of dump request
-                                 # belonging to this sub-details from naming conventions. We need special mapping
-                                 'bridge_domain_sw_if_details': 'bridge_domain',
-                                 # This is standard dump call + details reply. However it's not called details but entry
-                                 'l2_fib_table_entry': 'l2_fib_table'
-                                 }
-
-#
-# FIXME no convention in the naming of events (notifications) in vpe.api
-notifications_message_suffixes = ("event", "counters")
-notification_messages_reused = ["sw_interface_set_flags"]
-
-# messages that must be ignored. These messages are INSUFFICIENTLY marked as disabled in vpe.api
-# FIXME
-ignored_messages = ["is_address_reachable"]
-
-
-def is_notification(name):
-    """ Returns true if the structure is a notification regardless of its no other use """
-    return is_just_notification(name) or name.lower() in notification_messages_reused
-
-
-def is_just_notification(name):
-    """ Returns true if the structure is just a notification and has no other use """
-    return name.lower().endswith(notifications_message_suffixes)
-
-
-def is_ignored(param):
-    return param.lower() in ignored_messages
-
-
-def remove_reply_suffix(camel_case_name_with_suffix):
-    return remove_suffix(camel_case_name_with_suffix, get_reply_suffix(camel_case_name_with_suffix))
-
-
-def remove_suffix(camel_case_name_with_suffix, suffix):
-    suffix_length = len(suffix)
-    return camel_case_name_with_suffix[:-suffix_length] if suffix_length != 0 else camel_case_name_with_suffix
-
-
-def is_control_ping(camel_case_name_with_suffix):
-    return camel_case_name_with_suffix.lower().startswith("controlping");
-
-
-def api_message_to_javadoc(api_message):
-    """ Converts vpe.api message description to javadoc """
-    str = pprint.pformat(api_message, indent=4, width=120, depth=None)
-    return " * " + str.replace("\n", "\n * ")
-
-
-notification_dto_suffix = "Notification"
-
-
-def add_notification_suffix(camel_case_dto_name):
-    camel_case_dto_name += notification_dto_suffix
-    return camel_case_dto_name
-
-
-def is_array(java_type_as_string):
-    return java_type_as_string.endswith("[]")
diff --git a/vpp-api/java/m4/ax_check_java_home.m4 b/vpp-api/java/m4/ax_check_java_home.m4
deleted file mode 100644
index cfe8f589..00000000
--- a/vpp-api/java/m4/ax_check_java_home.m4
+++ /dev/null
@@ -1,80 +0,0 @@
-# ===========================================================================
-#    http://www.gnu.org/software/autoconf-archive/ax_check_java_home.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_CHECK_JAVA_HOME
-#
-# DESCRIPTION
-#
-#   Check for Sun Java (JDK / JRE) installation, where the 'java' VM is in.
-#   If found, set environment variable JAVA_HOME = Java installation home,
-#   else left JAVA_HOME untouch, which in most case means JAVA_HOME is
-#   empty.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Gleen Salmon 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 6
-
-AU_ALIAS([AC_CHECK_JAVA_HOME], [AX_CHECK_JAVA_HOME])
-
-AC_DEFUN([AX_CHECK_JAVA_HOME],
-[AC_MSG_CHECKING([for JAVA_HOME])
-# We used a fake loop so that we can use "break" to exit when the result
-# is found.
-while true
-do
-  # If the user defined JAVA_HOME, don't touch it.
-  test "${JAVA_HOME+set}" = set && break
-
-  # On Mac OS X 10.5 and following, run /usr/libexec/java_home to get
-  # the value of JAVA_HOME to use.
-  # (http://developer.apple.com/library/mac/#qa/qa2001/qa1170.html).
-  JAVA_HOME=`/usr/libexec/java_home 2>/dev/null`
-  test x"$JAVA_HOME" != x && break
-
-  # See if we can find the java executable, and compute from there.
-  TRY_JAVA_HOME=`ls -dr /usr/java/* 2> /dev/null | head -n 1`
-  if test x$TRY_JAVA_HOME != x; then
-    PATH=$PATH:$TRY_JAVA_HOME/bin
-  fi
-  AC_PATH_PROG([JAVA_PATH_NAME], [java])
-  if test "x$JAVA_PATH_NAME" != x; then
-    JAVA_HOME=`echo $JAVA_PATH_NAME | sed "s/\(.*\)[[/]]bin[[/]]java.*/\1/"`
-    break
-  fi
-
-  AC_MSG_NOTICE([Could not compute JAVA_HOME])
-  break
-done
-AC_MSG_RESULT([$JAVA_HOME])
-])
diff --git a/vpp-api/java/m4/ax_check_java_plugin.m4 b/vpp-api/java/m4/ax_check_java_plugin.m4
deleted file mode 100644
index 920753e5..00000000
--- a/vpp-api/java/m4/ax_check_java_plugin.m4
+++ /dev/null
@@ -1,101 +0,0 @@
-# ===========================================================================
-#   http://www.gnu.org/software/autoconf-archive/ax_check_java_plugin.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_CHECK_JAVA_PLUGIN()
-#
-# DESCRIPTION
-#
-#   This macro sets  to empty on failure and to a compatible
-#   version of plugin.jar otherwise. Directories searched are /usr/java/*
-#   and /usr/local/java/*, which are assumed to be j{dk,re} installations.
-#   Apply the shell variable as you see fit. If sun changes things so
-#   /lib/plugin.jar is not the magic file it will stop working.
-#
-#   This macro assumes that unzip, zipinfo or pkzipc is available (and can
-#   list the contents of the jar archive). The first two are assumed to work
-#   similarly enough to the infozip versisonms. The pkzipc version is
-#   assumed to work if I undertstand the documentation on pkware's site but
-#   YMMV. I do not have access to pwkware's version to test it.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Duncan Simpson 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 7
-
-AU_ALIAS([DPS_CHECK_PLUGIN], [AX_CHECK_JAVA_PLUGIN])
-AC_DEFUN([AX_CHECK_JAVA_PLUGIN],
-[AC_REQUIRE([AC_PROG_AWK])
-AC_REQUIRE([AC_PROG_FGREP])
-AC_CHECK_PROG(ZIPINFO,[zipinfo unzip pkzipc])
-AC_MSG_CHECKING([for the java plugin])
-case "x$ZIPINFO" in
-[*/zipinfo)]
-	zipinf="zipinfo -1" ;;
-[*/unzip)]
-	zipinf="unzip -l";;
-[*/pkzipc)]
-	ziping="unzipc -view";;
-[x*)]
-	AC_MSG_RESULT([skiped, none of zipinfo, unzip and pkzipc found])
-	AC_SUBST($1,[])
-	zipinf="";;
-esac
-if test "x$zipinf" != "x"; then
-jplugin=""
-for jhome in `ls -dr /usr/java/* /usr/local/java/* 2> /dev/null`; do
-for jfile in lib/plugin.jar jre/lib/plugin.jar; do
-if test "x$jplugin" = "x" && test -f "$jhome/$jfile"; then
-eval "$zipinf $jhome/$jfile | $AWK '{ print \$NF; }' | $FGREP netscape/javascript/JSObject" >/dev/null 2>/dev/null
-if test $? -eq 0; then
-dnl Some version of gcj (and javac) refuse to work with some files
-dnl that pass this test. To stop this problem make sure that the compiler
-dnl still works with this jar file in the classpath
-cat << \EOF > Test.java
-/* [#]line __oline__ "configure" */
-public class Test {
-}
-EOF
-if eval "$JAVAC -classpath $jhome/$jfile Test.java 2>/dev/null >/dev/null" && test -f Test.class; then
-jplugin="$jhome/$jfile"
-fi
-rm -f Test.java Test.class
-fi; fi; done; done
-if test "x$jplugin" != "x"; then
-AC_SUBST($1,$jplugin)
-AC_MSG_RESULT($jplugin)
-else
-AC_MSG_RESULT([java plugin not found])
-AC_SUBST($1,[])
-fi
-fi
-])
diff --git a/vpp-api/java/m4/ax_java_check_class.m4 b/vpp-api/java/m4/ax_java_check_class.m4
deleted file mode 100644
index 917638ae..00000000
--- a/vpp-api/java/m4/ax_java_check_class.m4
+++ /dev/null
@@ -1,85 +0,0 @@
-# ===========================================================================
-#    http://www.gnu.org/software/autoconf-archive/ax_java_check_class.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_JAVA_CHECK_CLASS(,,)
-#
-# DESCRIPTION
-#
-#   Test if a Java class is available. Based on AX_PROG_JAVAC_WORKS. This
-#   version uses a cache variable which is both compiler, options and
-#   classpath dependent (so if you switch from javac to gcj it correctly
-#   notices and redoes the test).
-#
-#   The macro tries to compile a minimal program importing . Some
-#   newer compilers moan about the failure to use this but fail or produce a
-#   class file anyway. All moaing is sunk to /dev/null since I only wanted
-#   to know if the class could be imported. This is a recommended followup
-#   to AX_CHECK_JAVA_PLUGIN with classpath appropriately adjusted.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Duncan Simpson 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 9
-
-AU_ALIAS([DPS_JAVA_CHECK_CLASS], [AX_JAVA_CHECK_CLASS])
-AC_DEFUN([AX_JAVA_CHECK_CLASS],[
-m4_define([cache_val],[m4_translit(ax_cv_have_java_class_$1, " ." ,"__")])
-if test "x$CLASSPATH" != "x"; then
-xtra=" with classpath ${CLASSPATH}"
-xopts=`echo ${CLASSPATH} | ${SED} 's/^ *://'`
-xopts="-classpath $xopts"
-else xtra=""; xopts=""; fi
-cache_var="cache_val"AS_TR_SH([_Jc_${JAVAC}_Cp_${CLASSPATH}])
-AC_CACHE_CHECK([if the $1 class is available$xtra], [$cache_var], [
-JAVA_TEST=Test.java
-CLASS_TEST=Test.class
-cat << \EOF > $JAVA_TEST
-/* [#]xline __oline__ "configure" */
-import $1;
-public class Test {
-}
-EOF
-if AC_TRY_COMMAND($JAVAC $JAVACFLAGS $xopts $JAVA_TEST) >/dev/null 2>&1; then
-  eval "${cache_var}=yes"
-else
-  eval "${cache_var}=no"
-  echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-  cat $JAVA_TEST >&AS_MESSAGE_LOG_FD
-fi
-rm -f $JAVA_TEST $CLASS_TEST
-])
-if eval 'test "x$'${cache_var}'" = "xyes"'; then
-$2
-true; else
-$3
-false; fi])
diff --git a/vpp-api/java/m4/ax_java_options.m4 b/vpp-api/java/m4/ax_java_options.m4
deleted file mode 100644
index 36c10d92..00000000
--- a/vpp-api/java/m4/ax_java_options.m4
+++ /dev/null
@@ -1,48 +0,0 @@
-# ===========================================================================
-#      http://www.gnu.org/software/autoconf-archive/ax_java_options.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_JAVA_OPTIONS
-#
-# DESCRIPTION
-#
-#   AX_JAVA_OPTIONS adds configure command line options used for Java m4
-#   macros. This Macro is optional.
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission. The
-#   general documentation, as well as the sample configure.in, is included
-#   in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Devin Weaver 
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 6
-
-AU_ALIAS([AC_JAVA_OPTIONS], [AX_JAVA_OPTIONS])
-AC_DEFUN([AX_JAVA_OPTIONS],[
-AC_ARG_WITH(java-prefix,
-                        [  --with-java-prefix=PFX  prefix where Java runtime is installed (optional)])
-AC_ARG_WITH(javac-flags,
-                        [  --with-javac-flags=FLAGS flags to pass to the Java compiler (optional)])
-AC_ARG_WITH(java-flags,
-                        [  --with-java-flags=FLAGS flags to pass to the Java VM (optional)])
-JAVAPREFIX=$with_java_prefix
-JAVACFLAGS=$with_javac_flags
-JAVAFLAGS=$with_java_flags
-AC_SUBST(JAVAPREFIX)dnl
-AC_SUBST(JAVACFLAGS)dnl
-AC_SUBST(JAVAFLAGS)dnl
-AC_SUBST(JAVA)dnl
-AC_SUBST(JAVAC)dnl
-])
diff --git a/vpp-api/java/m4/ax_libgcj_jar.m4 b/vpp-api/java/m4/ax_libgcj_jar.m4
deleted file mode 100644
index 5e942857..00000000
--- a/vpp-api/java/m4/ax_libgcj_jar.m4
+++ /dev/null
@@ -1,83 +0,0 @@
-# ===========================================================================
-#       http://www.gnu.org/software/autoconf-archive/ax_libgcj_jar.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_LIBGCJ_JAR
-#
-# DESCRIPTION
-#
-#   Locate libgcj.jar so you can place it before everything else when using
-#   gcj.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Duncan Simpson 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 9
-
-AU_ALIAS([DPS_LIBGCJ_JAR], [AX_LIBGCJ_JAR])
-AC_DEFUN([AX_LIBGCJ_JAR],
-[
-AC_REQUIRE([AC_EXEEXT])
-AC_REQUIRE([AX_PROG_JAVAC])
-AC_REQUIRE([AC_PROG_FGREP])
-AC_PROG_SED
-if test "x$SED" = "x"; then
-AC_MSG_WARN([sed not avaiable, so libgcj.jar test skipped])
-else
-AC_MSG_CHECKING([if $JAVAC is gcj]);
-jc=`eval "[echo x$JAVAC | $SED 's/^x.*\\/\\([^/]*\\)\$/x\\1/;s/^ *\\([^ ]*\\) .*$/\\1/;s/"$EXEEXT"$//']"`
-if test "x$jc" != "xxgcj"; then
-AC_MSG_RESULT(no)
-else
-AC_MSG_RESULT(yes)
-AC_MSG_CHECKING([libgcj.jar location])
-save_cp="$CLASSPATH";
-unset CLASSPATH;
-AC_MSG_CHECKING([gcj default classpath])
-cat << \EOF > Test.java
-/* [#]line __oline__ "configure" */
-public class Test {
-}
-EOF
-lgcj=`eval "[$JAVAC -v -C Test.java 2>&1 | $FGREP \\(system\\) | $SED 's/^ *\\([^ ]*\\) .*$/\\1/;s/\\.jar\\//.jar/']"`;
-if test -f Test.class && test "x$lgcj" != "x"; then
-AC_MSG_RESULT($lgcj)
-$1="$lgcj:"
-else
-AC_MSG_RESULT(failed)
-$1=""
-fi
-if test "x$save_cp" != "x"; then CLASSPATH="$save_cp"; fi
-rm -f Test.java Test.class
-fi
-fi
-])
diff --git a/vpp-api/java/m4/ax_prog_jar.m4 b/vpp-api/java/m4/ax_prog_jar.m4
deleted file mode 100644
index 3c60fcaf..00000000
--- a/vpp-api/java/m4/ax_prog_jar.m4
+++ /dev/null
@@ -1,49 +0,0 @@
-# ===========================================================================
-#        http://www.gnu.org/software/autoconf-archive/ax_prog_jar.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAR
-#
-# DESCRIPTION
-#
-#   AX_PROG_JAR tests for an existing jar program. It uses the environment
-#   variable JAR then tests in sequence various common jar programs.
-#
-#   If you want to force a specific compiler:
-#
-#   - at the configure.in level, set JAR=yourcompiler before calling
-#   AX_PROG_JAR
-#
-#   - at the configure level, setenv JAR
-#
-#   You can use the JAR variable in your Makefile.in, with @JAR@.
-#
-#   Note: This macro depends on the autoconf M4 macros for Java programs. It
-#   is VERY IMPORTANT that you download that whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission.
-#
-#   The general documentation of those macros, as well as the sample
-#   configure.in, is included in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Egon Willighagen 
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 7
-
-AU_ALIAS([AC_PROG_JAR], [AX_PROG_JAR])
-AC_DEFUN([AX_PROG_JAR],[
-AS_IF([test "x$JAVAPREFIX" = x],
-      [test "x$JAR" = x && AC_CHECK_PROGS([JAR], [jar])],
-      [test "x$JAR" = x && AC_CHECK_PROGS([JAR], [jar], [], [$JAVAPREFIX/bin])])
-test "x$JAR" = x && AC_MSG_ERROR([no acceptable jar program found in \$PATH])
-AC_PROVIDE([$0])dnl
-])
diff --git a/vpp-api/java/m4/ax_prog_java.m4 b/vpp-api/java/m4/ax_prog_java.m4
deleted file mode 100644
index 03961db5..00000000
--- a/vpp-api/java/m4/ax_prog_java.m4
+++ /dev/null
@@ -1,115 +0,0 @@
-# ===========================================================================
-#       http://www.gnu.org/software/autoconf-archive/ax_prog_java.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVA
-#
-# DESCRIPTION
-#
-#   Here is a summary of the main macros:
-#
-#   AX_PROG_JAVAC: finds a Java compiler.
-#
-#   AX_PROG_JAVA: finds a Java virtual machine.
-#
-#   AX_CHECK_CLASS: finds if we have the given class (beware of CLASSPATH!).
-#
-#   AX_CHECK_RQRD_CLASS: finds if we have the given class and stops
-#   otherwise.
-#
-#   AX_TRY_COMPILE_JAVA: attempt to compile user given source.
-#
-#   AX_TRY_RUN_JAVA: attempt to compile and run user given source.
-#
-#   AX_JAVA_OPTIONS: adds Java configure options.
-#
-#   AX_PROG_JAVA tests an existing Java virtual machine. It uses the
-#   environment variable JAVA then tests in sequence various common Java
-#   virtual machines. For political reasons, it starts with the free ones.
-#   You *must* call [AX_PROG_JAVAC] before.
-#
-#   If you want to force a specific VM:
-#
-#   - at the configure.in level, set JAVA=yourvm before calling AX_PROG_JAVA
-#
-#     (but after AC_INIT)
-#
-#   - at the configure level, setenv JAVA
-#
-#   You can use the JAVA variable in your Makefile.in, with @JAVA@.
-#
-#   *Warning*: its success or failure can depend on a proper setting of the
-#   CLASSPATH env. variable.
-#
-#   TODO: allow to exclude virtual machines (rationale: most Java programs
-#   cannot run with some VM like kaffe).
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission.
-#
-#   A Web page, with a link to the latest CVS snapshot is at
-#   .
-#
-#   This is a sample configure.in Process this file with autoconf to produce
-#   a configure script.
-#
-#     AC_INIT(UnTag.java)
-#
-#     dnl Checks for programs.
-#     AC_CHECK_CLASSPATH
-#     AX_PROG_JAVAC
-#     AX_PROG_JAVA
-#
-#     dnl Checks for classes
-#     AX_CHECK_RQRD_CLASS(org.xml.sax.Parser)
-#     AX_CHECK_RQRD_CLASS(com.jclark.xml.sax.Driver)
-#
-#     AC_OUTPUT(Makefile)
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Stephane Bortzmeyer 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 9
-
-AU_ALIAS([AC_PROG_JAVA], [AX_PROG_JAVA])
-AC_DEFUN([AX_PROG_JAVA],[
-m4_define([m4_ax_prog_java_list], [kaffe java])dnl
-AS_IF([test "x$JAVAPREFIX" = x],
-      [test x$JAVA = x && AC_CHECK_PROGS([JAVA], [m4_ax_prog_java_list])],
-      [test x$JAVA = x && AC_CHECK_PROGS([JAVA], [m4_ax_prog_java_list], [], [$JAVAPREFIX/bin])])
-test x$JAVA = x && AC_MSG_ERROR([no acceptable Java virtual machine found in \$PATH])
-m4_undefine([m4_ax_prog_java_list])dnl
-AX_PROG_JAVA_WORKS
-AC_PROVIDE([$0])dnl
-])
diff --git a/vpp-api/java/m4/ax_prog_java_cc.m4 b/vpp-api/java/m4/ax_prog_java_cc.m4
deleted file mode 100644
index 3df064ff..00000000
--- a/vpp-api/java/m4/ax_prog_java_cc.m4
+++ /dev/null
@@ -1,104 +0,0 @@
-# ===========================================================================
-#      http://www.gnu.org/software/autoconf-archive/ax_prog_java_cc.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVA_CC
-#
-# DESCRIPTION
-#
-#   Finds the appropriate java compiler on your path. By preference the java
-#   compiler is gcj, then jikes then javac.
-#
-#   The macro can take one argument specifying a space separated list of
-#   java compiler names.
-#
-#   For example:
-#
-#     AX_PROG_JAVA_CC(javac, gcj)
-#
-#   The macro also sets the compiler options variable: JAVA_CC_OPTS to
-#   something sensible:
-#
-#    - for GCJ it sets it to: @GCJ_OPTS@
-#      (if GCJ_OPTS is not yet defined then it is set to "-C")
-#
-#    - no other compiler has applicable options yet
-#
-#   Here's an example configure.in:
-#
-#     AC_INIT(Makefile.in)
-#     AX_PROG_JAVA_CC()
-#     AC_OUTPUT(Makefile)
-#     dnl End.
-#
-#   And here's the start of the Makefile.in:
-#
-#     PROJECT_ROOT      := @srcdir@
-#     # Tool definitions.
-#     JAVAC             := @JAVA_CC@
-#     JAVAC_OPTS        := @JAVA_CC_OPTS@
-#     JAR_TOOL          := @jar_tool@
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Nic Ferrier 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 4
-
-# AX_PROG_JAVA_CC([COMPILER ...])
-# --------------------------
-# COMPILER ... is a space separated list of java compilers to search for.
-# This just gives the user an opportunity to specify an alternative
-# search list for the java compiler.
-AU_ALIAS([AC_PROG_JAVA_CC], [AX_PROG_JAVA_CC])
-AC_DEFUN([AX_PROG_JAVA_CC],
-[AC_ARG_VAR([JAVA_CC],                [java compiler command])dnl
-AC_ARG_VAR([JAVA_CC_FLAGS],           [java compiler flags])dnl
-m4_ifval([$1],
-      [AC_CHECK_TOOLS(JAVA_CC, [$1])],
-[AC_CHECK_TOOL(JAVA_CC, gcj)
-if test -z "$JAVA_CC"; then
-  AC_CHECK_TOOL(JAVA_CC, javac)
-fi
-if test -z "$JAVA_CC"; then
-  AC_CHECK_TOOL(JAVA_CC, jikes)
-fi
-])
-
-if test "$JAVA_CC" = "gcj"; then
-   if test "$GCJ_OPTS" = ""; then
-      AC_SUBST(GCJ_OPTS,-C)
-   fi
-   AC_SUBST(JAVA_CC_OPTS, @GCJ_OPTS@,
-        [Define the compilation options for GCJ])
-fi
-test -z "$JAVA_CC" && AC_MSG_ERROR([no acceptable java compiler found in \$PATH])
-])# AX_PROG_JAVA_CC
diff --git a/vpp-api/java/m4/ax_prog_java_works.m4 b/vpp-api/java/m4/ax_prog_java_works.m4
deleted file mode 100644
index 54e132af..00000000
--- a/vpp-api/java/m4/ax_prog_java_works.m4
+++ /dev/null
@@ -1,134 +0,0 @@
-# ===========================================================================
-#    http://www.gnu.org/software/autoconf-archive/ax_prog_java_works.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVA_WORKS
-#
-# DESCRIPTION
-#
-#   Internal use ONLY.
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission. The
-#   general documentation, as well as the sample configure.in, is included
-#   in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Stephane Bortzmeyer 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 9
-
-AU_ALIAS([AC_PROG_JAVA_WORKS], [AX_PROG_JAVA_WORKS])
-AC_DEFUN([AX_PROG_JAVA_WORKS], [
-AC_PATH_PROG(UUDECODE, uudecode, [no])
-if test x$UUDECODE != xno; then
-AC_CACHE_CHECK([if uudecode can decode base 64 file], ac_cv_prog_uudecode_base64, [
-dnl /**
-dnl  * Test.java: used to test if java compiler works.
-dnl  */
-dnl public class Test
-dnl {
-dnl
-dnl public static void
-dnl main( String[] argv )
-dnl {
-dnl     System.exit (0);
-dnl }
-dnl
-dnl }
-cat << \EOF > Test.uue
-begin-base64 644 Test.class
-yv66vgADAC0AFQcAAgEABFRlc3QHAAQBABBqYXZhL2xhbmcvT2JqZWN0AQAE
-bWFpbgEAFihbTGphdmEvbGFuZy9TdHJpbmc7KVYBAARDb2RlAQAPTGluZU51
-bWJlclRhYmxlDAAKAAsBAARleGl0AQAEKEkpVgoADQAJBwAOAQAQamF2YS9s
-YW5nL1N5c3RlbQEABjxpbml0PgEAAygpVgwADwAQCgADABEBAApTb3VyY2VG
-aWxlAQAJVGVzdC5qYXZhACEAAQADAAAAAAACAAkABQAGAAEABwAAACEAAQAB
-AAAABQO4AAyxAAAAAQAIAAAACgACAAAACgAEAAsAAQAPABAAAQAHAAAAIQAB
-AAEAAAAFKrcAErEAAAABAAgAAAAKAAIAAAAEAAQABAABABMAAAACABQ=
-====
-EOF
-if $UUDECODE Test.uue; then
-        ac_cv_prog_uudecode_base64=yes
-else
-        echo "configure: __oline__: uudecode had trouble decoding base 64 file 'Test.uue'" >&AS_MESSAGE_LOG_FD
-        echo "configure: failed file was:" >&AS_MESSAGE_LOG_FD
-        cat Test.uue >&AS_MESSAGE_LOG_FD
-        ac_cv_prog_uudecode_base64=no
-fi
-rm -f Test.uue])
-fi
-if test x$ac_cv_prog_uudecode_base64 != xyes; then
-        rm -f Test.class
-        AC_MSG_WARN([I have to compile Test.class from scratch])
-        if test x$ac_cv_prog_javac_works = xno; then
-                AC_MSG_ERROR([Cannot compile java source. $JAVAC does not work properly])
-        fi
-        if test x$ac_cv_prog_javac_works = x; then
-                AX_PROG_JAVAC
-        fi
-fi
-AC_CACHE_CHECK(if $JAVA works, ac_cv_prog_java_works, [
-JAVA_TEST=Test.java
-CLASS_TEST=Test.class
-TEST=Test
-changequote(, )dnl
-cat << \EOF > $JAVA_TEST
-/* [#]line __oline__ "configure" */
-public class Test {
-public static void main (String args[]) {
-        System.exit (0);
-} }
-EOF
-changequote([, ])dnl
-if test x$ac_cv_prog_uudecode_base64 != xyes; then
-        if AC_TRY_COMMAND($JAVAC $JAVACFLAGS $JAVA_TEST) && test -s $CLASS_TEST; then
-                :
-        else
-          echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-          cat $JAVA_TEST >&AS_MESSAGE_LOG_FD
-          AC_MSG_ERROR(The Java compiler $JAVAC failed (see config.log, check the CLASSPATH?))
-        fi
-fi
-if AC_TRY_COMMAND($JAVA -classpath . $JAVAFLAGS $TEST) >/dev/null 2>&1; then
-  ac_cv_prog_java_works=yes
-else
-  echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-  cat $JAVA_TEST >&AS_MESSAGE_LOG_FD
-  AC_MSG_ERROR(The Java VM $JAVA failed (see config.log, check the CLASSPATH?))
-fi
-rm -fr $JAVA_TEST $CLASS_TEST Test.uue
-])
-AC_PROVIDE([$0])dnl
-]
-)
diff --git a/vpp-api/java/m4/ax_prog_javac.m4 b/vpp-api/java/m4/ax_prog_javac.m4
deleted file mode 100644
index d061243c..00000000
--- a/vpp-api/java/m4/ax_prog_javac.m4
+++ /dev/null
@@ -1,79 +0,0 @@
-# ===========================================================================
-#       http://www.gnu.org/software/autoconf-archive/ax_prog_javac.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVAC
-#
-# DESCRIPTION
-#
-#   AX_PROG_JAVAC tests an existing Java compiler. It uses the environment
-#   variable JAVAC then tests in sequence various common Java compilers. For
-#   political reasons, it starts with the free ones.
-#
-#   If you want to force a specific compiler:
-#
-#   - at the configure.in level, set JAVAC=yourcompiler before calling
-#   AX_PROG_JAVAC
-#
-#   - at the configure level, setenv JAVAC
-#
-#   You can use the JAVAC variable in your Makefile.in, with @JAVAC@.
-#
-#   *Warning*: its success or failure can depend on a proper setting of the
-#   CLASSPATH env. variable.
-#
-#   TODO: allow to exclude compilers (rationale: most Java programs cannot
-#   compile with some compilers like guavac).
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission. The
-#   general documentation, as well as the sample configure.in, is included
-#   in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Stephane Bortzmeyer 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 7
-
-AU_ALIAS([AC_PROG_JAVAC], [AX_PROG_JAVAC])
-AC_DEFUN([AX_PROG_JAVAC],[
-m4_define([m4_ax_prog_javac_list],["gcj -C" guavac jikes javac])dnl
-AS_IF([test "x$JAVAPREFIX" = x],
-      [test "x$JAVAC" = x && AC_CHECK_PROGS([JAVAC], [m4_ax_prog_javac_list])],
-      [test "x$JAVAC" = x && AC_CHECK_PROGS([JAVAC], [m4_ax_prog_javac_list], [], [$JAVAPREFIX/bin])])
-m4_undefine([m4_ax_prog_javac_list])dnl
-test "x$JAVAC" = x && AC_MSG_ERROR([no acceptable Java compiler found in \$PATH])
-AX_PROG_JAVAC_WORKS
-AC_PROVIDE([$0])dnl
-])
diff --git a/vpp-api/java/m4/ax_prog_javac_works.m4 b/vpp-api/java/m4/ax_prog_javac_works.m4
deleted file mode 100644
index 7dfa1e37..00000000
--- a/vpp-api/java/m4/ax_prog_javac_works.m4
+++ /dev/null
@@ -1,72 +0,0 @@
-# ===========================================================================
-#    http://www.gnu.org/software/autoconf-archive/ax_prog_javac_works.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVAC_WORKS
-#
-# DESCRIPTION
-#
-#   Internal use ONLY.
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission. The
-#   general documentation, as well as the sample configure.in, is included
-#   in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Stephane Bortzmeyer 
-#
-#   This program is free software; you can redistribute it and/or modify it
-#   under the terms of the GNU General Public License as published by the
-#   Free Software Foundation; either version 2 of the License, or (at your
-#   option) any later version.
-#
-#   This program is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-#   Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see .
-#
-#   As a special exception, the respective Autoconf Macro's copyright owner
-#   gives unlimited permission to copy, distribute and modify the configure
-#   scripts that are the output of Autoconf when processing the Macro. You
-#   need not follow the terms of the GNU General Public License when using
-#   or distributing such scripts, even though portions of the text of the
-#   Macro appear in them. The GNU General Public License (GPL) does govern
-#   all other use of the material that constitutes the Autoconf Macro.
-#
-#   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Archive. When you make and distribute a
-#   modified version of the Autoconf Macro, you may extend this special
-#   exception to the GPL to apply to your modified version as well.
-
-#serial 6
-
-AU_ALIAS([AC_PROG_JAVAC_WORKS], [AX_PROG_JAVAC_WORKS])
-AC_DEFUN([AX_PROG_JAVAC_WORKS],[
-AC_CACHE_CHECK([if $JAVAC works], ac_cv_prog_javac_works, [
-JAVA_TEST=Test.java
-CLASS_TEST=Test.class
-cat << \EOF > $JAVA_TEST
-/* [#]line __oline__ "configure" */
-public class Test {
-}
-EOF
-if AC_TRY_COMMAND($JAVAC $JAVACFLAGS $JAVA_TEST) >/dev/null 2>&1; then
-  ac_cv_prog_javac_works=yes
-else
-  AC_MSG_ERROR([The Java compiler $JAVAC failed (see config.log, check the CLASSPATH?)])
-  echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-  cat $JAVA_TEST >&AS_MESSAGE_LOG_FD
-fi
-rm -f $JAVA_TEST $CLASS_TEST
-])
-AC_PROVIDE([$0])dnl
-])
diff --git a/vpp-api/java/m4/ax_prog_javadoc.m4 b/vpp-api/java/m4/ax_prog_javadoc.m4
deleted file mode 100644
index bcb6045a..00000000
--- a/vpp-api/java/m4/ax_prog_javadoc.m4
+++ /dev/null
@@ -1,50 +0,0 @@
-# ===========================================================================
-#      http://www.gnu.org/software/autoconf-archive/ax_prog_javadoc.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVADOC
-#
-# DESCRIPTION
-#
-#   AX_PROG_JAVADOC tests for an existing javadoc generator. It uses the
-#   environment variable JAVADOC then tests in sequence various common
-#   javadoc generator.
-#
-#   If you want to force a specific compiler:
-#
-#   - at the configure.in level, set JAVADOC=yourgenerator before calling
-#   AX_PROG_JAVADOC
-#
-#   - at the configure level, setenv JAVADOC
-#
-#   You can use the JAVADOC variable in your Makefile.in, with @JAVADOC@.
-#
-#   Note: This macro depends on the autoconf M4 macros for Java programs. It
-#   is VERY IMPORTANT that you download that whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission.
-#
-#   The general documentation of those macros, as well as the sample
-#   configure.in, is included in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Egon Willighagen 
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 8
-
-AU_ALIAS([AC_PROG_JAVADOC], [AX_PROG_JAVADOC])
-AC_DEFUN([AX_PROG_JAVADOC],[
-AS_IF([test "x$JAVAPREFIX" = x],
-      [test "x$JAVADOC" = x && AC_CHECK_PROGS([JAVADOC], [javadoc])],
-      [test "x$JAVADOC" = x && AC_CHECK_PROGS([JAVADOC], [javadoc], [], [$JAVAPREFIX/bin])])
-test "x$JAVADOC" = x && AC_MSG_ERROR([no acceptable javadoc generator found in \$PATH])
-AC_PROVIDE([$0])dnl
-])
diff --git a/vpp-api/java/m4/ax_prog_javah.m4 b/vpp-api/java/m4/ax_prog_javah.m4
deleted file mode 100644
index cefc616d..00000000
--- a/vpp-api/java/m4/ax_prog_javah.m4
+++ /dev/null
@@ -1,64 +0,0 @@
-# ===========================================================================
-#       http://www.gnu.org/software/autoconf-archive/ax_prog_javah.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_PROG_JAVAH
-#
-# DESCRIPTION
-#
-#   AX_PROG_JAVAH tests the availability of the javah header generator and
-#   looks for the jni.h header file. If available, JAVAH is set to the full
-#   path of javah and CPPFLAGS is updated accordingly.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Luc Maisonobe 
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 8
-
-AU_ALIAS([AC_PROG_JAVAH], [AX_PROG_JAVAH])
-AC_DEFUN([AX_PROG_JAVAH],[
-AC_REQUIRE([AC_CANONICAL_BUILD])dnl
-AC_REQUIRE([AC_PROG_CPP])dnl
-AC_PATH_PROG(JAVAH,javah)
-AS_IF([test -n "$ac_cv_path_JAVAH"],
-      [
-        AC_TRY_CPP([#include ],,[
-        ac_save_CPPFLAGS="$CPPFLAGS"
-		_ACJAVAH_FOLLOW_SYMLINKS("$ac_cv_path_JAVAH")
-        ax_prog_javah_bin_dir=`AS_DIRNAME([$_ACJAVAH_FOLLOWED])`
-        ac_dir="`AS_DIRNAME([$ax_prog_javah_bin_dir])`/include"
-        AS_CASE([$build_os],
-                [cygwin*],
-                [ac_machdep=win32],
-                [ac_machdep=`AS_ECHO($build_os) | sed 's,[[-0-9]].*,,'`])
-        CPPFLAGS="$ac_save_CPPFLAGS -I$ac_dir -I$ac_dir/$ac_machdep"
-        AC_TRY_CPP([#include ],
-                   ac_save_CPPFLAGS="$CPPFLAGS",
-                   AC_MSG_WARN([unable to include ]))
-        CPPFLAGS="$ac_save_CPPFLAGS"])
-      ])
-])
-
-AC_DEFUN([_ACJAVAH_FOLLOW_SYMLINKS],[
-# find the include directory relative to the javac executable
-_cur="$1"
-while ls -ld "$_cur" 2>/dev/null | grep " -> " >/dev/null; do
-        AC_MSG_CHECKING([symlink for $_cur])
-        _slink=`ls -ld "$_cur" | sed 's/.* -> //'`
-        case "$_slink" in
-        /*) _cur="$_slink";;
-        # 'X' avoids triggering unwanted echo options.
-        *) _cur=`echo "X$_cur" | sed -e 's/^X//' -e 's:[[^/]]*$::'`"$_slink";;
-        esac
-        AC_MSG_RESULT([$_cur])
-done
-_ACJAVAH_FOLLOWED="$_cur"
-])
diff --git a/vpp-api/java/m4/ax_try_compile_java.m4 b/vpp-api/java/m4/ax_try_compile_java.m4
deleted file mode 100644
index a8ed6b2a..00000000
--- a/vpp-api/java/m4/ax_try_compile_java.m4
+++ /dev/null
@@ -1,55 +0,0 @@
-# ===========================================================================
-#    http://www.gnu.org/software/autoconf-archive/ax_try_compile_java.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_TRY_COMPILE_JAVA
-#
-# DESCRIPTION
-#
-#   AX_TRY_COMPILE_JAVA attempt to compile user given source.
-#
-#   *Warning*: its success or failure can depend on a proper setting of the
-#   CLASSPATH env. variable.
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission. The
-#   general documentation, as well as the sample configure.in, is included
-#   in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Devin Weaver 
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 8
-
-AU_ALIAS([AC_TRY_COMPILE_JAVA], [AX_TRY_COMPILE_JAVA])
-AC_DEFUN([AX_TRY_COMPILE_JAVA],[
-AC_REQUIRE([AX_PROG_JAVAC])dnl
-cat << \EOF > Test.java
-/* [#]line __oline__ "configure" */
-ifelse([$1], , , [import $1;])
-public class Test {
-[$2]
-}
-EOF
-if AC_TRY_COMMAND($JAVAC $JAVACFLAGS Test.java) && test -s Test.class
-then
-dnl Don't remove the temporary files here, so they can be examined.
-  ifelse([$3], , :, [$3])
-else
-  echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-  cat Test.java >&AS_MESSAGE_LOG_FD
-ifelse([$4], , , [  rm -fr Test.java Test.class
-  $4
-])dnl
-fi
-rm -fr Test.java Test.class])
diff --git a/vpp-api/java/m4/ax_try_run_java.m4 b/vpp-api/java/m4/ax_try_run_java.m4
deleted file mode 100644
index c680f03f..00000000
--- a/vpp-api/java/m4/ax_try_run_java.m4
+++ /dev/null
@@ -1,56 +0,0 @@
-# ===========================================================================
-#      http://www.gnu.org/software/autoconf-archive/ax_try_run_java.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_TRY_RUN_JAVA
-#
-# DESCRIPTION
-#
-#   AX_TRY_RUN_JAVA attempt to compile and run user given source.
-#
-#   *Warning*: its success or failure can depend on a proper setting of the
-#   CLASSPATH env. variable.
-#
-#   Note: This is part of the set of autoconf M4 macros for Java programs.
-#   It is VERY IMPORTANT that you download the whole set, some macros depend
-#   on other. Unfortunately, the autoconf archive does not support the
-#   concept of set of macros, so I had to break it for submission. The
-#   general documentation, as well as the sample configure.in, is included
-#   in the AX_PROG_JAVA macro.
-#
-# LICENSE
-#
-#   Copyright (c) 2008 Devin Weaver 
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 2
-
-AU_ALIAS([AC_TRY_RUN_JAVA], [AX_TRY_RUN_JAVA])
-AC_DEFUN([AX_TRY_RUN_JAVA],[
-AC_REQUIRE([AX_PROG_JAVAC])dnl
-AC_REQUIRE([AX_PROG_JAVA])dnl
-cat << \EOF > Test.java
-/* [#]line __oline__ "configure" */
-ifelse([$1], , , [include $1;])
-public class Test {
-[$2]
-}
-EOF
-if AC_TRY_COMMAND($JAVAC $JAVACFLAGS Test.java) && test -s Test.class && ($JAVA $JAVAFLAGS Test; exit) 2>/dev/null
-then
-dnl Don't remove the temporary files here, so they can be examined.
-  ifelse([$3], , :, [$3])
-else
-  echo "configure: failed program was:" >&AS_MESSAGE_LOG_FD
-  cat Test.java >&AS_MESSAGE_LOG_FD
-ifelse([$4], , , [  rm -fr Test.java Test.class
-  $4
-])dnl
-fi
-rm -fr Test.java Test.class])
diff --git a/vpp-api/lua/README.md b/vpp-api/lua/README.md
deleted file mode 100644
index 4ecdb34d..00000000
--- a/vpp-api/lua/README.md
+++ /dev/null
@@ -1,50 +0,0 @@
-This is the experimental version of Lua API, aimed for the luajit use.
-
-Please take a look and send the feedback to ayourtch@gmail.com.
-
-To run the examples here:
-
-1) install luajit - "sudo apt-get install luajit" on ubuntu
-
-2) "make build-vpp-api" in the top VPP directory
-
-3) "make run" in a separate terminal window
-   This ensures you have an instance of VPP running
-
-4) sudo luajit examples/example-cli.lua
-
-This will result in something like this:
-
-Version:
-00000000  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  ................
-00000010  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  ................
-
-{ [1] = { ["luaapi_message_name"] = show_version_reply,["program"] = vpe,["version"] = ,["build_date"] = Fri Nov 25 10:58:48 UTC 2016,["retval"] = 0,["build_directory"] = /home/ubuntu/vpp,["_vl_msg_id"] = 170,["context"] = 0,} ,}
----
-{ [1] = { ["luaapi_message_name"] = cli_inband_reply,["_vl_msg_id"] = 94,["length"] = 66,["reply"] = vpp v built by ubuntu on vpp-toys at Fri Nov 25 10:58:48 UTC 2016
-,["retval"] = 0,["context"] = 0,} ,}
----
-
-5) You can also run the performance test bench:
-
-$ sudo luajit bench.lua
-10001 iterations, average speed 5624LL per second
-10001 iterations, average speed 6650LL per second
-10001 iterations, average speed 6053LL per second
-10001 iterations, average speed 7056LL per second
-10001 iterations, average speed 6388LL per second
-10001 iterations, average speed 5849LL per second
-10001 iterations, average speed 6321LL per second
-10001 iterations, average speed 6368LL per second
-10001 iterations, average speed 5958LL per second
-10001 iterations, average speed 6482LL per second
-Average tps across the tests: 6274LL
-
-Note: the above is run in an lxd container running inside 2-core
-xhyve VM on a Macbook Pro, so I would not take the performance numbers for granted :)
-
-The "examples" directory contains a few naive examples, as well as a couple of more 
-advanced ones - a tab-completing CLI for VPP that can call both the APIs and CLI,
-and also a small test utility which I use for automating some small tests using
-VPP.
-
diff --git a/vpp-api/lua/bench.lua b/vpp-api/lua/bench.lua
deleted file mode 100644
index 8e5a0b4b..00000000
--- a/vpp-api/lua/bench.lua
+++ /dev/null
@@ -1,70 +0,0 @@
---[[
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
-local vpp = require "vpp-lapi"
-
-local ffi = require "ffi"
-
-ffi.cdef([[
-   struct timespec {
-               long tv_sec;        /* seconds */
-               long tv_nsec;       /* nanoseconds */
-           };
-
-   int clock_gettime(int clk_id, struct timespec *tp);
-]])
-
-
-local time_cache = ffi.new("struct timespec[1]")
-local time_cache_1 = time_cache[0]
-function get_ns()
-  ffi.C.clock_gettime(0, time_cache)
-  return time_cache_1.tv_nsec + 1000000000 * time_cache_1.tv_sec
-end
-
-function do_bench()
-  local cycle_start = get_ns()
-  local n_iterations =  10000
-  local count = 1
-  for i = 1,n_iterations do
-    -- print(i)
-    vpp:api_call("show_version")
-    count = count + 1
-    -- print(i, "done")
-  end
-  cycle_end = get_ns()
-  local tps = n_iterations*1000000000LL/(cycle_end - cycle_start)
-  print (tostring(count) .. " iterations, average speed " .. tostring(tps) .. " per second")
-  return tps
-end
-
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
-vpp:init({ pneum_path = pneum_path })
-vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
-
-vpp:connect("lua-bench")
-local n_tests = 10
-local tps_acc = 0LL
-for i=1,n_tests do
-  tps_acc = tps_acc + do_bench()
-end
-print("Average tps across the tests: " .. tostring(tps_acc/n_tests))
-
-vpp:disconnect()
-
-
diff --git a/vpp-api/lua/examples/cli/README.md b/vpp-api/lua/examples/cli/README.md
deleted file mode 100644
index 3a5f8ee9..00000000
--- a/vpp-api/lua/examples/cli/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-This is a small experiment to have a wrapper CLI which can call both API functions as well as debug CLI.
-
-To facilitate tab completion and help, the API call names are broken up with spaces replacing the underscores.
-
-
diff --git a/vpp-api/lua/examples/cli/lua-cli.lua b/vpp-api/lua/examples/cli/lua-cli.lua
deleted file mode 100644
index b3a24d7d..00000000
--- a/vpp-api/lua/examples/cli/lua-cli.lua
+++ /dev/null
@@ -1,747 +0,0 @@
---[[
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
--- Experimental prototype CLI using API to VPP, with tab completion
---
--- Written by Andrew Yourtchenko (ayourtch@cisco.com) 2010,2016
---
-
-vpp = require "vpp-lapi"
-
-
-local dotdotdot = "..."
-
--- First the "readline" routine
-
-readln = {
-split = function(str, pat)
-  local t = {}  -- NOTE: use {n = 0} in Lua-5.0
-  local fpat = "(.-)" .. pat
-  local last_end = 1
-  if str then
-    local s, e, cap = str:find(fpat, 1)
-    while s do
-      if s ~= 1 or cap ~= "" then
-        table.insert(t,cap)
-      end
-      last_end = e+1
-      s, e, cap = str:find(fpat, last_end)
-    end
-    if last_end <= #str then
-      cap = str:sub(last_end)
-      table.insert(t, cap)
-    end
-  end
-  return t
-end,
-
-reader = function()
-  local rl = {}
-
-  rl.init = function()
-    os.execute("stty -icanon min 1 -echo")
-    rl.rawmode = true
-  end
-
-  rl.done = function()
-    os.execute("stty icanon echo")
-    rl.rawmode = false
-  end
-
-  rl.prompt = ">"
-  rl.history = { "" }
-  rl.history_index = 1
-  rl.history_length = 1
-
-  rl.hide_cmd = function()
-    local bs = string.char(8) .. " " .. string.char(8)
-    for i = 1, #rl.command do
-      io.stdout:write(bs)
-    end
-  end
-
-  rl.show_cmd = function()
-    if rl.command then
-      io.stdout:write(rl.command)
-    end
-  end
-
-  rl.store_history = function(cmd)
-    if cmd == "" then
-      return
-    end
-    rl.history[rl.history_length] = cmd
-    rl.history_length = rl.history_length + 1
-    rl.history_index = rl.history_length
-    rl.history[rl.history_length] = ""
-  end
-
-  rl.readln = function()
-    local done = false
-    local need_prompt = true
-    rl.command = ""
-
-    if not rl.rawmode then
-      rl.init()
-    end
-
-    while not done do
-      if need_prompt then
-        io.stdout:write(rl.prompt)
-	io.stdout:write(rl.command)
-        need_prompt = false
-      end
-
-      local ch = io.stdin:read(1)
-      if ch:byte(1) == 27 then
-        -- CONTROL
-        local ch2 = io.stdin:read(1)
-        -- arrows
-        if ch2:byte(1) == 91 then
-          local ch3 = io.stdin:read(1)
-          local b = ch3:byte(1)
-          if b == 65 then
-            ch = "UP"
-          elseif b == 66 then
-            ch = "DOWN"
-          elseif b == 67 then
-            ch = "RIGHT"
-          elseif b == 68 then
-            ch = "LEFT"
-          end
-          -- print("Byte: " .. ch3:byte(1))
-          -- if ch3:byte(1)
-        end
-      end
-
-      if ch == "?" then
-        io.stdout:write(ch)
-        io.stdout:write("\n")
-        if rl.help then
-          rl.help(rl)
-        end
-        need_prompt = true
-      elseif ch == "\t" then
-        if rl.tab_complete then
-          rl.tab_complete(rl)
-        end
-        io.stdout:write("\n")
-        need_prompt = true
-      elseif ch == "\n" then
-        io.stdout:write(ch)
-        done = true
-      elseif ch == "\004" then
-        io.stdout:write("\n")
-        rl.command = nil
-	done = true
-      elseif ch == string.char(127) then
-        if rl.command ~= "" then
-          io.stdout:write(string.char(8) .. " " .. string.char(8))
-          rl.command = string.sub(rl.command, 1, -2)
-        end
-      elseif #ch > 1 then
-        -- control char
-        if ch == "UP" then
-          rl.hide_cmd()
-          if rl.history_index == #rl.history then
-            rl.history[rl.history_index] = rl.command
-          end
-          if rl.history_index > 1 then
-            rl.history_index = rl.history_index - 1
-            rl.command = rl.history[rl.history_index]
-          end
-          rl.show_cmd()
-        elseif ch == "DOWN" then
-          rl.hide_cmd()
-          if rl.history_index < rl.history_length then
-            rl.history_index = rl.history_index + 1
-            rl.command = rl.history[rl.history_index]
-          end
-          rl.show_cmd()
-        end
-      else
-        io.stdout:write(ch)
-        rl.command = rl.command .. ch
-      end
-    end
-    if rl.command then
-      rl.store_history(rl.command)
-    end
-    return rl.command
-  end
-  return rl
-end
-
-}
-
---[[
-
-r = reader()
-
-local done = false
-
-while not done do
-  local cmd = r.readln()
-  print("Command: " .. tostring(cmd))
-  if not cmd or cmd == "quit" then
-    done = true
-  end
-end
-
-r.done()
-
-]]
-
---------- MDS show tech parser
-
-local print_section = nil
-local list_sections = false
-
-local curr_section = "---"
-local curr_parser = nil
-
--- by default operate in batch mode
-local batch_mode = true
-
-local db = {}
-local device = {}
-device.output = {}
-local seen_section = {}
-
-function start_collection(name)
-  device = {}
-  seen_section = {}
-end
-
-function print_error(errmsg)
-  print("@#$:" .. errmsg)
-end
-
-function keys(tbl)
-  local t = {}
-  for k, v in pairs(tbl) do
-    table.insert(t, k)
-  end
-  return t
-end
-
-function tset (parent, ...)
-
-  -- print ('set', ...)
-
-  local len = select ('#', ...)
-  local key, value = select (len-1, ...)
-  local cutpoint, cutkey
-
-  for i=1,len-2 do
-
-    local key = select (i, ...)
-    local child = parent[key]
-
-    if value == nil then
-      if child == nil then  return
-      elseif next (child, next (child)) then  cutpoint = nil  cutkey = nil
-      elseif cutpoint == nil then  cutpoint = parent  cutkey = key  end
-
-    elseif child == nil then  child = {}  parent[key] = child  end
-
-    parent = child
-    end
-
-  if value == nil and cutpoint then  cutpoint[cutkey] = nil
-  else  parent[key] = value  return value  end
-  end
-
-
-function tget (parent, ...)
-  local len = select ('#', ...)
-  for i=1,len do
-    parent = parent[select (i, ...)]
-    if parent == nil then  break  end
-    end
-  return parent
-  end
-
-
-local pager_lines = 23
-local pager_printed = 0
-local pager_skipping = false
-local pager_filter_pipe = nil
-
-function pager_reset()
-  pager_printed = 0
-  pager_skipping = false
-  if pager_filter_pipe then
-    pager_filter_pipe:close()
-    pager_filter_pipe = nil
-  end
-end
-
-
-function print_more()
-  io.stdout:write(" --More-- ")
-end
-
-function print_nomore()
-  local bs = string.char(8)
-  local bs10 = bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs ..  bs
-  io.stdout:write(bs10 .. "          " .. bs10)
-end
-
-function print_line(txt)
-  if pager_filter_pipe then
-    pager_filter_pipe:write(txt .. "\n")
-    return
-  end
-  if pager_printed >= pager_lines then
-    print_more()
-    local ch = io.stdin:read(1)
-    if ch == " " then
-      pager_printed = 0
-    elseif ch == "\n" then
-      pager_printed = pager_printed - 1
-    elseif ch == "q" then
-      pager_printed = 0
-      pager_skipping = true
-    end
-    print_nomore()
-  end
-  if not pager_skipping then
-    print(txt)
-    pager_printed = pager_printed + 1
-  else
-    -- skip printing
-  end
-end
-
-function paged_write(text)
-  local t = readln.split(text, "[\n]")
-  if string.sub(text, -1) == "\n" then
-    table.insert(t, "")
-  end
-  for i, v in ipairs(t) do
-    if i < #t then
-      print_line(v)
-    else
-      if pager_filter_pipe then
-        pager_filter_pipe:write(v)
-      else
-        io.stdout:write(v)
-      end
-    end
-  end
-end
-
-
-
-
-
-function get_choices(tbl, key)
-  local res = {}
-  for k, v in pairs(tbl) do
-    if string.sub(k, 1, #key) == key then
-      table.insert(res, k)
-    elseif 0 < #key and dotdotdot == k then
-      table.insert(res, k)
-    end
-  end
-  return res
-end
-
-function get_exact_choice(choices, val)
-  local exact_idx = nil
-  local substr_idx = nil
-  local substr_seen = false
-
-  if #choices == 1 then
-    if choices[1] == dotdotdot then
-      return 1
-    elseif string.sub(choices[1], 1, #val) == val then
-      return 1
-    else
-      return nil
-    end
-  else
-    for i, v in ipairs(choices) do
-      if v == val then
-        exact_idx = i
-        substr_seen = true
-      elseif choices[i] ~= dotdotdot and string.sub(choices[i], 1, #val) == val then
-        if substr_seen then
-          substr_idx = nil
-        else
-          substr_idx = i
-          substr_seen = true
-        end
-      elseif choices[i] == dotdotdot then
-        if substr_seen then
-          substr_idx = nil
-        else
-          substr_idx = i
-          substr_seen = true
-        end
-      end
-    end
-  end
-  return exact_idx or substr_idx
-end
-
-function device_cli_help(rl)
-  local key = readln.split(rl.command, "[ ]+")
-  local tree = rl.tree
-  local keylen = #key
-  local fullcmd = ""
-  local error = false
-  local terse = true
-
-  if ((#rl.command >= 1) and (string.sub(rl.command, -1) == " ")) or (#rl.command == 0) then
-    table.insert(key, "")
-    terse = false
-  end
-
-  for i, v in ipairs(key) do
-    local choices = get_choices(tree, v)
-    local idx = get_exact_choice(choices, v)
-    if idx then
-      local choice = choices[idx]
-      tree = tree[choice]
-      fullcmd = fullcmd .. choice .. " "
-    else
-      if i < #key then
-        error = true
-      end
-    end
-
-    if i == #key and not error then
-      for j, w in ipairs(choices) do
-        if terse then
-          paged_write(w .. "\t")
-        else
-          paged_write("  " .. w .. "\n")
-        end
-      end
-      paged_write("\n")
-      if terse then
-        paged_write(" \n")
-      end
-    end
-  end
-  pager_reset()
-end
-
-function device_cli_tab_complete(rl)
-  local key = readln.split(rl.command, "[ ]+")
-  local tree = rl.tree
-  local keylen = #key
-  local fullcmd = ""
-  local error = false
-
-  for i, v in ipairs(key) do
-    local choices = get_choices(tree, v)
-    local idx = get_exact_choice(choices, v)
-    if idx and choices[idx] ~= dotdotdot then
-      local choice = choices[idx]
-      tree = tree[choice]
-      -- print("level " .. i .. " '" .. choice .. "'")
-      fullcmd = fullcmd .. choice .. " "
-    else
-      -- print("level " .. i .. " : " .. table.concat(choices, " ") .. " ")
-      error = true
-    end
-  end
-  if not error then
-    rl.command = fullcmd
-  else
-    -- print("\n\nerror\n")
-  end
-  pager_reset()
-end
-
-function device_cli_exec(rl)
-
-  local cmd_nopipe = rl.command
-  local cmd_pipe = nil
-
-  local pipe1, pipe2 = string.find(rl.command, "[|]")
-  if pipe1 then
-    cmd_nopipe = string.sub(rl.command, 1, pipe1-1)
-    cmd_pipe = string.sub(rl.command, pipe2+1, -1)
-  end
-
-  local key = readln.split(cmd_nopipe .. " ", "[ ]+")
-  local tree = rl.tree
-  local keylen = #key
-  local fullcmd = ""
-  local error = false
-  local func = nil
-
-  if cmd_pipe then
-    pager_filter_pipe = io.popen(cmd_pipe, "w")
-  end
-
-
-  rl.choices = {}
-
-  for i, v in ipairs(key) do
-    local choices = get_choices(tree, v)
-    local idx = get_exact_choice(choices, v)
-    if idx then
-      local choice = choices[idx]
-      if i == #key then
-        func = tree[choice]
-      else
-        if choice == dotdotdot then
-          -- keep the tree the same, update the choice value to match the input string
-          choices[idx] = v
-          choice = v
-        else
-          tree = tree[choice]
-        end
-      end
-      -- print("level " .. i .. " '" .. choice .. "'")
-      table.insert(rl.choices, choice)
-    else
-      -- print("level " .. i .. " : " .. table.concat(choices, " ") .. " ")
-      error = true
-      return nil
-    end
-  end
-  return func
-end
-
-function populate_tree(commands)
-  local tree = {}
-
-  for k, v in pairs(commands) do
-    local key = readln.split(k .. " ", "[ ]+")
-    local xtree = tree
-    for i, kk in ipairs(key) do
-      if i == 1 and kk == "sh" then
-        kk = "show"
-      end
-      if i == #key then
-        if type(v) == "function" then
-          xtree[kk] = v
-        else
-          xtree[kk] = function(rl) paged_write(table.concat(v, "\n") .. "\n") end
-        end
-      else
-        if not xtree[kk] then
-          xtree[kk] = {}
-        end
-        xtree = xtree[kk]
-      end
-    end
-  end
-  return tree
-end
-
-function trim (s)
-  return (string.gsub(s, "^%s*(.-)%s*$", "%1"))
-end
-
-
-function init_vpp(vpp)
-  local root_dir = "/home/ubuntu/vpp"
-  local pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
-
-  vpp:init({ pneum_path = pneum_path })
-
-  vpp:init({ pneum_path = pneum_path })
-  vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
-
-
-
-  vpp:connect("lua_cli")
-end
-
-function run_cli(vpp, cli)
-  local reply = vpp:api_call("cli_inband", { cmd = cli })
-  if reply and #reply == 1 then
-    local rep = reply[1]
-    if 0 == rep.retval then
-      return rep.reply
-    else
-      return "XXXXXLUACLI: API RETVAL ERROR : " .. tostring(rep.retval)
-    end
-  else
-    return "XXXXXLUACLI ERROR, RAW REPLY: " .. vpp.dump(reply)
-  end
-end
-
-
-function toprintablestring(s)
-  if type(s) == "string" then
-    return "\n"..vpp.hex_dump(s)
-  else
-    return tostring(s)
-  end
-end
-
-function interactive_cli(r)
-  while not done do
-    pager_reset()
-    local cmd = r.readln()
-    if not cmd then
-      done = true
-    elseif cmd == "quit" or cmd == "exit" then
-      done = true
-    else
-      local func = device_cli_exec(r)
-      if func then
-	func(r)
-      else
-	if trim(cmd) == "" then
-	else
-	  for i = 1, #r.prompt do
-	    paged_write(" ")
-	  end
-	  paged_write("^\n% Invalid input detected at '^' marker.\n\n")
-	end
-      end
-    end
-  end
-end
-
-device = {}
-device.output = {}
-
-init_vpp(vpp)
-cmds_str = run_cli(vpp, "?")
-vpp_cmds = readln.split(cmds_str, "\n")
-vpp_clis = {}
-
-for linenum, line in ipairs(vpp_cmds) do
-  local m,h = string.match(line, "^  (.-)  (.*)$")
-  if m and #m > 0 then
-    table.insert(vpp_clis, m)
-    device.output["vpp debug cli " .. m] = function(rl)
-      -- print("ARBITRARY CLI" .. vpp.dump(rl.choices))
-      print("LUACLI command: " .. table.concat(rl.choices, " "))
-      local sub = {}
-      --
-      for i=4, #rl.choices -1 do
-        table.insert(sub, rl.choices[i])
-      end
-      local cli = table.concat(sub, " ")
-      print("Running CLI: " .. tostring(cli))
-      paged_write(run_cli(vpp, cli))
-    end
-    device.output["vpp debug cli " .. m .. " " .. dotdotdot] = function(rl)
-      print("ARGH")
-    end
-
-    local ret = run_cli(vpp, "help " .. m)
-    device.output["help vpp debug cli " .. m] = { ret }
-  end
-end
-
-for linenum, line in ipairs(vpp_clis) do
-  -- print(line, ret)
-end
-
-for msgnum, msgname in pairs(vpp.msg_number_to_name) do
-  local cli, numspaces = string.gsub(msgname, "_", " ")
-  device.output["call " .. cli .. " " .. dotdotdot] = function(rl)
-    print("ARGH")
-  end
-  device.output["call " .. cli] = function(rl)
-    print("LUACLI command: " .. table.concat(rl.choices, " "))
-    print("Running API: " .. msgname) -- vpp.dump(rl.choices))
-    local out = {}
-    local args = {}
-    local ntaken = 0
-    local argname = ""
-    for i=(1+1+numspaces+1), #rl.choices-1 do
-      -- print(i, rl.choices[i])
-      if ntaken > 0 then
-        ntaken = ntaken -1
-      else
-        local fieldname = rl.choices[i]
-        local field = vpp.msg_name_to_fields[msgname][fieldname]
-        if field then
-          local s = rl.choices[i+1]
-          s=s:gsub("\\x(%x%x)",function (x) return string.char(tonumber(x,16)) end)
-          args[fieldname] = s
-          ntaken = 1
-        end
-      end
-    end
-    -- print("ARGS: ", vpp.dump(args))
-    local ret = vpp:api_call(msgname, args)
-    for i, reply in ipairs(ret) do
-      table.insert(out, "=================== Entry #" .. tostring(i))
-      for k, v in pairs(reply) do
-        table.insert(out, "   " .. tostring(k) .. " : " .. toprintablestring(v))
-      end
-    end
-    -- paged_write(vpp.dump(ret) .. "\n\n")
-    paged_write(table.concat(out, "\n").."\n\n")
-  end
-  device.output["call " .. cli .. " help"] = function(rl)
-    local out = {}
-    for k, v in pairs(vpp.msg_name_to_fields[msgname]) do
-      table.insert(out, tostring(k) .. " : " .. v["ctype"] .. " ; " .. tostring(vpp.dump(v)) )
-    end
-    -- paged_write(vpp.dump(vpp.msg_name_to_fields[msgname]) .. "\n\n")
-    paged_write(table.concat(out, "\n").."\n\n")
-  end
--- vpp.msg_name_to_number = {}
-end
-
-
-
-local r = readln.reader()
-local done = false
-
-r.prompt = "VPP(luaCLI)#"
-
-r.help = device_cli_help
-r.tab_complete = device_cli_tab_complete
-print("===== CLI view, use ^D to end =====")
-
-r.tree = populate_tree(device.output)
--- readln.pretty("xxxx", r.tree)
-
-
-for idx, an_arg in ipairs(arg) do
-  local fname = an_arg
-  if fname == "-i" then
-    pager_lines = 23
-    interactive_cli(r)
-  else
-    pager_lines = 100000000
-    for line in io.lines(fname) do
-      r.command = line
-      local func = device_cli_exec(r)
-      if func then
-	func(r)
-      end
-    end
-  end
-end
-
-if #arg == 0 then
-  print("You should specify '-i' as an argument for the interactive session,")
-  print("but with no other sources of commands, we start interactive session now anyway")
-   interactive_cli(r)
-end
-
-vpp:disconnect()
-r.done()
-
-
diff --git a/vpp-api/lua/examples/example-acl-plugin.lua b/vpp-api/lua/examples/example-acl-plugin.lua
deleted file mode 100644
index ca01f18d..00000000
--- a/vpp-api/lua/examples/example-acl-plugin.lua
+++ /dev/null
@@ -1,110 +0,0 @@
---[[
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
-
-vpp = require "vpp-lapi"
-
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
-
-vpp:init({ pneum_path = pneum_path })
-
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
-vpp:connect("aytest")
-vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
-
--- api calls
-reply = vpp:api_call("show_version")
-print("Version: ", reply[1].version)
-print(vpp.hex_dump(reply[1].version))
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add", { context = 42, count = 0 })
-print(vpp.dump(reply))
-print("---")
-
-acl_index_to_delete = reply[1].acl_index
-print("Deleting " .. tostring(acl_index_to_delete))
-reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-for ri, rv in ipairs(reply) do 
-  print("Reply message #" .. tostring(ri))
-  print(vpp.dump(rv))
-  for ai, av in ipairs(rv.r) do
-    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
-  end
-   
-end
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-print(vpp.dump(reply))
-print("---")
-
-
-vpp:disconnect()
-
-
diff --git a/vpp-api/lua/examples/example-classifier.lua b/vpp-api/lua/examples/example-classifier.lua
deleted file mode 100644
index ec9c3d3e..00000000
--- a/vpp-api/lua/examples/example-classifier.lua
+++ /dev/null
@@ -1,51 +0,0 @@
---[[
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
-
-local vpp = require "vpp-lapi"
-local bit = require("bit")
-
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
-
-
-vpp:init({ pneum_path = pneum_path })
-
-vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
-
-vpp:connect("aytest")
-
--- api calls
-
-print("Calling API to add a new classifier table")
-reply = vpp:api_call("classify_add_del_table", {
-  context = 43,
-  memory_size = bit.lshift(2, 20),
-  client_index = 42,
-  is_add = 1,
-  nbuckets = 32,
-  skip_n_vectors = 0,
-  match_n_vectors = 1,
-  mask = "\255\255\255\255\255\255\255\255" .. "\255\255\255\255\255\255\255\255"
-})
-print(vpp.dump(reply))
-print("---")
-
-
-vpp:disconnect()
-
-
diff --git a/vpp-api/lua/examples/example-cli.lua b/vpp-api/lua/examples/example-cli.lua
deleted file mode 100644
index 8b84989f..00000000
--- a/vpp-api/lua/examples/example-cli.lua
+++ /dev/null
@@ -1,44 +0,0 @@
---[[
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
-vpp = require "vpp-lapi"
-
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
-
-vpp:init({ pneum_path = pneum_path })
-
-vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
-
-vpp:connect("aytest")
-
--- api calls
-reply = vpp:api_call("show_version")
-print("Version: ", reply[1].version)
-print(vpp.hex_dump(reply[1].version))
-print(vpp.dump(reply))
-print("---")
-
-
-reply = vpp:api_call("cli_inband", { cmd = "show vers" }) 
-print(vpp.dump(reply))
-print("---")
-
-
-vpp:disconnect()
-
-
diff --git a/vpp-api/lua/examples/lute/README.md b/vpp-api/lua/examples/lute/README.md
deleted file mode 100644
index 8d37250a..00000000
--- a/vpp-api/lua/examples/lute/README.md
+++ /dev/null
@@ -1,66 +0,0 @@
-LUTE: Lua Unit Test Environment
-
-This is a small helper utility to automate some simple tests
-that one might need to do.
-
-Think of it as a hybrid of a screen and expect who
-also took some habits from HTML inline code.
-
-It is quite probably useless for building anything serious,
-but practice shows it is quite efficient at allowing
-convenient temporary quick tests, and for something
-that was written over a course of a couple of evenings it
-is quite a nice little helper tool.
-
-It allows do launch and drive multiple shell sessions,
-and by virtue of having been written in Lua, it of course
-also allows to add the business logic using the Lua code.
-
-If you launch the lute without parameters, it gives you
-the interactive shell to execute the commands in.
-
-If you launch it with an argument, it will attempt to
-read and execute the commands from the file.
-
-Commands:
-
-shell FOO
-
-  spawn a shell in a new PTY under the label FOO.
-
-run FOO bar
-
-  Send "bar" keystrokes followed by "ENTER" to the session FOO
-
-  Special case: "break" word on its own gets translated into ^C being sent.
-
-cd FOO
-
-  "change domain" into session FOO. All subsequent inputs will go,
-  line-buffered, into the session FOO. To jump back up, use ^D (Control-D),
-  or within the file, use ^D^D^D (caret D caret D caret D on its own line)
-
-expect FOO blablabla
-
-  Pause further interpretation of the batch mode until you see "blablabla"
-  in the output of session FOO, or until timeout happens.
-
-sleep N
-
-  Sleep an integer N seconds, if you are in batch mode.
-
-echo blabla
-
-  Echo the remainder of the line to standard output.
-
-For Lua code, there is a pre-existing pseudo-session called "lua",
-which accepts "run lua" command which does what you would expect
-(evaluate the rest of the string in Lua context - being the same
-as lute itself). Also you can do "cd lua" and get into a
-multiline-enabled interpreter shell.
-
-This way for the VPP case you can automate some of the things in your routine
-that you would have to have done manually, and test drive API as well
-as use the realistic native OS components to create the environment around it.
-
-
diff --git a/vpp-api/lua/examples/lute/lute.lua b/vpp-api/lua/examples/lute/lute.lua
deleted file mode 100644
index 89b9924b..00000000
--- a/vpp-api/lua/examples/lute/lute.lua
+++ /dev/null
@@ -1,777 +0,0 @@
---[[
-version = 1
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
--- LUTE: Lua Unit Test Environment
--- AKA what happens when screen tries to marry with lua and expect,
--- but escapes mid-ceremony.
---
--- comments: @ayourtch
-
-ffi = require("ffi")
-
-vpp = {}
-function vpp.dump(o)
-   if type(o) == 'table' then
-      local s = '{ '
-      for k,v in pairs(o) do
-         if type(k) ~= 'number' then k = '"'..k..'"' end
-         s = s .. '['..k..'] = ' .. vpp.dump(v) .. ','
-      end
-      return s .. '} '
-   else
-      return tostring(o)
-   end
-end
-
-
-ffi.cdef([[
-
-int posix_openpt(int flags);
-int grantpt(int fd);
-int unlockpt(int fd);
-char *ptsname(int fd);
-
-typedef long pid_t;
-typedef long ssize_t;
-typedef long size_t;
-typedef int nfds_t;
-typedef long time_t;
-typedef long suseconds_t;
-
-pid_t fork(void);
-pid_t setsid(void);
-
-int close(int fd);
-int open(char *pathname, int flags);
-
-int dup2(int oldfd, int newfd);
-
-ssize_t read(int fd, void *buf, size_t count);
-ssize_t write(int fd, const void *buf, size_t count);
-
-struct pollfd {
-               int   fd;         /* file descriptor */
-               short events;     /* requested events */
-               short revents;    /* returned events */
-           };
-
-int poll(struct pollfd *fds, nfds_t nfds, int timeout);
-
-struct timeval {
-               time_t      tv_sec;     /* seconds */
-               suseconds_t tv_usec;    /* microseconds */
-           };
-
-int gettimeofday(struct timeval *tv, struct timezone *tz);
-
-int inet_pton(int af, const char *src, void *dst);
-
-]])
-
-ffi.cdef([[
-void *memset(void *s, int c, size_t n);
-void *memcpy(void *dest, void *src, size_t n);
-void *memmove(void *dest, const void *src, size_t n);
-void *memmem(const void *haystack, size_t haystacklen,
-        const void *needle, size_t needlelen);
-]])
-
-
-
-local O_RDWR = 2
-
-
-function os_time()
-  local tv = ffi.new("struct timeval[1]")
-  local ret = ffi.C.gettimeofday(tv, nil)
-  return tonumber(tv[0].tv_sec) + (tonumber(tv[0].tv_usec)/1000000.0)
-end
-
-function sleep(n)
-  local when_wakeup = os_time() + n
-  while os_time() <= when_wakeup do
-    ffi.C.poll(nil, 0, 10)
-  end
-end
-
-
-function c_str(text_in)
-  local text = text_in 
-  local c_str = ffi.new("char[?]", #text+1)
-  ffi.copy(c_str, text)
-  return c_str
-end
-
-function ip46(addr_text)
-  local out = ffi.new("char [200]")
-  local AF_INET6 = 10
-  local AF_INET = 2
-  local is_ip6 = ffi.C.inet_pton(AF_INET6, c_str(addr_text), out)
-  if is_ip6 == 1 then
-    return ffi.string(out, 16), true
-  end
-  local is_ip4 = ffi.C.inet_pton(AF_INET, c_str(addr_text), out)
-  if is_ip4 then
-    return (string.rep("4", 12).. ffi.string(out, 4)), false
-  end
-end
-
-function pty_master_open()
-  local fd = ffi.C.posix_openpt(O_RDWR)
-  ffi.C.grantpt(fd)
-  ffi.C.unlockpt(fd)
-  local p = ffi.C.ptsname(fd)
-  print("PTS:" .. ffi.string(p))
-  return fd, ffi.string(p)
-end
-
-function pty_run(cmd)
-  local master_fd, pts_name = pty_master_open()
-  local child_pid = ffi.C.fork()
-  if (child_pid == -1) then
-    print("Error fork()ing")
-    return -1
-  end 
- 
-  if child_pid ~= 0 then
-    -- print("Parent")
-    return master_fd, child_pid
-  end
-
-  -- print("Child")
-  if (ffi.C.setsid() == -1) then
-    print("Child error setsid")
-    os.exit(-1)
-  end
-
-  ffi.C.close(master_fd)
-
-  local slave_fd = ffi.C.open(c_str(pts_name), O_RDWR)
-  if slave_fd == -1 then
-    print("Child can not open slave fd")
-    os.exit(-2)
-  end
-
-  ffi.C.dup2(slave_fd, 0)
-  ffi.C.dup2(slave_fd, 1)
-  ffi.C.dup2(slave_fd, 2)
-  os.execute(cmd)
-end
-
-function readch()
-  local buf = ffi.new("char[1]")
-  local nread= ffi.C.read(0, buf, 1)
-  -- print("\nREADCH : " .. string.char(buf[0]))
-  return string.char(buf[0])
-end
-
-function stdout_write(str)
-  ffi.C.write(1, c_str(str), #str)
-end
-
-
-readln = {
-split = function(str, pat)
-  local t = {}  -- NOTE: use {n = 0} in Lua-5.0
-  local fpat = "(.-)" .. pat
-  local last_end = 1
-  if str then
-    local s, e, cap = str:find(fpat, 1)
-    while s do
-      if s ~= 1 or cap ~= "" then
-        table.insert(t,cap)
-      end
-      last_end = e+1
-      s, e, cap = str:find(fpat, last_end)
-    end
-    if last_end <= #str then
-      cap = str:sub(last_end)
-      table.insert(t, cap)
-    end
-  end
-  return t
-end,
-
-reader = function()
-  local rl = {}
-
-  rl.init = function()
-    os.execute("stty -icanon min 1 -echo")
-    rl.rawmode = true
-  end
-
-  rl.done = function()
-    os.execute("stty icanon echo")
-    rl.rawmode = false
-  end
-
-  rl.prompt = ">"
-  rl.history = { "" }
-  rl.history_index = 1
-  rl.history_length = 1
-
-  rl.hide_cmd = function()
-    local bs = string.char(8) .. " " .. string.char(8)
-    for i = 1, #rl.command do
-      stdout_write(bs)
-    end
-  end
-
-  rl.show_cmd = function()
-    if rl.command then
-      stdout_write(rl.command)
-    end
-  end
-
-  rl.store_history = function(cmd)
-    if cmd == "" then
-      return
-    end
-    rl.history[rl.history_length] = cmd
-    rl.history_length = rl.history_length + 1
-    rl.history_index = rl.history_length
-    rl.history[rl.history_length] = ""
-  end
-
-  rl.readln = function(stdin_select_fn, batch_cmd, batch_when, batch_expect)
-    local done = false
-    local need_prompt = true
-    rl.command = ""
-
-    if not rl.rawmode then
-      rl.init()
-    end
-
-    while not done do
-      local indent_value = #rl.prompt + #rl.command
-      if need_prompt then
-        stdout_write(rl.prompt)
-        stdout_write(rl.command)
-        need_prompt = false
-      end
-      if type(stdin_select_fn) == "function" then
-        while not stdin_select_fn(indent_value, batch_cmd, batch_when, batch_expect) do
-          stdout_write(rl.prompt)
-          stdout_write(rl.command)
-          indent_value = #rl.prompt + #rl.command
-        end
-        if batch_cmd and ((os_time() > batch_when) or (batch_expect and expect_success(batch_expect, buf, 0))) then
-          stdout_write("\n" .. rl.prompt .. batch_cmd .. "\n")
-          if batch_expect then
-            expect_done(batch_expect)
-          end
-          return batch_cmd, batch_expect
-        end
-      end
-      local ch = readch()
-      if ch:byte(1) == 27 then
-        -- CONTROL
-        local ch2 = readch()
-        -- arrows
-        if ch2:byte(1) == 91 then
-          local ch3 = readch()
-          local b = ch3:byte(1)
-          if b == 65 then
-            ch = "UP"
-          elseif b == 66 then
-            ch = "DOWN"
-          elseif b == 67 then
-            ch = "RIGHT"
-          elseif b == 68 then
-            ch = "LEFT"
-          end
-          -- print("Byte: " .. ch3:byte(1))
-          -- if ch3:byte(1)
-        end
-      end
-
-      if ch == "?" then
-        stdout_write(ch)
-        stdout_write("\n")
-        if rl.help then
-          rl.help(rl)
-        end
-        need_prompt = true
-      elseif ch == "\t" then
-        if rl.tab_complete then
-          rl.tab_complete(rl)
-        end
-        stdout_write("\n")
-        need_prompt = true
-      elseif ch == "\n" then
-        stdout_write(ch)
-        done = true
-      elseif ch == "\004" then
-        stdout_write("\n")
-        rl.command = nil
-        done = true
-      elseif ch == string.char(127) then
-        if rl.command ~= "" then
-          stdout_write(string.char(8) .. " " .. string.char(8))
-          rl.command = string.sub(rl.command, 1, -2)
-        end
-      elseif #ch > 1 then
-        -- control char
-        if ch == "UP" then
-          rl.hide_cmd()
-          if rl.history_index == #rl.history then
-            rl.history[rl.history_index] = rl.command
-          end
-          if rl.history_index > 1 then
-            rl.history_index = rl.history_index - 1
-            rl.command = rl.history[rl.history_index]
-          end
-          rl.show_cmd()
-        elseif ch == "DOWN" then
-          rl.hide_cmd()
-          if rl.history_index < rl.history_length then
-            rl.history_index = rl.history_index + 1
-            rl.command = rl.history[rl.history_index]
-          end
-          rl.show_cmd()
-        end
-      else
-        stdout_write(ch)
-        rl.command = rl.command .. ch
-      end
-    end
-    if rl.command then
-      rl.store_history(rl.command)
-    end
-    return rl.command
-  end
-  return rl
-end
-
-}
-
-local select_fds = {}
-local sessions = {}
-
-local line_erased = false
-
-function erase_line(indent)
-  if not line_erased then
-    line_erased = true
-    stdout_write(string.rep(string.char(8), indent)..string.rep(" ", indent)..string.rep(string.char(8), indent))
-  end
-end
-
-function do_select_stdin(indent, batch_cmd, batch_when, batch_expect)
-  while true do
-    local nfds = 1+#select_fds
-    local pfds = ffi.new("struct pollfd[?]", nfds)
-    pfds[0].fd = 0;
-    pfds[0].events = 1;
-    pfds[0].revents = 0;
-    for i = 1,#select_fds do
-      pfds[i].fd = select_fds[i].fd
-      pfds[i].events = 1
-      pfds[i].revents = 0
-    end
-    if batch_cmd and ((os_time() > batch_when) or (batch_expect and expect_success(batch_expect, buf, 0))) then
-      return true
-    end
-    while ffi.C.poll(pfds, nfds, 10) == 0 do
-      if batch_cmd and ((os_time() > batch_when) or (batch_expect and expect_success(batch_expect, buf, 0))) then
-        return true
-      end
-      if line_erased then
-        line_erased = false
-        return false
-      end
-    end
-    if pfds[0].revents == 1 then
-      return true
-    end
-    for i = 1,#select_fds do
-      if(pfds[i].revents > 0) then
-        if pfds[i].fd ~= select_fds[i].fd then
-          print("File descriptors unequal", pfds[i].fd, select_fds[i].fd)
-        end
-        select_fds[i].cb(select_fds[i], pfds[i].revents, indent)
-      end
-    end
-  end
-end
-
-local buf = ffi.new("char [32768]")
-
-function session_stdout_write(prefix, data)
-  data = prefix .. data:gsub("\n", "\n"..prefix):gsub("\n"..prefix.."$", "\n")
-  
-  stdout_write(data)
-end
-
-function expect_success(sok, buf, nread)
-  local expect_buf_sz = ffi.sizeof(sok.expect_buf) - 128
-  local expect_buf_avail = expect_buf_sz - sok.expect_buf_idx
-  -- print("EXPECT_SUCCESS: nread ".. tostring(nread).. " expect_buf_idx: " .. tostring(sok.expect_buf_idx) .. " expect_buf_avail: " .. tostring(expect_buf_avail) )
-  if expect_buf_avail < 0 then
-    print "EXPECT BUFFER OVERRUN ALREADY"
-    os.exit(1)
-  end
-  if expect_buf_avail < nread then
-    if (nread >= ffi.sizeof(sok.expect_buf)) then
-      print("Read too large of a chunk to fit into expect buffer")
-      return nil
-    end
-    local delta = nread - expect_buf_avail
-
-    ffi.C.memmove(sok.expect_buf, sok.expect_buf + delta, expect_buf_sz - delta)
-    sok.expect_buf_idx = sok.expect_buf_idx - delta
-    expect_buf_avail = nread 
-  end
-  if sok.expect_buf_idx + nread > expect_buf_sz then
-    print("ERROR, I have just overrun the buffer !")
-    os.exit(1)
-  end
-  ffi.C.memcpy(sok.expect_buf + sok.expect_buf_idx, buf, nread)
-  sok.expect_buf_idx = sok.expect_buf_idx + nread
-  if sok.expect_str == nil then
-    return true
-  end
-  local match_p = ffi.C.memmem(sok.expect_buf, sok.expect_buf_idx, sok.expect_str, sok.expect_str_len)
-  if match_p ~= nil then
-    return true
-  end
-  return false
-end
-
-function expect_done(sok)
-  local expect_buf_sz = ffi.sizeof(sok.expect_buf) - 128
-  if not sok.expect_str then 
-    return false
-  end
-  local match_p = ffi.C.memmem(sok.expect_buf, sok.expect_buf_idx, sok.expect_str, sok.expect_str_len)
-  if match_p ~= nil then
-    if sok.expect_cb then
-      sok.expect_cb(sok)
-    end
-    local match_idx = ffi.cast("char *", match_p) - ffi.cast("char *", sok.expect_buf)
-    ffi.C.memmove(sok.expect_buf, ffi.cast("char *", match_p) + sok.expect_str_len, expect_buf_sz - match_idx - sok.expect_str_len)
-    sok.expect_buf_idx = match_idx + sok.expect_str_len
-    sok.expect_success = true
-
-    sok.expect_str = nil
-    sok.expect_str_len = 0
-    return true
-  end
-end
-
-function slave_events(sok, revents, indent)
-  local fd = sok.fd
-  local nread = ffi.C.read(fd, buf, ffi.sizeof(buf)-128)
-  local idx = nread - 1
-  while idx >= 0 and buf[idx] ~= 10 do
-    idx = idx - 1
-  end
-  if idx >= 0 then
-    erase_line(indent)
-    session_stdout_write(sok.prefix, sok.buf .. ffi.string(buf, idx+1))
-    sok.buf = ""
-  end
-  sok.buf = sok.buf .. ffi.string(buf+idx+1, nread-idx-1)
-  -- print("\nRead: " .. tostring(nread))
-  -- stdout_write(ffi.string(buf, nread))
-  if expect_success(sok, buf, nread) then
-    return true
-  end
-  return false
-end
-
-
-function start_session(name)
-  local mfd, cpid = pty_run("/bin/bash")
-  local sok =  { ["fd"] = mfd, ["cb"] = slave_events, ["buf"] = "", ["prefix"] = name .. ":", ["expect_buf"] = ffi.new("char [165536]"), ["expect_buf_idx"] = 0, ["expect_str"] = nil }
-  table.insert(select_fds, sok)
-  sessions[name] = sok
-end
-
-function command_transform(exe)
-  if exe == "break" then
-    exe = string.char(3)
-  end
-  return exe
-end
-
-function session_write(a_session, a_str)
-  if has_session(a_session) then
-    return tonumber(ffi.C.write(sessions[a_session].fd, c_str(a_str), #a_str))
-  else
-    return 0
-  end
-end
-
-function session_exec(a_session, a_cmd)
-  local exe = command_transform(a_cmd) .. "\n"
-  session_write(a_session, exe)
-end
-
-function session_cmd(ui, a_session, a_cmd)
-  if not has_session(a_session) then
-    stdout_write("ERR: No such session '" .. tostring(a_session) .. "'\n")
-    return nil
-  end
-  if a_session == "lua" then
-    local func, msg = loadstring(ui.lua_acc .. a_cmd)
-    -- stdout_write("LOADSTR: " .. vpp.dump({ ret, msg }) .. "\n")
-    if not func and string.match(msg, "") then
-      if a_session ~= ui.in_session then
-         stdout_write("ERR LOADSTR: " .. tostring(msg) .. "\n")
-         return nil
-      end
-      ui.lua_acc = ui.lua_acc .. a_cmd  .. "\n"
-      return true
-    end
-    ui.lua_acc = ""
-    local ret, msg = pcall(func)
-    if ret then
-      return true
-    else
-      stdout_write("ERR: " .. msg .. "\n") 
-      return nil
-    end
-  else
-    session_exec(a_session, a_cmd)
-    if ui.session_cmd_delay then
-      return { "delay", ui.session_cmd_delay }
-    end
-    return true
-  end
-end
-
-function has_session(a_session)
-  if a_session == "lua" then
-    return true
-  end
-  return (sessions[a_session] ~= nil)
-end
-
-function command_match(list, input, output)
-  for i, v in ipairs(list) do
-    local m = {}
-    m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8], m[9] = string.match(input, v[1])
-    -- print("MATCH: ", vpp.dump(m))
-    if m[1] then
-       output["result"] = m
-       output["result_index"] = i
-       return m
-    end 
-  end
-  return nil
-end
-
-function cmd_spawn_shell(ui, a_arg)
-  start_session(a_arg[1])
-  return true
-end
-
-function cmd_run_cmd(ui, a_arg)
-  local a_sess = a_arg[1]
-  local a_cmd = a_arg[2]
-  return session_cmd(ui, a_sess, a_cmd)
-end
-
-function cmd_cd(ui, a_arg)
-  local a_sess = a_arg[1]
-  if has_session(a_sess) then
-    ui.in_session = a_sess
-    return true
-  else
-    stdout_write("ERR: Unknown session '".. tostring(a_sess) .. "'\n")
-    return nil
-  end
-end
-
-function cmd_sleep(ui, a_arg)
-  return { "delay", tonumber(a_arg[1]) }
-end
-
-function cmd_expect(ui, a_arg)
-  local a_sess = a_arg[1]
-  local a_expect = a_arg[2]
-  local sok = sessions[a_sess]
-  if not sok then
-    stdout_write("ERR: unknown session '" .. tostring(a_sess) .. "'\n")
-    return nil
-  end
-  sok.expect_str = c_str(a_expect)
-  sok.expect_str_len = #a_expect
-  return { "expect", a_sess }
-end
-
-function cmd_info(ui, a_arg)
-  local a_sess = a_arg[1]
-  local sok = sessions[a_sess]
-  if not sok then
-    stdout_write("ERR: unknown session '" .. tostring(a_sess) .. "'\n")
-    return nil
-  end
-  print("Info for session " .. tostring(a_sess) .. "\n")
-  print("Expect buffer index: " .. tostring(sok.expect_buf_idx))
-  print("Expect buffer: '" .. tostring(ffi.string(sok.expect_buf, sok.expect_buf_idx)) .. "'\n")
-  if sok.expect_str then
-    print("Expect string: '" .. tostring(ffi.string(sok.expect_str, sok.expect_str_len)) .. "'\n")
-  else
-    print("Expect string not set\n")
-  end
-end
-
-function cmd_echo(ui, a_arg)
-  local a_data = a_arg[1]
-  print("ECHO: " .. tostring(a_data))
-end
-
-main_command_table = {
-  { "^shell ([a-zA-Z0-9_]+)$", cmd_spawn_shell },
-  { "^run ([a-zA-Z0-9_]+) (.+)$", cmd_run_cmd },
-  { "^cd ([a-zA-Z0-9_]+)$", cmd_cd  },
-  { "^sleep ([0-9]+)$", cmd_sleep },
-  { "^expect ([a-zA-Z0-9_]+) (.-)$", cmd_expect },
-  { "^info ([a-zA-Z0-9_]+)$", cmd_info },
-  { "^echo (.-)$", cmd_echo }
-}
-
-
-
-function ui_set_prompt(ui)
-  if ui.in_session then 
-    if ui.in_session == "lua" then
-      if #ui.lua_acc > 0 then
-        ui.r.prompt = ui.in_session .. ">>"
-      else
-        ui.r.prompt = ui.in_session .. ">"
-      end 
-    else
-      ui.r.prompt = ui.in_session .. "> "
-    end
-  else
-    ui.r.prompt = "> "
-  end
-  return ui.r.prompt
-end
-
-function ui_run_command(ui, cmd)
-  -- stdout_write("Command: " .. tostring(cmd) .. "\n")
-  local ret = false
-  if ui.in_session then
-    if cmd then
-      if cmd == "^D^D^D" then
-        ui.in_session = nil
-        ret = true
-      else
-        ret = session_cmd(ui, ui.in_session, cmd)
-      end
-    else
-      ui.in_session = nil
-      ret = true
-    end
-  else  
-    if cmd then
-      local out = {}
-      if cmd == "" then
-        ret = true
-      end
-      if command_match(main_command_table, cmd, out) then
-        local i = out.result_index
-        local m = out.result
-        if main_command_table[i][2] then
-          ret = main_command_table[i][2](ui, m)
-        end
-      end
-    end
-    if not cmd or cmd == "quit" then
-      return "quit"
-    end
-  end
-  return ret
-end
-
-local ui = {}
-ui.in_session = nil
-ui.r = readln.reader() 
-ui.lua_acc = ""
-ui.session_cmd_delay = 0.3
-
-local lines = ""
-
-local done = false
--- a helper function which always returns nil
-local no_next_line = function() return nil end
-
--- a function which returns the next batch line
-local next_line = no_next_line
-
-local batchfile = arg[1]
-
-if batchfile then
-  local f = io.lines(batchfile)
-  next_line = function() 
-    local line = f()
-    if line then 
-      return line
-    else 
-      next_line = no_next_line
-      session_stdout_write(batchfile .. ":", "End of batch\n")
-      return nil
-    end
-  end
-end
-
-
-local batch_when = 0
-local batch_expect = nil
-while not done do
-  local prompt = ui_set_prompt(ui)
-  local batch_cmd = next_line() 
-  local cmd, expect_sok = ui.r.readln(do_select_stdin, batch_cmd, batch_when, batch_expect)
-  if expect_sok and not expect_success(expect_sok, buf, 0) then
-    if not cmd_ret and next_line ~= no_next_line then
-      print("ERR: expect timeout\n")
-      next_line = no_next_line
-    end
-  else 
-    local cmd_ret = ui_run_command(ui, cmd)
-    if not cmd_ret and next_line ~= no_next_line then
-      print("ERR: Error during batch execution\n")
-      next_line = no_next_line
-    end
-
-    if cmd_ret  == "quit" then
-      done = true
-    end
-    batch_expect = nil
-    batch_when = 0
-    if type(cmd_ret) == "table" then
-      if cmd_ret[1] == "delay" then
-	batch_when = os_time() + tonumber(cmd_ret[2])
-      end
-      if cmd_ret[1] == "expect" then
-	batch_expect = sessions[cmd_ret[2]]
-	batch_when = os_time() + 15
-      end
-    end
-  end
-end
-ui.r.done()
-
-os.exit(1)
-
-
-
diff --git a/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute b/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute
deleted file mode 100644
index a24d04bf..00000000
--- a/vpp-api/lua/examples/lute/script-inout-acl-noacl.lute
+++ /dev/null
@@ -1,329 +0,0 @@
-shell vppbuild
-run vppbuild stty -echo
-run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
-expect vppbuild ALLGOOD
-
-shell s0
-shell s1
-shell s2
-
-
-cd s1
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-cd s2
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-
-cd lua
-
-function session_get_bash_pid(s)
-  if not has_session(s) then
-    return nil
-  end
-  local fname = "/tmp/lute-"..s.."-pid.txt"
-
-  session_exec(s, "echo $$ >" .. fname)
-  -- it's a dirty hack but it's quick
-  sleep(0.5)
-  local pid = io.lines(fname)()
-  print("Got pid for " .. s .. " : " .. tostring(pid))
-  return(tonumber(pid))
-end
-
-function session_connect_with(s0, s1)
-  -- local pid0 = tostring(session_get_bash_pid(s0))
-  local pid1 = tostring(session_get_bash_pid(s1))
-  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
-  local this_end = s0 .. "_" .. s1
-  local other_end = s1 .. "_" .. s0
-  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
-  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
-  for i, option in ipairs(eth_options) do
-    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
-    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
-  end
-  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
-  sleep(0.5)
-end
-
-^D^D^D
-run lua session_connect_with("s0", "s1")
-run lua session_connect_with("s0", "s2")
-
-cd s1
-ip -6 addr add dev s1_s0 2001:db8:1::1/64
-ip -4 addr add dev s1_s0 192.0.2.1/24
-ip link set dev s1_s0 up promisc on
-^D^D^D
-
-cd s2
-ip -6 addr add dev s2_s0 2001:db8:1::2/64
-ip -6 addr add dev s2_s0 2001:db8:1::3/64
-ip -6 addr add dev s2_s0 2001:db8:1::4/64
-ip -4 addr add dev s2_s0 192.0.2.2/24
-ip -4 addr add dev s2_s0:1 192.0.2.3/24
-ip -4 addr add dev s2_s0:2 192.0.2.4/24
-ip link set dev s2_s0 up promisc on
-^D^D^D
-
-run s1 ip addr
-run s2 ip addr
-shell VPP
-cd VPP
-cd /home/ubuntu/vpp
-make debug 
-r
-^D^D^D
-expect VPP DBGvpp#
-
-cd lua
--- Initialization of the Lua environment for talking to VPP
-vpp = require("vpp-lapi")
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
-vpp:init({ pneum_path = pneum_path })
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
-vpp:connect("aytest")
-vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
-
-^D^D^D
-
-cd lua
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
-vpp_if_to_s1 = reply[1].sw_if_index
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
-vpp_if_to_s2 = reply[1].sw_if_index
-
-ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
-
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-
-bd_id = 42
-
-reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
-print(vpp.dump(reply))
-
-for i, v in ipairs(ifaces) do
-  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
-  print(vpp.dump(reply))
-end
-
-^D^D^D
-
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.4
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-
-
-cd lua
---- ACL testing
-
---[[ temporary comment out
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add_replace", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add_replace", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add_replace", { context = 42, count = 0 })
-print(vpp.dump(reply))
-print("---")
-
-acl_index_to_delete = reply[1].acl_index
-print("Deleting " .. tostring(acl_index_to_delete))
-reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-for ri, rv in ipairs(reply) do
-  print("Reply message #" .. tostring(ri))
-  print(vpp.dump(rv))
-  for ai, av in ipairs(rv.r) do
-    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
-  end
-
-end
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
-print(vpp.dump(reply))
-print("---")
-
-
-]] -- end of comment out
-
----- Should be nothing ^^
-r = {
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
-  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
-  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
-}
-
-reply = vpp:api_call("acl_add_replace", { context = 42, count = 5, r = r })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add_replace", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_in
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
---reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
--- print(vpp.dump(reply))
---print("---")
-
-^D^D^D
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 0
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run VPP show trace 
-expect VPP match: inacl 0 rule 1
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 2
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 3
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 4
-
-
-cd lua
-
---- TEST OUTBOUND ACL
-
-r1 = {
-  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 }
-}
-
-reply = vpp:api_call("acl_add_replace", { context = 42, count = 3, r = r1 })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-
-^D^D^D
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: outacl 2 rule 0
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run VPP show trace 
-expect VPP match: inacl 0 rule 1
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-run VPP show trace
-expect VPP match: outacl 2 rule 1
-
-run lua print("ALL GOOD!")
-
diff --git a/vpp-api/lua/examples/lute/script-inout-acl-old.lute b/vpp-api/lua/examples/lute/script-inout-acl-old.lute
deleted file mode 100644
index 9edebf02..00000000
--- a/vpp-api/lua/examples/lute/script-inout-acl-old.lute
+++ /dev/null
@@ -1,329 +0,0 @@
-shell vppbuild
-run vppbuild stty -echo
-run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
-expect vppbuild ALLGOOD
-
-shell s0
-shell s1
-shell s2
-
-
-cd s1
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-cd s2
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-
-cd lua
-
-function session_get_bash_pid(s)
-  if not has_session(s) then
-    return nil
-  end
-  local fname = "/tmp/lute-"..s.."-pid.txt"
-
-  session_exec(s, "echo $$ >" .. fname)
-  -- it's a dirty hack but it's quick
-  sleep(0.5)
-  local pid = io.lines(fname)()
-  print("Got pid for " .. s .. " : " .. tostring(pid))
-  return(tonumber(pid))
-end
-
-function session_connect_with(s0, s1)
-  -- local pid0 = tostring(session_get_bash_pid(s0))
-  local pid1 = tostring(session_get_bash_pid(s1))
-  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
-  local this_end = s0 .. "_" .. s1
-  local other_end = s1 .. "_" .. s0
-  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
-  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
-  for i, option in ipairs(eth_options) do
-    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
-    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
-  end
-  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
-  sleep(0.5)
-end
-
-^D^D^D
-run lua session_connect_with("s0", "s1")
-run lua session_connect_with("s0", "s2")
-
-cd s1
-ip -6 addr add dev s1_s0 2001:db8:1::1/64
-ip -4 addr add dev s1_s0 192.0.2.1/24
-ip link set dev s1_s0 up promisc on
-^D^D^D
-
-cd s2
-ip -6 addr add dev s2_s0 2001:db8:1::2/64
-ip -6 addr add dev s2_s0 2001:db8:1::3/64
-ip -6 addr add dev s2_s0 2001:db8:1::4/64
-ip -4 addr add dev s2_s0 192.0.2.2/24
-ip -4 addr add dev s2_s0:1 192.0.2.3/24
-ip -4 addr add dev s2_s0:2 192.0.2.4/24
-ip link set dev s2_s0 up promisc on
-^D^D^D
-
-run s1 ip addr
-run s2 ip addr
-shell VPP
-cd VPP
-cd /home/ubuntu/vpp
-make debug 
-r
-^D^D^D
-expect VPP DBGvpp#
-
-cd lua
--- Initialization of the Lua environment for talking to VPP
-vpp = require("vpp-lapi")
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
-vpp:init({ pneum_path = pneum_path })
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
-vpp:connect("aytest")
-vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
-
-^D^D^D
-
-cd lua
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
-vpp_if_to_s1 = reply[1].sw_if_index
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
-vpp_if_to_s2 = reply[1].sw_if_index
-
-ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
-
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-
-bd_id = 42
-
-reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
-print(vpp.dump(reply))
-
-for i, v in ipairs(ifaces) do
-  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
-  print(vpp.dump(reply))
-end
-
-^D^D^D
-
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.4
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-
-
-cd lua
---- ACL testing
-
---[[ temporary comment out
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add", { context = 42, count = 0 })
-print(vpp.dump(reply))
-print("---")
-
-acl_index_to_delete = reply[1].acl_index
-print("Deleting " .. tostring(acl_index_to_delete))
-reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-for ri, rv in ipairs(reply) do
-  print("Reply message #" .. tostring(ri))
-  print(vpp.dump(rv))
-  for ai, av in ipairs(rv.r) do
-    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
-  end
-
-end
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
-print(vpp.dump(reply))
-print("---")
-
-
-]] -- end of comment out
-
----- Should be nothing ^^
-r = {
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
-  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
-  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
-}
-
-reply = vpp:api_call("acl_add", { context = 42, count = 5, r = r })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_in
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
---reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
--- print(vpp.dump(reply))
---print("---")
-
-^D^D^D
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 0
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run VPP show trace 
-expect VPP match: inacl 0 rule 1
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 2
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 3
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 4
-
-
-cd lua
-
---- TEST OUTBOUND ACL
-
-r1 = {
-  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 }
-}
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = r1 })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-
-^D^D^D
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: outacl 2 rule 0
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run VPP show trace 
-expect VPP match: inacl 0 rule 1
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-run VPP show trace
-expect VPP match: outacl 2 rule 1
-
-run lua print("ALL GOOD!")
-
diff --git a/vpp-api/lua/examples/lute/script-inout-acl.lute b/vpp-api/lua/examples/lute/script-inout-acl.lute
deleted file mode 100644
index d7e7423c..00000000
--- a/vpp-api/lua/examples/lute/script-inout-acl.lute
+++ /dev/null
@@ -1,329 +0,0 @@
-shell vppbuild
-run vppbuild stty -echo
-run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
-expect vppbuild ALLGOOD
-
-shell s0
-shell s1
-shell s2
-
-
-cd s1
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-cd s2
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-
-cd lua
-
-function session_get_bash_pid(s)
-  if not has_session(s) then
-    return nil
-  end
-  local fname = "/tmp/lute-"..s.."-pid.txt"
-
-  session_exec(s, "echo $$ >" .. fname)
-  -- it's a dirty hack but it's quick
-  sleep(0.5)
-  local pid = io.lines(fname)()
-  print("Got pid for " .. s .. " : " .. tostring(pid))
-  return(tonumber(pid))
-end
-
-function session_connect_with(s0, s1)
-  -- local pid0 = tostring(session_get_bash_pid(s0))
-  local pid1 = tostring(session_get_bash_pid(s1))
-  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
-  local this_end = s0 .. "_" .. s1
-  local other_end = s1 .. "_" .. s0
-  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
-  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
-  for i, option in ipairs(eth_options) do
-    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
-    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
-  end
-  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
-  sleep(0.5)
-end
-
-^D^D^D
-run lua session_connect_with("s0", "s1")
-run lua session_connect_with("s0", "s2")
-
-cd s1
-ip -6 addr add dev s1_s0 2001:db8:1::1/64
-ip -4 addr add dev s1_s0 192.0.2.1/24
-ip link set dev s1_s0 up promisc on
-^D^D^D
-
-cd s2
-ip -6 addr add dev s2_s0 2001:db8:1::2/64
-ip -6 addr add dev s2_s0 2001:db8:1::3/64
-ip -6 addr add dev s2_s0 2001:db8:1::4/64
-ip -4 addr add dev s2_s0 192.0.2.2/24
-ip -4 addr add dev s2_s0:1 192.0.2.3/24
-ip -4 addr add dev s2_s0:2 192.0.2.4/24
-ip link set dev s2_s0 up promisc on
-^D^D^D
-
-run s1 ip addr
-run s2 ip addr
-shell VPP
-cd VPP
-cd /home/ubuntu/vpp
-make debug 
-r
-^D^D^D
-expect VPP DBGvpp#
-
-cd lua
--- Initialization of the Lua environment for talking to VPP
-vpp = require("vpp-lapi")
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
-vpp:init({ pneum_path = pneum_path })
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
-vpp:connect("aytest")
-vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
-
-^D^D^D
-
-cd lua
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
-vpp_if_to_s1 = reply[1].sw_if_index
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
-vpp_if_to_s2 = reply[1].sw_if_index
-
-ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
-
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-
-bd_id = 42
-
-reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
-print(vpp.dump(reply))
-
-for i, v in ipairs(ifaces) do
-  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
-  print(vpp.dump(reply))
-end
-
-^D^D^D
-
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.4
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-
-
-cd lua
---- ACL testing
-
---[[ temporary comment out
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 0 })
-print(vpp.dump(reply))
-print("---")
-
-acl_index_to_delete = reply[1].acl_index
-print("Deleting " .. tostring(acl_index_to_delete))
-reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-for ri, rv in ipairs(reply) do
-  print("Reply message #" .. tostring(ri))
-  print(vpp.dump(rv))
-  for ai, av in ipairs(rv.r) do
-    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
-  end
-
-end
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
-print(vpp.dump(reply))
-print("---")
-
-
-]] -- end of comment out
-
----- Should be nothing ^^
-r = {
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
-  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
-  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
-}
-
-reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 5, r = r })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_in
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
---reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
--- print(vpp.dump(reply))
---print("---")
-
-^D^D^D
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 0
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run VPP show trace 
-expect VPP match: inacl 0 rule 1
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 2
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 3
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run VPP show trace
-expect VPP match: inacl 0 rule 4
-
-
-cd lua
-
---- TEST OUTBOUND ACL
-
-r1 = {
-  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 }
-}
-
-reply = vpp:api_call("acl_add_replace", { context = 42, acl_index = -1, count = 3, r = r1 })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-
-^D^D^D
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run VPP show trace
-expect VPP match: outacl 2 rule 0
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run VPP show trace 
-expect VPP match: inacl 0 rule 1
-
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-run VPP show trace
-expect VPP match: outacl 2 rule 1
-
-run lua print("ALL GOOD!")
-
diff --git a/vpp-api/lua/examples/lute/script.lute b/vpp-api/lua/examples/lute/script.lute
deleted file mode 100644
index c3dd90f2..00000000
--- a/vpp-api/lua/examples/lute/script.lute
+++ /dev/null
@@ -1,7 +0,0 @@
-shell s1
-expect s1 $
-run s1 echo testing123
-expect s1 $
-run s1 echo done
-quit
-
diff --git a/vpp-api/lua/examples/lute/sessions-acl.lute b/vpp-api/lua/examples/lute/sessions-acl.lute
deleted file mode 100644
index ac237ef9..00000000
--- a/vpp-api/lua/examples/lute/sessions-acl.lute
+++ /dev/null
@@ -1,308 +0,0 @@
-run lua -- collectgarbage("stop")
-
-shell vppbuild
-run vppbuild stty -echo
-run vppbuild sudo -u ubuntu -i bash -c "(cd vpp && make plugins && echo ALLGOOD)"
-expect vppbuild ALLGOOD
-
-shell s0
-shell s1
-shell s2
-
-
-cd s1
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-cd s2
-unshare -n /bin/bash
-/sbin/ifconfig -a
-^D^D^D
-
-
-cd lua
-
-function session_get_bash_pid(s)
-  if not has_session(s) then
-    return nil
-  end
-  local fname = "/tmp/lute-"..s.."-pid.txt"
-
-  session_exec(s, "echo $$ >" .. fname)
-  -- it's a dirty hack but it's quick
-  sleep(0.5)
-  local pid = io.lines(fname)()
-  print("Got pid for " .. s .. " : " .. tostring(pid))
-  return(tonumber(pid))
-end
-
-function session_connect_with(s0, s1)
-  -- local pid0 = tostring(session_get_bash_pid(s0))
-  local pid1 = tostring(session_get_bash_pid(s1))
-  local eth_options = { "rx", "tx",  "sg", "tso", "ufo", "gso", "gro", "lro", "rxvlan", "txvlan", "rxhash" }
-  local this_end = s0 .. "_" .. s1
-  local other_end = s1 .. "_" .. s0
-  session_exec(s0, "ip link add name " .. this_end .. " type veth peer name " .. other_end)
-  session_exec(s0, "ip link set dev " .. this_end .. " up promisc on")
-  for i, option in ipairs(eth_options) do
-    session_exec(s0, "/sbin/ethtool --offload " .. this_end .. " " .. option .. " off")
-    session_exec(s0, "/sbin/ethtool --offload " .. other_end .. " " .. option .. " off")
-  end
-  session_exec(s0, "ip link set dev " .. other_end .. " up promisc on netns /proc/" .. pid1 .. "/ns/net")
-  sleep(0.5)
-end
-
-^D^D^D
-run lua session_connect_with("s0", "s1")
-run lua session_connect_with("s0", "s2")
-
-cd s1
-ip -6 addr add dev s1_s0 2001:db8:1::1/64
-ip -4 addr add dev s1_s0 192.0.2.1/24
-ip link set dev s1_s0 up promisc on
-^D^D^D
-
-cd s2
-ip -6 addr add dev s2_s0 2001:db8:1::2/64
-ip -6 addr add dev s2_s0 2001:db8:1::3/64
-ip -6 addr add dev s2_s0 2001:db8:1::4/64
-ip -4 addr add dev s2_s0 192.0.2.2/24
-ip -4 addr add dev s2_s0:1 192.0.2.3/24
-ip -4 addr add dev s2_s0:2 192.0.2.4/24
-ip link set dev s2_s0 up promisc on
-^D^D^D
-
-run s1 ip addr
-run s2 ip addr
-shell VPP
-cd VPP
-cd /home/ubuntu/vpp
-make debug 
-r
-^D^D^D
-expect VPP DBGvpp#
-
-cd lua
--- Initialization of the Lua environment for talking to VPP
-vpp = require("vpp-lapi")
-root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
-vpp:init({ pneum_path = pneum_path })
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
-vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
-vpp:connect("aytest")
-vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
-
-^D^D^D
-
-cd lua
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s1", hw_addr = "AAAAAA" })
-vpp_if_to_s1 = reply[1].sw_if_index
-
-reply = vpp:api_call("af_packet_create", { host_if_name = "s0_s2", hw_addr = "AAAAAA" })
-vpp_if_to_s2 = reply[1].sw_if_index
-
-ifaces = { vpp_if_to_s1, vpp_if_to_s2 }
-
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s1, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-reply = vpp:api_call("sw_interface_set_flags", { sw_if_index = vpp_if_to_s2, admin_up_down = 1, link_up_down = 1 })
-print(vpp.dump(reply))
-
-bd_id = 42
-
-reply = vpp:api_call("bridge_domain_add_del", { bd_id = bd_id, flood = 1, uu_flood = 1, forward = 1, learn = 1, arp_term = 0, is_add = 1 })
-print(vpp.dump(reply))
-
-for i, v in ipairs(ifaces) do
-  reply = vpp:api_call("sw_interface_set_l2_bridge", { rx_sw_if_index = v, bd_id = bd_id, shg = 0, bvi = 0, enable = 1 } )
-  print(vpp.dump(reply))
-end
-
-^D^D^D
-
-run s1 ping -c 3 192.0.2.2
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.3
-expect s1 packet loss
-run s1 ping -c 3 192.0.2.4
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::2
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::3
-expect s1 packet loss
-run s1 ping6 -c 3 2001:db8:1::4
-expect s1 packet loss
-
-
-cd lua
---- ACL testing
-
---[[ temporary comment out
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_add", { context = 42, count = 0 })
-print(vpp.dump(reply))
-print("---")
-
-acl_index_to_delete = reply[1].acl_index
-print("Deleting " .. tostring(acl_index_to_delete))
-reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-for ri, rv in ipairs(reply) do
-  print("Reply message #" .. tostring(ri))
-  print(vpp.dump(rv))
-  for ai, av in ipairs(rv.r) do
-    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
-  end
-
-end
-print("---")
-
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
-print(vpp.dump(reply))
-print("---")
-
-reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 4294967295 })
-print(vpp.dump(reply))
-print("---")
-
-
-]] -- end of comment out
-
----- Should be nothing ^^
-r = {
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8:1::3"), dst_ip_prefix_len = 128 },
-  { is_permit = 1, is_ipv6 = 1, dst_ip_addr = ip46("2001:db8::"), dst_ip_prefix_len = 32  },
-  { is_permit = 1, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.2"), dst_ip_prefix_len = 32},
-  { is_permit = 0, is_ipv6 = 0, dst_ip_addr = ip46("192.0.2.3"), dst_ip_prefix_len = 32 },
-}
-
-reply = vpp:api_call("acl_add", { context = 42, count = 5, r = r })
-print(vpp.dump(reply))
-print("---")
-interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_in
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s1, is_add = 1, is_input = 1, acl_index = interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
---- TEST OUTBOUND ACL
-
-r1 = {
-  { is_permit = 1, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::2"), dst_ip_prefix_len = 128 },
-  { is_permit = 0, is_ipv6 = 1, src_ip_addr = ip46("2001:db8:1::1"), src_ip_prefix_len = 128, dst_ip_addr = ip46("2001:db8:1::4"), dst_ip_prefix_len = 128 },
-  { is_permit = 2, is_ipv6 = 0 }
-}
-
-reply = vpp:api_call("acl_add", { context = 42, count = 3, r = r1 })
-print(vpp.dump(reply))
-print("---")
-interface_acl_out = reply[1].acl_index
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 0, acl_index = interface_acl_out })
-print(vpp.dump(reply))
-print("---")
-
-r2 = {
-  { is_permit = 1, is_ipv6 = 1 },
-  { is_permit = 0, is_ipv6 = 0 }
-}
-
-reply = vpp:api_call("acl_add", { context = 42, count = 2, r = r2 })
-print(vpp.dump(reply))
-print("---")
-second_interface_acl_in = reply[1].acl_index
-
-reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = vpp_if_to_s2, is_add = 1, is_input = 1, acl_index = second_interface_acl_in })
-print(vpp.dump(reply))
-print("---")
-
-^D^D^D
-
-run VPP show classify tables
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s2 nc -v -l -p 22
-run s1 nc 192.0.2.2 22
-run s1 echo
-sleep 1
-run s1 break
-sleep 1
-run VPP show trace
-expect VPP match: outacl 2 rule 2
-run VPP show classify tables
-
-
-run VPP show classify tables
-run VPP clear trace
-run VPP trace add af-packet-input 100
-run s2 nc -v -l -p 22
-run s1 nc 192.0.2.2 22
-run s1 echo
-sleep 1
-run s1 break
-sleep 1
-run VPP show trace
-expect VPP match: outacl 2 rule 2
-run VPP show classify tables
-
-
-run lua print("ALL GOOD!")
-
diff --git a/vpp-api/lua/vpp-lapi.lua b/vpp-api/lua/vpp-lapi.lua
deleted file mode 100644
index ebfd032b..00000000
--- a/vpp-api/lua/vpp-lapi.lua
+++ /dev/null
@@ -1,989 +0,0 @@
---[[
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-]]
-
--- json decode/encode from https://gist.github.com/tylerneylon/59f4bcf316be525b30ab
--- licensed by the author tylerneylon into public domain. Thanks!
-
-local json = {}
-
--- Internal functions.
-
-local function kind_of(obj)
-  if type(obj) ~= 'table' then return type(obj) end
-  local i = 1
-  for _ in pairs(obj) do
-    if obj[i] ~= nil then i = i + 1 else return 'table' end
-  end
-  if i == 1 then return 'table' else return 'array' end
-end
-
-local function escape_str(s)
-  local in_char  = {'\\', '"', '/', '\b', '\f', '\n', '\r', '\t'}
-  local out_char = {'\\', '"', '/',  'b',  'f',  'n',  'r',  't'}
-  for i, c in ipairs(in_char) do
-    s = s:gsub(c, '\\' .. out_char[i])
-  end
-  return s
-end
-
--- Returns pos, did_find; there are two cases:
--- 1. Delimiter found: pos = pos after leading space + delim; did_find = true.
--- 2. Delimiter not found: pos = pos after leading space;     did_find = false.
--- This throws an error if err_if_missing is true and the delim is not found.
-local function skip_delim(str, pos, delim, err_if_missing)
-  pos = pos + #str:match('^%s*', pos)
-  if str:sub(pos, pos) ~= delim then
-    if err_if_missing then
-      error('Expected ' .. delim .. ' near position ' .. pos)
-    end
-    return pos, false
-  end
-  return pos + 1, true
-end
-
--- Expects the given pos to be the first character after the opening quote.
--- Returns val, pos; the returned pos is after the closing quote character.
-local function parse_str_val(str, pos, val)
-  val = val or ''
-  local early_end_error = 'End of input found while parsing string.'
-  if pos > #str then error(early_end_error) end
-  local c = str:sub(pos, pos)
-  if c == '"'  then return val, pos + 1 end
-  if c ~= '\\' then return parse_str_val(str, pos + 1, val .. c) end
-  -- We must have a \ character.
-  local esc_map = {b = '\b', f = '\f', n = '\n', r = '\r', t = '\t'}
-  local nextc = str:sub(pos + 1, pos + 1)
-  if not nextc then error(early_end_error) end
-  return parse_str_val(str, pos + 2, val .. (esc_map[nextc] or nextc))
-end
-
--- Returns val, pos; the returned pos is after the number's final character.
-local function parse_num_val(str, pos)
-  local num_str = str:match('^-?%d+%.?%d*[eE]?[+-]?%d*', pos)
-  local val = tonumber(num_str)
-  if not val then error('Error parsing number at position ' .. pos .. '.') end
-  return val, pos + #num_str
-end
-
-
--- Public values and functions.
-
-function json.stringify(obj, as_key)
-  local s = {}  -- We'll build the string as an array of strings to be concatenated.
-  local kind = kind_of(obj)  -- This is 'array' if it's an array or type(obj) otherwise.
-  if kind == 'array' then
-    if as_key then error('Can\'t encode array as key.') end
-    s[#s + 1] = '['
-    for i, val in ipairs(obj) do
-      if i > 1 then s[#s + 1] = ', ' end
-      s[#s + 1] = json.stringify(val)
-    end
-    s[#s + 1] = ']'
-  elseif kind == 'table' then
-    if as_key then error('Can\'t encode table as key.') end
-    s[#s + 1] = '{'
-    for k, v in pairs(obj) do
-      if #s > 1 then s[#s + 1] = ', ' end
-      s[#s + 1] = json.stringify(k, true)
-      s[#s + 1] = ':'
-      s[#s + 1] = json.stringify(v)
-    end
-    s[#s + 1] = '}'
-  elseif kind == 'string' then
-    return '"' .. escape_str(obj) .. '"'
-  elseif kind == 'number' then
-    if as_key then return '"' .. tostring(obj) .. '"' end
-    return tostring(obj)
-  elseif kind == 'boolean' then
-    return tostring(obj)
-  elseif kind == 'nil' then
-    return 'null'
-  else
-    error('Unjsonifiable type: ' .. kind .. '.')
-  end
-  return table.concat(s)
-end
-
-json.null = {}  -- This is a one-off table to represent the null value.
-
-function json.parse(str, pos, end_delim)
-  pos = pos or 1
-  if pos > #str then error('Reached unexpected end of input.') end
-  local pos = pos + #str:match('^%s*', pos)  -- Skip whitespace.
-  local first = str:sub(pos, pos)
-  if first == '{' then  -- Parse an object.
-    local obj, key, delim_found = {}, true, true
-    pos = pos + 1
-    while true do
-      key, pos = json.parse(str, pos, '}')
-      if key == nil then return obj, pos end
-      if not delim_found then error('Comma missing between object items.') end
-      pos = skip_delim(str, pos, ':', true)  -- true -> error if missing.
-      obj[key], pos = json.parse(str, pos)
-      pos, delim_found = skip_delim(str, pos, ',')
-    end
-  elseif first == '[' then  -- Parse an array.
-    local arr, val, delim_found = {}, true, true
-    pos = pos + 1
-    while true do
-      val, pos = json.parse(str, pos, ']')
-      if val == nil then return arr, pos end
-      if not delim_found then error('Comma missing between array items.') end
-      arr[#arr + 1] = val
-      pos, delim_found = skip_delim(str, pos, ',')
-    end
-  elseif first == '"' then  -- Parse a string.
-    return parse_str_val(str, pos + 1)
-  elseif first == '-' or first:match('%d') then  -- Parse a number.
-    return parse_num_val(str, pos)
-  elseif first == end_delim then  -- End of an object or array.
-    return nil, pos + 1
-  else  -- Parse true, false, or null.
-    local literals = {['true'] = true, ['false'] = false, ['null'] = json.null}
-    for lit_str, lit_val in pairs(literals) do
-      local lit_end = pos + #lit_str - 1
-      if str:sub(pos, lit_end) == lit_str then return lit_val, lit_end + 1 end
-    end
-    local pos_info_str = 'position ' .. pos .. ': ' .. str:sub(pos, pos + 10)
-    error('Invalid json syntax starting at ' .. pos_info_str)
-  end
-end
-
-
-local vpp = {}
-
-local ffi = require("ffi")
-
---[[
-
-The basic type definitions. A bit of weird gymnastic with
-unionization of the hton* and ntoh* functions results
-is to make handling of signed and unsigned types a bit cleaner,
-essentially building typecasting into a C union.
-
-The vl_api_opaque_message_t is a synthetic type assumed to have
-enough storage to hold the entire API message regardless of the type.
-During the operation it is casted to the specific message struct types.
-
-]]
-
-
-ffi.cdef([[
-
-typedef uint8_t u8;
-typedef int8_t i8;
-typedef uint16_t u16;
-typedef int16_t i16;
-typedef uint32_t u32;
-typedef int32_t i32;
-typedef uint64_t u64;
-typedef int64_t i64;
-typedef double f64;
-typedef float f32;
-
-#pragma pack(1)
-typedef union {
-  u16 u16;
-  i16 i16;
-} lua_ui16t;
-
-#pragma pack(1)
-typedef union {
-  u32 u32;
-  i32 i32;
-} lua_ui32t;
-
-u16 ntohs(uint16_t hostshort);
-u16 htons(uint16_t hostshort);
-u32 htonl(uint32_t along);
-u32 ntohl(uint32_t along);
-void *memset(void *s, int c, size_t n);
-void *memcpy(void *dest, void *src, size_t n);
-
-#pragma pack(1)
-typedef struct _vl_api_opaque_message {
-  u16 _vl_msg_id;
-  u8  data[65536];
-} vl_api_opaque_message_t;
-]])
-
-
--- CRC-based version stuff
-
-local crc32c_table = ffi.new('const uint32_t[256]',
-  { 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
-  0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
-  0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
-  0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
-  0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
-  0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
-  0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
-  0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
-  0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
-  0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
-  0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
-  0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
-  0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
-  0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
-  0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
-  0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
-  0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
-  0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
-  0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
-  0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
-  0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
-  0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
-  0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
-  0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
-  0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
-  0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
-  0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
-  0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
-  0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
-  0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
-  0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
-  0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
-  0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
-  0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
-  0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
-  0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
-  0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
-  0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
-  0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
-  0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
-  0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
-  0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
-  0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
-  0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
-  0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
-  0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
-  0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
-  0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
-  0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
-  0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
-  0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
-  0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
-  0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
-  0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
-  0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
-  0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
-  0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
-  0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
-  0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
-  0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
-  0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
-  0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
-  0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
-  0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 }
-);
-
-local function CRC8(crc, d)
-  return bit.bxor(bit.rshift(crc, 8), crc32c_table[bit.band(0xff, bit.bxor(crc, d))])
-end
-
-local function CRC16(crc, d)
-  crc = CRC8(crc, bit.band(d, 0xFF))
-  d = bit.rshift(d, 8)
-  crc = CRC8(crc, bit.band(d, 0xFF))
-  return crc
-end
-
-local function string_crc(str, crc)
-  for i=1,#str do
-    -- print("S", i, string.byte(str, i), string.char(string.byte(str, i)))
-    crc = CRC8(crc, string.byte(str, i))
-  end
-  return crc
-end
-
-local tokens = {
-  { ["match"] =' ', ["act"]             = { }  },
-  { ["match"] ='\n', ["act"]             = { }  },
-  { ["match"] ="manual_endian", ["act"]  = { "NODE_MANUAL_ENDIAN", "MANUAL_ENDIAN",    276 } },
-  { ["match"] ="define", ["act"]         = { "NODE_DEFINE",        "DEFINE",           267 } },
-  { ["match"] ="dont_trace", ["act"]     = { "NODE_DONT_TRACE",    "DONT_TRACE",       279 } },
-  { ["match"] ="f64", ["act"]            = { "NODE_F64",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="i16", ["act"]            = { "NODE_I16",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="i32", ["act"]            = { "NODE_I32",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="i64", ["act"]            = { "NODE_I64",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="i8", ["act"]             = { "NODE_I8",            "PRIMTYPE",         string_crc } },
-  { ["match"] ="manual_print", ["act"]   = { "NODE_MANUAL_PRINT",  "MANUAL_PRINT",     275 } },
-  { ["match"] ="noversion", ["act"]      = { "NODE_NOVERSION",     "NOVERSION",        274 } },
-  { ["match"] ="packed", ["act"]         = { "NODE_PACKED",        "TPACKED",          266 } },
-  { ["match"] ="typeonly", ["act"]       = { "NODE_TYPEONLY",      "TYPEONLY",         278 } },
-  { ["match"] ="u16", ["act"]            = { "NODE_U16",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="u32", ["act"]            = { "NODE_U32",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="u64", ["act"]            = { "NODE_U64",           "PRIMTYPE",         string_crc } },
-  { ["match"] ="u8", ["act"]             = { "NODE_U8",            "PRIMTYPE",         string_crc } },
-  { ["match"] ="union", ["act"]          = { "NODE_UNION",         "UNION",            271 } },
-  { ["match"] ="uword", ["act"]          = { "NODE_UWORD",         "PRIMTYPE",         string_crc } },
-  { ["match"] ="%(", ["act"]             = { "NODE_LPAR",          "LPAR",             259 } },
-  { ["match"] ="%)", ["act"]             = { "NODE_RPAR",          "RPAR",             258 } },
-  { ["match"] =";", ["act"]              = { "NODE_SEMI",          "SEMI",             260 } },
-  { ["match"] ="%[", ["act"]             = { "NODE_LBRACK",        "LBRACK",           261 } },
-  { ["match"] ="%]", ["act"]             = { "NODE_RBRACK",        "RBRACK",           262 } },
-  { ["match"] ="%{", ["act"]             = { "NODE_LCURLY",        "LCURLY",           268 } },
-  { ["match"] ="%}", ["act"]             = { "NODE_RCURLY",        "RCURLY",           269 } },
-  { ["match"] ='%b""', ["act"]           = { "NODE_STRING",        "STRING",           string_crc } },
-  { ["match"] ='%b@@', ["act"]           = { "NODE_HELPER",        "HELPER_STRING",    string_crc } },
-  -- TODO: \ must be consumed
-  { ["match"] ='[_a-zA-Z][_a-zA-Z0-9]*',
-                       ["act"]           = { "NODE_NAME",          "NAME",             string_crc } },
-  { ["match"] ='[0-9]+', ["act"]         = { "NODE_NUMBER",        "NUMBER",           string_crc } },
-  { ["match"] ='#[^\n]+', ["act"]            = { "NODE_PRAGMA",        "PRAGMA",           nil } },
-}
-
-
-function vpp.crc_version_string(data)
-  local input_crc = 0
-  -- Get rid of comments
-  data = data:gsub("/%*.-%*/", "")
-  data = data:gsub("//[^\n]+", "")
-  -- print(data)
-  idx = 1
-  while (true) do
-    local matched = nil
-    for k, v in ipairs(tokens) do
-      if not matched then
-        local x, y, cap = string.find(data, v["match"], idx)
-        if x == idx then
-          matched = { ["node"] = v["act"], ["x"] = x, ["y"] = y, ["cap"] = cap, ["chars"] = string.sub(data, x, y)  }
-          -- print(k, v, x, y, cap, matched.chars, matched.node[0] )
-        end
-      end
-    end
-    if matched then
-      idx = idx + (matched.y - matched.x + 1)
-      if matched.node[1] then
-        local act = matched.node[3]
-        if type(act) == "function" then
-          input_crc = act(matched.chars, input_crc)
-        elseif type(act) == "number" then
-          input_crc = CRC16(input_crc, act)
-        end
-        -- print(vpp.dump(matched))
-      end
-    else
-      -- print("NOT MATCHED!")
-      local crc = CRC16(input_crc, 0xFFFFFFFF)
-      return string.sub(string.format("%x", crc), -8)
-    end
-  end
-end
-
-
-function vpp.dump(o)
-   if type(o) == 'table' then
-      local s = '{ '
-      for k,v in pairs(o) do
-         if type(k) ~= 'number' then k = '"'..k..'"' end
-         s = s .. '['..k..'] = ' .. vpp.dump(v) .. ','
-      end
-      return s .. '} '
-   else
-      return tostring(o)
-   end
-end
-
-function vpp.hex_dump(buf)
-  local ret = {}
-  for i=1,math.ceil(#buf/16) * 16 do
-    if (i-1) % 16 == 0 then table.insert(ret, string.format('%08X  ', i-1)) end
-    table.insert(ret, ( i > #buf and '   ' or string.format('%02X ', buf:byte(i)) ))
-    if i %  8 == 0 then table.insert(ret, ' ') end
-    if i % 16 == 0 then table.insert(ret, buf:sub(i-16+1, i):gsub('%c','.')..'\n' ) end
-  end
-  return table.concat(ret)
-end
-
-
-function vpp.c_str(text_in)
-  local text = text_in -- \000 will be helpfully added by ffi.copy
-  local c_str = ffi.new("char[?]", #text+1)
-  ffi.copy(c_str, text)
-  return c_str
-end
-
-
-function vpp.init(vpp, args)
-  local pneum_api = args.pneum_api or [[
- int cough_pneum_attach(char *pneum_path, char *cough_path);
- int pneum_connect(char *name, char *chroot_prefix, void *cb);
- int pneum_disconnect(void);
- int pneum_read(char **data, int *l);
- int pneum_write(char *data, int len);
- void pneum_free(char *data);
- uint32_t pneum_get_msg_index(unsigned char * name);
-]]
-
-  vpp.pneum_path = args.pneum_path
-  ffi.cdef(pneum_api)
-  local init_res = 0
-  vpp.pneum = ffi.load(vpp.pneum_path)
-  if (init_res < 0) then
-    return nil
-  end
-
-  vpp.next_msg_num = 1
-  vpp.msg_name_to_number = {}
-  vpp.msg_name_to_fields = {}
-  vpp.msg_number_to_name = {}
-  vpp.msg_number_to_type = {}
-  vpp.msg_number_to_pointer_type = {}
-  vpp.msg_name_to_crc = {}
-  vpp.c_type_to_fields = {}
-  vpp.events = {}
-  vpp.plugin_version = {}
-  vpp.is_connected = false
-
-
-  vpp.t_lua2c = {}
-  vpp.t_c2lua = {}
-  vpp.t_lua2c["u8"] = function(c_type, src, dst_c_ptr)
-    if type(src) == "string" then
-      -- ffi.copy adds a zero byte at the end. Grrr.
-      -- ffi.copy(dst_c_ptr, src)
-      ffi.C.memcpy(dst_c_ptr, vpp.c_str(src), #src)
-      return(#src)
-    elseif type(src) == "table" then
-      for i,v in ipairs(src) do
-        ffi.cast("u8 *", dst_c_ptr)[i-1] = v
-      end
-      return(#src)
-    else
-      return 1, src -- ffi.cast("u8", src)
-    end
-  end
-  vpp.t_c2lua["u8"] = function(c_type, src_ptr, src_len)
-    if src_len then
-      return ffi.string(src_ptr, src_len)
-    else
-      return (tonumber(src_ptr))
-    end
-  end
-
-  vpp.t_lua2c["u16"] = function(c_type, src, dst_c_ptr)
-    if type(src) == "table" then
-      for i,v in ipairs(src) do
-        ffi.cast("u16 *", dst_c_ptr)[i-1] = ffi.C.htons(v)
-      end
-      return(2 * #src)
-    else
-      return 2, (ffi.C.htons(src))
-    end
-  end
-  vpp.t_c2lua["u16"] = function(c_type, src_ptr, src_len)
-    if src_len then
-      local out = {}
-      for i = 0,src_len-1 do
-        out[i+1] = tonumber(ffi.C.ntohs(src_ptr[i]))
-      end
-      return out
-    else
-      return (tonumber(ffi.C.ntohs(src_ptr)))
-    end
-  end
-
-  vpp.t_lua2c["u32"] = function(c_type, src, dst_c_ptr)
-    if type(src) == "table" then
-      for i,v in ipairs(src) do
-        ffi.cast("u32 *", dst_c_ptr)[i-1] = ffi.C.htonl(v)
-      end
-      return(4 * #src)
-    else
-      return 4, (ffi.C.htonl(src))
-    end
-  end
-  vpp.t_c2lua["u32"] = function(c_type, src_ptr, src_len)
-    if src_len then
-      local out = {}
-      for i = 0,src_len-1 do
-        out[i+1] = tonumber(ffi.C.ntohl(src_ptr[i]))
-      end
-      return out
-    else
-      return (tonumber(ffi.C.ntohl(src_ptr)))
-    end
-  end
-  vpp.t_lua2c["i32"] = function(c_type, src, dst_c_ptr)
-    if type(src) == "table" then
-      for i,v in ipairs(src) do
-        ffi.cast("i32 *", dst_c_ptr)[i-1] = ffi.C.htonl(v)
-      end
-      return(4 * #src)
-    else
-      return 4, (ffi.C.htonl(src))
-    end
-  end
-  vpp.t_c2lua["i32"] = function(c_type, src_ptr, src_len)
-    local ntohl = function(src)
-      local u32val = ffi.cast("u32", src)
-      local ntohlval = (ffi.C.ntohl(u32val))
-      local out = tonumber(ffi.cast("i32", ntohlval + 0LL))
-      return out
-    end
-    if src_len then
-      local out = {}
-      for i = 0,src_len-1 do
-        out[i+1] = tonumber(ntohl(src_ptr[i]))
-      end
-    else
-      return (tonumber(ntohl(src_ptr)))
-    end
-  end
-
-  vpp.t_lua2c["u64"] = function(c_type, src, dst_c_ptr)
-    if type(src) == "table" then
-      for i,v in ipairs(src) do
-        ffi.cast("u64 *", dst_c_ptr)[i-1] = v --- FIXME ENDIAN
-      end
-      return(8 * #src)
-    else
-      return 8, ffi.cast("u64", src) --- FIXME ENDIAN
-    end
-  end
-  vpp.t_c2lua["u64"] = function(c_type, src_ptr, src_len)
-    if src_len then
-      local out = {}
-      for i = 0,src_len-1 do
-        out[i+1] = tonumber(src_ptr[i]) -- FIXME ENDIAN
-      end
-      return out
-    else
-      return (tonumber(src_ptr)) --FIXME ENDIAN
-    end
-  end
-
-
-
-
-  vpp.t_lua2c["__MSG__"] = function(c_type, src, dst_c_ptr)
-    local dst = ffi.cast(c_type .. " *", dst_c_ptr)
-    local additional_len = 0
-    local fields_info = vpp.c_type_to_fields[c_type]
-    -- print("__MSG__ type: " .. tostring(c_type))
-    ffi.C.memset(dst_c_ptr, 0, ffi.sizeof(dst[0]))
-    -- print(vpp.dump(fields_info))
-    -- print(vpp.dump(src))
-    for k,v in pairs(src) do
-      local field = fields_info[k]
-      if not field then
-        print("ERROR: field " .. tostring(k) .. " in message " .. tostring(c_type) .. " is unknown")
-      end
-      local lua2c = vpp.t_lua2c[field.c_type]
-      -- print("__MSG__ field " .. tostring(k) .. " : " .. vpp.dump(field))
-      -- if the field is not an array type, try to coerce the argument to a number
-      if not field.array and type(v) == "string" then
-        v = tonumber(v)
-      end
-      if not lua2c then
-        print("__MSG__ " .. tostring(c_type) .. " t_lua2c: can not store field " .. field.name ..
-              " type " .. field.c_type .. " dst " .. tostring(dst[k]))
-        return 0
-      end
-      local len = 0
-      local val = nil
-      if field.array and (type(v) == "table") then
-        -- print("NTFY: field " .. tostring(k) .. " in message " .. tostring(c_type) .. " is an array")
-        for field_i, field_v in ipairs(v) do
-          -- print("NTFY: setting member#" .. tostring(field_i) .. " to value " .. vpp.dump(field_v))
-          local field_len, field_val = lua2c(field.c_type, field_v, dst[k][field_i-1])
-          len = len + field_len
-        end
-      else
-        len, val = lua2c(field.c_type, v, dst[k])
-      end
-      if not field.array then
-        dst[k] = val
-      else
-        if 0 == field.array then
-          additional_len = additional_len + len
-          -- print("Adding " .. tostring(len) .. " bytes due to field " .. tostring(field.name))
-          -- If there is a variable storing the length
-          -- and the input table does not set it, do magic
-          if field.array_size and not src[field.array_size] then
-            local size_field = fields_info[field.array_size]
-            if size_field then
-              dst[field.array_size] = vpp.t_c2lua[size_field.c_type](size_field.c_type, len)
-            end
-          end
-        end
-      end
-      -- print("Full message:\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', req_store_cache), 64)))
-    end
-    return (ffi.sizeof(dst[0])+additional_len)
-  end
-
-  vpp.t_c2lua["__MSG__"] = function(c_type, src_ptr, src_len)
-    local out = {}
-    local reply_typed_ptr = ffi.cast(c_type .. " *", src_ptr)
-    local field_desc = vpp.c_type_to_fields[c_type]
-    if src_len then
-      for i = 0,src_len-1 do
-        out[i+1] = vpp.t_c2lua[c_type](c_type, src_ptr[i])
-      end
-      return out
-    end
-
-    for k, v in pairs(field_desc) do
-      local v_c2lua = vpp.t_c2lua[v.c_type]
-      if v_c2lua then
-        local len = v.array
-        -- print(dump(v))
-        if len then
-          local len_field_name = k .. "_length"
-          local len_field = field_desc[len_field_name]
-          if (len_field) then
-            local real_len = vpp.t_c2lua[len_field.c_type](len_field.c_type, reply_typed_ptr[len_field_name])
-            out[k] =  v_c2lua(v.c_type, reply_typed_ptr[k], real_len)
-          elseif len == 0 then
-            -- check if len = 0, then must be a field which contains the size
-            len_field =  field_desc[v.array_size]
-            local real_len = vpp.t_c2lua[len_field.c_type](len_field.c_type, reply_typed_ptr[v.array_size])
-            -- print("REAL length: " .. vpp.dump(v) .. " : " .. tostring(real_len))
-            out[k] = v_c2lua(v.c_type, reply_typed_ptr[k], real_len)
-          else
-            -- alas, just stuff the entire array
-            out[k] = v_c2lua(v.c_type, reply_typed_ptr[k], len)
-          end
-        else
-          out[k] =  v_c2lua(v.c_type, reply_typed_ptr[k])
-        end
-      else
-        out[k] = ""
-      end
-      -- print(k, out[k])
-    end
-    return out
-  end
-
-  return vpp
-end
-
-function vpp.resolve_message_number(msgname)
-  local name = msgname .. "_" .. vpp.msg_name_to_crc[msgname]
-  local idx = vpp.pneum.pneum_get_msg_index(vpp.c_str(name))
-  if vpp.debug_dump then
-    print("Index for " .. tostring(name) .. " is " .. tostring(idx))
-  end
-  vpp.msg_name_to_number[msgname] = idx
-  vpp.msg_number_to_name[idx] = msgname
-  vpp.msg_number_to_type[idx] = "vl_api_" .. msgname .. "_t"
-  vpp.msg_number_to_pointer_type[idx] = vpp.msg_number_to_type[idx] .. " *"
-  ffi.cdef("\n\n enum { vl_msg_" .. msgname .. " = " .. idx .. " };\n\n")
-end
-
-function vpp.connect(vpp, client_name)
-    local name = "lua_client"
-    if client_name then
-      name = client_name
-    end
-    local ret = vpp.pneum.pneum_connect(vpp.c_str(client_name), nil, nil)
-    if tonumber(ret) == 0 then
-      vpp.is_connected = true
-    end
-    for k, v in pairs(vpp.msg_name_to_number) do
-      vpp.resolve_message_number(k)
-    end
-  end
-
-function vpp.disconnect(vpp)
-    vpp.pneum.pneum_disconnect()
-  end
-
-function vpp.json_api(vpp, path, plugin_name)
-    -- print("Consuming the VPP api from "..path)
-    local ffii = {}
-    local f = io.open(path, "r")
-    if not f then
-      print("Could not open " .. path)
-      return nil
-    end
-    local data = f:read("*all")
-    local json = json.parse(data)
-    if not (json.types or json.messages) then
-      print("Can not parse " .. path)
-      return nil
-    end
-
-    local all_types = {}
-
-    for i, v in ipairs(json.types) do
-      table.insert(all_types, { typeonly = 1, desc = v })
-    end
-    for i, v in ipairs(json.messages) do
-      table.insert(all_types, { typeonly = 0, desc = v })
-    end
-    for i, v in ipairs(all_types) do
-      local typeonly = v.typeonly
-      local name = v.desc[1]
-      local c_type = "vl_api_" .. name .. "_t"
-
-      local fields = {}
-      -- vpp.msg_name_to_fields[name] = fields
-      -- print("CTYPE " .. c_type)
-      vpp.c_type_to_fields[c_type] = fields
-      vpp.t_lua2c[c_type] = vpp.t_lua2c["__MSG__"]
-      vpp.t_c2lua[c_type] = vpp.t_c2lua["__MSG__"]
-
-      local cdef = { "\n\n#pragma pack(1)\ntypedef struct _vl_api_", name, " {\n" }
-      for ii, vv in ipairs(v.desc) do
-        if type(vv) == "table" then
-          if vv.crc then
-            vpp.msg_name_to_crc[name] = string.sub(vv.crc, 3) -- strip the leading 0x
-          else
-            local fieldtype = vv[1]
-            local fieldname = vv[2]
-            local fieldcount = vv[3]
-            local fieldcountvar = vv[4]
-            local fieldrec = { name = fieldname, c_type = fieldtype, array = fieldcount, array_size = fieldcountvar }
-            if fieldcount then
-              table.insert(cdef, "  " .. fieldtype .. " " .. fieldname .. "[" .. fieldcount .. "];\n")
-              if fieldtype == "u8" then
-                -- any array of bytes is treated as a string
-              elseif vpp.t_lua2c[fieldtype] then
-                -- print("Array of " .. fieldtype .. " is ok!")
-              else
-                print("Unknown array type: ", name,  " : " , fieldname, " : ", fieldtype, ":", fieldcount, ":", fieldcountvar)
-              end
-            else
-              table.insert(cdef, "  " .. fieldtype .. " " .. fieldname .. ";\n")
-            end
-            fields[fieldname] = fieldrec
-          end
-        end
-      end
-
-      table.insert(cdef, "} vl_api_" .. name .. "_t;")
-      table.insert(ffii, table.concat(cdef))
-
-      if typeonly == 0 then
-        -- we will want to resolve this later
-        if vpp.debug_dump then
-          print("Remember to resolve " .. name)
-        end
-        vpp.msg_name_to_number[name] = -1
-        if vpp.is_connected then
-          vpp.resolve_message_number(name)
-        end
-      end
-
-    end
-    local cdef_full = table.concat(ffii)
-    ffi.cdef(cdef_full)
-end
-
-function vpp.consume_api(vpp, path, plugin_name)
-    -- print("Consuming the VPP api from "..path)
-    local ffii = {}
-    local f = io.open(path, "r")
-    if not f then
-      print("Could not open " .. path)
-      return nil
-    end
-    local data = f:read("*all")
-    -- Remove all C comments
-    data = data:gsub("/%*.-%*/", "")
-    if vpp.is_connected and not plugin_name then
-      print(path .. ": must specify plugin name!")
-      return
-    end
-    if plugin_name then
-      vpp.plugin_version[plugin_name] = vpp.crc_version_string(data)
-      local full_plugin_name = plugin_name .. "_" .. vpp.plugin_version[plugin_name]
-      local reply = vpp:api_call("get_first_msg_id", { name = full_plugin_name } )
-      vpp.next_msg_num = tonumber(reply[1].first_msg_id)
-      print("Plugin " .. full_plugin_name .. " first message is " .. tostring(vpp.next_msg_num))
-    end
-    -- print ("data len: ", #data)
-    data = data:gsub("\n(.-)(%S+)%s*{([^}]*)}", function (preamble, name, members)
-      local _, typeonly = preamble:gsub("typeonly", "")
-      local maybe_msg_id_field = { [0] = "u16 _vl_msg_id;", "" }
-      local onedef = "\n\n#pragma pack(1)\ntypedef struct _vl_api_"..name.. " {\n" ..
-	   -- "   u16 _vl_msg_id;" ..
-           maybe_msg_id_field[typeonly] ..
-	   members:gsub("%[[a-zA-Z_]+]", "[0]") ..
-	   "} vl_api_" .. name .. "_t;"
-
-      local c_type = "vl_api_" .. name .. "_t"
-
-      local fields = {}
-      -- vpp.msg_name_to_fields[name] = fields
-      -- print("CTYPE " .. c_type)
-      vpp.c_type_to_fields[c_type] = fields
-      vpp.t_lua2c[c_type] = vpp.t_lua2c["__MSG__"]
-      vpp.t_c2lua[c_type] = vpp.t_c2lua["__MSG__"]
-      local mirec = { name = "_vl_msg_id", c_type = "u16", array = nil, array_size = nil }
-      if typeonly == 0 then
-        fields[mirec.name] = mirec
-      end
-
-      -- populate the field reflection table for the message
-      -- sets the various type information as well as the accessors for lua<->C conversion
-      members:gsub("(%S+)%s+(%S+);", function (fieldtype, fieldname)
-          local fieldcount = nil
-          local fieldcountvar = nil
-          -- data = data:gsub("%[[a-zA-Z_]+]", "[0]")
-          fieldname = fieldname:gsub("(%b[])", function(cnt)
-              fieldcount = tonumber(cnt:sub(2, -2));
-              if not fieldcount then
-                fieldcount = 0
-                fieldcountvar = cnt:sub(2, -2)
-              end
-              return ""
-            end)
-	  local fieldrec = { name = fieldname, c_type = fieldtype, array = fieldcount, array_size = fieldcountvar }
-          if fieldcount then
-            if fieldtype == "u8" then
-              -- any array of bytes is treated as a string
-            elseif vpp.t_lua2c[fieldtype] then
-              -- print("Array of " .. fieldtype .. " is ok!")
-            else
-              print("Unknown array type: ", name,  " : " , fieldname, " : ", fieldtype, ":", fieldcount, ":", fieldcountvar)
-            end
-          end
-	  fields[fieldname] = fieldrec
-	end)
-
-      -- print(dump(fields))
-
-      if typeonly == 0 then
-	local this_message_number = vpp.next_msg_num
-	vpp.next_msg_num = vpp.next_msg_num + 1
-	vpp.msg_name_to_number[name] = this_message_number
-	vpp.msg_number_to_name[this_message_number] = name
-	vpp.msg_number_to_type[this_message_number] = "vl_api_" .. name .. "_t"
-	vpp.msg_number_to_pointer_type[this_message_number] = vpp.msg_number_to_type[this_message_number] .. " *"
-	onedef = onedef .. "\n\n enum { vl_msg_" .. name .. " = " .. this_message_number .. " };\n\n"
-      end
-      table.insert(ffii, onedef);
-      return "";
-      end)
-    local cdef = table.concat(ffii)
-    -- print(cdef)
-    ffi.cdef(cdef)
-  end
-
-
-function vpp.lua2c(vpp, c_type, src, dst_c_ptr)
-  -- returns the number of bytes written to memory pointed by dst
-  local lua2c = vpp.t_lua2c[c_type]
-  if lua2c then
-    return(lua2c(c_type, src, dst_c_ptr))
-  else
-    print("vpp.lua2c: do not know how to store type " .. tostring(c_type))
-    local x = "a" .. nil
-    return 0
-  end
-end
-
-function vpp.c2lua(vpp, c_type, src_ptr, src_len)
-  -- returns the lua data structure
-  local c2lua = vpp.t_c2lua[c_type]
-  if c2lua then
-    return(c2lua(c_type, src_ptr, src_len))
-  else
-    print("vpp.c2lua: do not know how to load type " .. c_type)
-    return nil
-  end
-end
-
-local req_store_cache = ffi.new("vl_api_opaque_message_t[1]")
-
-function vpp.api_write(vpp, api_name, req_table)
-    local msg_num = vpp.msg_name_to_number[api_name]
-    if not msg_num then
-      print ("API call "..api_name.." is not known")
-      return nil
-    end
-
-    if not req_table then
-      req_table = {}
-    end
-    req_table._vl_msg_id = msg_num
-
-    local packed_len = vpp:lua2c(vpp.msg_number_to_type[msg_num], req_table, req_store_cache)
-    if vpp.debug_dump then
-      print("Write Message length: " .. tostring(packed_len) .. "\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', req_store_cache), packed_len)))
-    end
-
-    res = vpp.pneum.pneum_write(ffi.cast('void *', req_store_cache), packed_len)
-    return res
-  end
-
-local rep_store_cache = ffi.new("vl_api_opaque_message_t *[1]")
-local rep_len_cache = ffi.new("int[1]")
-
-function vpp.api_read(vpp)
-    local rep_type = "vl_api_opaque_message_t"
-    local rep = rep_store_cache
-    local replen = rep_len_cache
-    res = vpp.pneum.pneum_read(ffi.cast("void *", rep), replen)
-    if vpp.debug_dump then
-      print("Read Message length: " .. tostring(replen[0]) .. "\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', rep[0]), replen[0])))
-    end
-
-    local reply_msg_num = ffi.C.ntohs(rep[0]._vl_msg_id)
-    local reply_msg_name = vpp.msg_number_to_name[reply_msg_num]
-
-    local reply_typed_ptr = ffi.cast(vpp.msg_number_to_pointer_type[reply_msg_num], rep[0])
-    local out = vpp:c2lua(vpp.msg_number_to_type[reply_msg_num], rep[0], nil, replen[0])
-    if type(out) == "table" then
-      out["luaapi_message_name"] = reply_msg_name
-    end
-
-    vpp.pneum.pneum_free(ffi.cast('void *',rep[0]))
-
-    return reply_msg_name, out
-  end
-
-function vpp.api_call(vpp, api_name, req_table, options_in)
-    local msg_num = vpp.msg_name_to_number[api_name]
-    local end_message_name = api_name .."_reply"
-    local replies = {}
-    local cstruct = ""
-    local options = options_in or {}
-    if msg_num then
-      if vpp.debug_dump then
-        print("Message #" .. tostring(msg_num) .. " for name " .. tostring(api_name))
-      end
-      vpp:api_write(api_name, req_table)
-      if not vpp.msg_name_to_number[end_message_name] or options.force_ping then
-        end_message_name = "control_ping_reply"
-        vpp:api_write("control_ping")
-      end
-      repeat
-        reply_message_name, reply = vpp:api_read()
-        if reply and not reply.context then
-          -- there may be async events inbetween
-          table.insert(vpp.events, reply)
-        else
-          if reply_message_name ~= "control_ping_reply" then
-            -- do not insert the control ping encapsulation
-            table.insert(replies, reply)
-          end
-        end
-        -- print(reply)
-      until reply_message_name == end_message_name
-    else
-      print(api_name .. " is an unknown API call")
-      return nil
-    end
-    return replies
-  end
-
-return vpp
-- 
cgit 1.2.3-korg


From 0be5ec304d2d4cfc6faecbb519ef165d9ab5d34e Mon Sep 17 00:00:00 2001
From: Damjan Marion 
Date: Wed, 28 Dec 2016 17:51:56 +0100
Subject: Do not require external vppapigen when not cross-compiling

Change-Id: I80b8348ed4efd53d292c37a1ff69c13ee4741986
Signed-off-by: Damjan Marion 
---
 build-data/platforms.mk |  2 +-
 build-root/rpm/vpp.spec |  1 -
 src/Makefile.am         |  1 -
 src/configure.ac        | 16 ++++++++++++++++
 src/plugins/Makefile.am |  1 -
 src/suffix-rules.mk     | 12 ++++++------
 src/vppapigen.am        |  9 ++++++---
 7 files changed, 29 insertions(+), 13 deletions(-)

(limited to 'src/plugins/Makefile.am')

diff --git a/build-data/platforms.mk b/build-data/platforms.mk
index 6f21b6dc..e192155b 100644
--- a/build-data/platforms.mk
+++ b/build-data/platforms.mk
@@ -76,7 +76,7 @@ install-deb: $(patsubst %,%-find-source,$(ROOT_PACKAGES))
 	   >> deb/debian/vpp.install ;					\
 									\
 	: dev package needs a couple of additions ;			\
-	echo ../build-tool-native/tools/vppapigen /usr/bin		\
+	echo ../$(INSTALL_PREFIX)$(ARCH)/vpp/bin/vppapigen /usr/bin	\
 	   >> deb/debian/vpp-dev.install ;				\
 	echo ../../src/vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin	\
 	   >> deb/debian/vpp-dev.install ;				\
diff --git a/build-root/rpm/vpp.spec b/build-root/rpm/vpp.spec
index 149ac51c..c3c0d927 100644
--- a/build-root/rpm/vpp.spec
+++ b/build-root/rpm/vpp.spec
@@ -113,7 +113,6 @@ groupadd -f -r vpp
 mkdir -p -m755 %{buildroot}%{_bindir}
 mkdir -p -m755 %{buildroot}%{_unitdir}
 install -p -m 755 %{_mu_build_dir}/%{_vpp_install_dir}/*/bin/* %{buildroot}%{_bindir}
-install -p -m 755 %{_mu_build_dir}/%{_vpp_build_dir}/tools/vppapigen %{buildroot}%{_bindir}
 
 # api
 mkdir -p -m755 %{buildroot}/usr/share/vpp/api
diff --git a/src/Makefile.am b/src/Makefile.am
index 0fc437a4..bba90eae 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -102,7 +102,6 @@ api_DATA = \
 	$(patsubst %.api,%.api.json,$(API_FILES))
 
 BUILT_SOURCES += \
-	$(patsubst %.api,%.api.json,$(API_FILES)) \
 	$(patsubst %.api,%.api.h,$(API_FILES))
 
 endif # if ENABLE_VLIB
diff --git a/src/configure.ac b/src/configure.ac
index 4ed55705..eb380d8b 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -11,6 +11,8 @@ AM_PROG_AS
 AM_PROG_LIBTOOL
 AC_PROG_YACC
 
+AM_CONDITIONAL([CROSSCOMPILE], [test "$cross_compiling" == "yes"])
+
 ###############################################################################
 # Macros
 ###############################################################################
@@ -172,6 +174,20 @@ AM_COND_IF([ENABLE_G2],
   PKG_CHECK_MODULES(g2, gtk+-2.0)
 ])
 
+# If cross-compiling, we need external vppapigen and we cannot continue without it
+# For native builds, we just set dependency on vpppaigen binary in top_builddir
+AM_COND_IF([CROSSCOMPILE],
+[
+  AC_PATH_PROG([VPPAPIGEN], [vppapigen], [no])
+  if test "$VPPAPIGEN" = "no"; then
+    AC_MSG_ERROR([Externaly built vppapigen is needed when cross-compiling...])
+  fi
+],[
+  VPPAPIGEN=\$\(top_builddir\)/vppapigen
+])
+AC_SUBST([VPPAPIGEN])
+
+
 ###############################################################################
 # JAVA
 ###############################################################################
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 987310b7..3e9ab91c 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -73,6 +73,5 @@ api_DATA = \
 	$(patsubst %.api,%.api.json,$(API_FILES))
 
 BUILT_SOURCES += \
-	$(patsubst %.api,%.api.json,$(API_FILES)) \
 	$(patsubst %.api,%.api.h,$(API_FILES))
 
diff --git a/src/suffix-rules.mk b/src/suffix-rules.mk
index f97b2276..9115c55b 100644
--- a/src/suffix-rules.mk
+++ b/src/suffix-rules.mk
@@ -14,14 +14,14 @@
 # Shared suffix rules
 # Please do not set "SUFFIXES = .api.h .api" here
 
-%.api.h: %.api
+%.api.h: %.api @VPPAPIGEN@
 	@echo "  APIGEN  " $@ ;					\
 	mkdir -p `dirname $@` ;					\
-	$(CC) $(CPPFLAGS) -E -P -C -x c $^			\
-	| vppapigen --input - --output $@ --show-name $@ > /dev/null
+	$(CC) $(CPPFLAGS) -E -P -C -x c $<			\
+	| @VPPAPIGEN@ --input - --output $@ --show-name $@ > /dev/null
 
-%.api.json: %.api
+%.api.json: %.api @VPPAPIGEN@
 	@echo "  JSON API" $@ ;					\
 	mkdir -p `dirname $@` ;					\
-	$(CC) $(CPPFLAGS) -E -P -C -x c $^			\
-	| vppapigen --input - --json $@ > /dev/null
+	$(CC) $(CPPFLAGS) -E -P -C -x c $<			\
+	| @VPPAPIGEN@ --input - --json $@ > /dev/null
diff --git a/src/vppapigen.am b/src/vppapigen.am
index edde339d..3207c83a 100644
--- a/src/vppapigen.am
+++ b/src/vppapigen.am
@@ -13,14 +13,17 @@
 
 bin_PROGRAMS += vppapigen
 
-BUILT_SOURCES += tools/vppapigen/gram.h
+# We cannot rely on BUILT_SOURCES here as other built sources are relying
+# on vppapigen, so make can start compiling lex.c before gram.h is created.
+# This way we introduce new dependency by running C preprocessor.
 
-tools/vppapigen/gram.h: tools/vppapigen/gram.y
+tools/vppapigen/lex_e.c: tools/vppapigen/lex.c tools/vppapigen/gram.y
 	@$(YACC) -d @srcdir@/tools/vppapigen/gram.y
 	@mv y.tab.h tools/vppapigen/gram.h
 	@rm y.tab.c
+	@$(CC) -I. -E -o $@ $<
 
-vppapigen_SOURCES = tools/vppapigen/gram.y tools/vppapigen/lex.c tools/vppapigen/node.c
+vppapigen_SOURCES = tools/vppapigen/gram.y tools/vppapigen/lex_e.c tools/vppapigen/node.c
 vppapigen_LDADD = libvppinfra.la
 vppapigen_LDFLAGS = -static
 
-- 
cgit 1.2.3-korg


From 724f64ccf691e170973bdcdc6c09bfdc7e2ab5e4 Mon Sep 17 00:00:00 2001
From: Damjan Marion 
Date: Wed, 11 Jan 2017 11:11:00 +0100
Subject: Makefile.am cleanup

 - remove unused stuff
 - add --quiet flag to libtool
 - avoid building some tests programs when tests are not enabled

Change-Id: Ie34aeec1a598ad811256a00354f66cfddae9d0b9
Signed-off-by: Damjan Marion 
---
 build-root/emacs-lisp/README           |  2 +-
 build-root/scripts/make-plugin-toolkit | 40 --------------------------
 src/Makefile.am                        |  1 +
 src/plugins/Makefile.am                |  1 +
 src/vnet.am                            |  9 ++----
 src/vnet/plugin/p1.c                   | 52 ----------------------------------
 src/vpp-api/java/Makefile.am           |  1 +
 src/vpp-api/python/Makefile.am         |  5 ++--
 src/vpp.am                             | 36 ++---------------------
 src/vppinfra.am                        |  8 +-----
 10 files changed, 12 insertions(+), 143 deletions(-)
 delete mode 100755 build-root/scripts/make-plugin-toolkit
 delete mode 100644 src/vnet/plugin/p1.c

(limited to 'src/plugins/Makefile.am')

diff --git a/build-root/emacs-lisp/README b/build-root/emacs-lisp/README
index 483e1c39..1f09a3fa 100644
--- a/build-root/emacs-lisp/README
+++ b/build-root/emacs-lisp/README
@@ -66,7 +66,7 @@ Or, generate each file individually:
 
    $ mkdir build
    $ cd build
-   $ ../configure --with-plugin-toolkit
+   $ ../configure
    $ make
    $ sudo make install
 
diff --git a/build-root/scripts/make-plugin-toolkit b/build-root/scripts/make-plugin-toolkit
deleted file mode 100755
index e1d6fcfb..00000000
--- a/build-root/scripts/make-plugin-toolkit
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-build_tarball() {
-    for dir in vppinfra dpdk svm vlib-api vlib vnet vpp vpp-api-test
-    do
-        tar -C install-$1/$dir/include -cf - . | tar -C $tmp_dir/include -xf -
-    done
-    tar -C ../sample-plugin -cf - . \
-       | tar -C $tmp_dir/src/sample-plugin -xf -
-    cp tools/bin/vppapigen $tmp_dir/tools/bin
-    echo Created by `id -u -n` on `hostname` at `date` >		\
-		$tmp_dir/toolkit-version-stamp
-    cp scripts/vpp-plugin-toolkit-readme $tmp_dir/README
-    tar -C $tmp_dir -zcf $PWD/vpp-plugin-toolkit-$1.tar.gz .
-}
-
-if [ `basename $PWD` != "build-root" ] ; then
-   echo Please run this script from build-root
-   exit 1
-fi   
-
-echo Pull additional tookit repos
-make PLATFORM=vpp sample-plugin-find-source
-
-make PLATFORM=vpp TAG=vpp wipe-all
-echo Build vpp forwarder production package
-make PLATFORM=vpp TAG=vpp strip_sumbols=yes install-packages
-
-tmp_dir="`mktemp -d /tmp/plugin-XXXXXX`"
-trap "rm -rf $tmp_dir" err 
-
-echo Create vpp forwarder production plugin toolkit tarball
-mkdir -p $tmp_dir/tools/bin $tmp_dir/include $tmp_dir/lib64 \
-         $tmp_dir/src/sample-plugin
-build_tarball vpp-native
-rm -rf $tmp_dir
-
-exit 0
diff --git a/src/Makefile.am b/src/Makefile.am
index bba90eae..18a41a15 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -17,6 +17,7 @@
 
 AUTOMAKE_OPTIONS = foreign subdir-objects
 ACLOCAL_AMFLAGS = -I m4
+AM_LIBTOOLFLAGS = --quiet
 
 AM_CFLAGS = -Wall
 
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 3e9ab91c..f0c455a5 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -16,6 +16,7 @@ AUTOMAKE_OPTIONS = foreign subdir-objects
 
 AM_CFLAGS = -Wall -I${top_srcdir} -I${top_builddir}
 AM_LDFLAGS = -module -shared -avoid-version
+AM_LIBTOOLFLAGS = --quiet
 SUFFIXES = .api.h .api .api.json
 API_FILES =
 BUILT_SOURCES =
diff --git a/src/vnet.am b/src/vnet.am
index bca56227..665a16ea 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -11,8 +11,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+lib_LTLIBRARIES += libvnet.la
+
 libvnet_la_SOURCES =
-libvnetplugin_la_SOURCES =
 
 libvnet_la_LIBADD = libvppinfra.la libsvm.la
 libvnet_la_DEPENDENCIES = libvppinfra.la libvlib.la libsvmdb.la libsvm.la libvlibapi.la libvlibmemory.la libvlibmemoryclient.la
@@ -966,15 +967,9 @@ nobase_include_HEADERS +=			\
 # Plugin client library
 ########################################
 
-libvnetplugin_la_SOURCES +=                    \
-  vnet/plugin/p1.c
-
 nobase_include_HEADERS +=                      \
   vnet/plugin/plugin.h
 
-libvnetdir = ${libdir}
-libvnet_LTLIBRARIES = libvnet.la libvnetplugin.la
-
 pcap2pg_SOURCES = 				\
   vnet/unix/pcap2pg.c				\
   vnet/unix/pcap.h
diff --git a/src/vnet/plugin/p1.c b/src/vnet/plugin/p1.c
deleted file mode 100644
index 3102ecce..00000000
--- a/src/vnet/plugin/p1.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/* 
- * This file and in fact the entire directory shouldn't even exist.
- *
- * Unfortunately, various things malfunction when we try to go there. 
- * Plugin DLL's end up with their own copies of critical
- * data structures. No one of these problems would be tough to fix, 
- * but there are quite a number of them.
- */
-
-/* 
- * Make certain that plugin .dll's which reference the following functions
- * can find them...
- */
-
-#if DPDK > 0
-#define foreach_dpdk_plugin_reference		\
-_(rte_calloc)                                   \
-_(rte_free)                                     \
-_(rte_malloc)                                   \
-_(rte_zmalloc)                                  \
-_(rte_malloc_virt2phy)                          \
-_(rte_eal_get_configuration)
-#else
-#define foreach_dpdk_plugin_reference
-#endif
-
-#define _(a) void a (void);
-foreach_dpdk_plugin_reference
-#undef _
-
-void *vnet_library_plugin_references[] =
-  {
-#define _(a) &a,
-    foreach_dpdk_plugin_reference
-#undef _
-  };
-
-void vnet_library_plugin_reference(void) { }
diff --git a/src/vpp-api/java/Makefile.am b/src/vpp-api/java/Makefile.am
index bfb5665a..00ee5946 100644
--- a/src/vpp-api/java/Makefile.am
+++ b/src/vpp-api/java/Makefile.am
@@ -13,6 +13,7 @@
 
 AUTOMAKE_OPTIONS = foreign
 ACLOCAL_AMFLAGS = -I m4
+AM_LIBTOOLFLAGS = --quiet
 
 AM_CFLAGS = -Wall -I${top_srcdir} -I${top_builddir} \
   -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux \
diff --git a/src/vpp-api/python/Makefile.am b/src/vpp-api/python/Makefile.am
index a5dabc3d..16c47627 100644
--- a/src/vpp-api/python/Makefile.am
+++ b/src/vpp-api/python/Makefile.am
@@ -13,6 +13,7 @@
 
 AUTOMAKE_OPTIONS = foreign
 ACLOCAL_AMFLAGS = -I m4
+AM_LIBTOOLFLAGS = --quiet
 AM_CFLAGS = -Wall -I${top_srcdir} -I${top_builddir}
 
 BUILT_SOURCES =
@@ -48,6 +49,7 @@ install-exec-local:
 #
 # Test client
 #
+if ENABLE_TESTS
 noinst_PROGRAMS += test_pneum
 test_pneum_SOURCES = pneum/pneum.c pneum/test_pneum.c
 test_pneum_LDADD = \
@@ -56,5 +58,4 @@ test_pneum_LDADD = \
   $(top_builddir)/libvlibapi.la \
   $(top_builddir)/libsvm.la \
   -lpthread -lm -lrt
-
-
+endif
diff --git a/src/vpp.am b/src/vpp.am
index 3bb2c1be..be55c400 100644
--- a/src/vpp.am
+++ b/src/vpp.am
@@ -68,39 +68,6 @@ VPP_VERSION = $(shell $(srcdir)/scripts/version)
 	  echo "#define VPP_BUILD_VER \"$(VPP_VERSION)\"" >> $$f			;\
 	fi
 
-
-# Generate a plugin configuration script. Misconfiguring a
-# plugin can cause very subtle problems.
-
-bin_SCRIPTS = vpp_plugin_configure
-
-BUILT_SOURCES += vpp_plugin_configure
-
-.PHONY: vpp_plugin_configure
-
-if WITH_DPDK
-PLUGIN_DPDK_ARG="--with-dpdk"
-else
-PLUGIN_DPDK_ARG=""
-endif
-
-vpp_plugin_configure:
-	@echo "PLUGIN CFG" $@
-	@echo "#!/bin/bash" > $@
-	@echo " " >> $@
-	@echo "set +eu" >> $@
-	@echo " " >> $@
-	@echo "if [ -f ./configure ] ; then" >> $@
-	@echo "    CFLAGS='$(CFLAGS) $(AM_CFLAGS) -I/usr/include/vpp-dpdk' ./configure --with-plugin-toolkit $(PLUGIN_DPDK_ARG)" >> $@
-	@echo "else" >> $@
-	@echo "    if [ -f ../configure ] ; then" >> $@
-	@echo "        CFLAGS='$(CFLAGS) $(AM_CFLAGS) -I/usr/include/vpp-dpdk' ../configure --with-plugin-toolkit $(PLUGIN_DPDK_ARG)" >> $@
-	@echo "    else" >> $@
-	@echo "        echo Couldnt find ./configure or ../configure " >> $@
-	@echo "        exit 1" >> $@
-	@echo "    fi" >> $@
-	@echo "fi" >> $@
-
 bin_vpp_LDADD = \
   libvlibapi.la \
   libvlibmemory.la \
@@ -108,12 +75,12 @@ bin_vpp_LDADD = \
   libvnet.la \
   libsvm.la \
   libsvmdb.la \
-  libvnetplugin.la \
   -lrt
 
 bin_vpp_LDFLAGS = $(DPDK_LD_FLAGS)
 bin_vpp_LDADD += libvppinfra.la -lm -lpthread -ldl $(DPDK_LD_ADD)
 
+if ENABLE_TESTS
 noinst_PROGRAMS += bin/test_client
 
 bin_test_client_SOURCES = \
@@ -138,6 +105,7 @@ bin_test_ha_LDADD = \
   libsvm.la \
   libvppinfra.la \
   -lpthread -lm -lrt
+endif
 
 noinst_PROGRAMS += bin/summary_stats_client
 
diff --git a/src/vppinfra.am b/src/vppinfra.am
index 836c7213..19485d2d 100644
--- a/src/vppinfra.am
+++ b/src/vppinfra.am
@@ -11,13 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-libvppinfradir = ${libdir}
-
-if WITH_UNIX
-  libvppinfra_LTLIBRARIES = libvppinfra.la
-endif
-
-lib_LIBRARIES =
+lib_LTLIBRARIES += libvppinfra.la
 
 TESTS =
 
-- 
cgit 1.2.3-korg


From 006eb478bb7a14ba7ba4199fecce29ebc495fb9c Mon Sep 17 00:00:00 2001
From: Burt Silverman 
Date: Fri, 27 Jan 2017 15:29:54 -0500
Subject: Add files to CLEANFILES for robust make clean.

At the least, $(BUILT_SOURCES) should be added to CLEANFILES. Also
beneficial is $(api_DATA), and in the case of Java, *.files and *.h.
Also there is a vpp/app/version.h, and some grammar and lex files in
vppapigen.

Change-Id: Ic6d3f2d40ce65e1d9a8b88217fa1f36de393ebb4
Signed-off-by: Burt Silverman 
---
 src/Makefile.am                        | 3 +++
 src/examples/sample-plugin/Makefile.am | 2 ++
 src/plugins/Makefile.am                | 1 +
 src/vpp-api/java/Makefile.am           | 2 +-
 src/vpp.am                             | 2 ++
 src/vppapigen.am                       | 2 ++
 6 files changed, 11 insertions(+), 1 deletion(-)

(limited to 'src/plugins/Makefile.am')

diff --git a/src/Makefile.am b/src/Makefile.am
index 239afeac..7da86fcb 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -28,6 +28,7 @@ noinst_HEADERS =
 dist_bin_SCRIPTS =
 lib_LTLIBRARIES =
 BUILT_SOURCES =
+CLEANFILES =
 install-data-local:
 	@echo "Building vppctl command list..."
 	@DIR_SEARCH="$(srcdir)" ; \
@@ -115,3 +116,5 @@ BUILT_SOURCES += \
 
 endif # if ENABLE_VLIB
 endif # if ENABLE_SVM
+
+CLEANFILES += $(BUILT_SOURCES) $(api_DATA)
diff --git a/src/examples/sample-plugin/Makefile.am b/src/examples/sample-plugin/Makefile.am
index 47642055..a105afdd 100644
--- a/src/examples/sample-plugin/Makefile.am
+++ b/src/examples/sample-plugin/Makefile.am
@@ -54,3 +54,5 @@ sample_test_plugin_la_SOURCES = sample/sample_test.c sample/sample_plugin.api.h
 install-data-hook:
 	@(cd $(vpppluginsdir) && $(RM) $(vppplugins_LTLIBRARIES))
 	@(cd $(vppapitestpluginsdir) && $(RM) $(vppapitestplugins_LTLIBRARIES))
+
+CLEANFILES = $(BUILT_SOURCES)
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index f0c455a5..06b575d1 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -76,3 +76,4 @@ api_DATA = \
 BUILT_SOURCES += \
 	$(patsubst %.api,%.api.h,$(API_FILES))
 
+CLEANFILES = $(BUILT_SOURCES) $(api_DATA)
diff --git a/src/vpp-api/java/Makefile.am b/src/vpp-api/java/Makefile.am
index 3696e5bf..6eb50084 100644
--- a/src/vpp-api/java/Makefile.am
+++ b/src/vpp-api/java/Makefile.am
@@ -218,7 +218,7 @@ all-local: $(JAR_FILES)
 #
 # Cleanup
 #
-CLEANFILES = jvpp-registry.ok $(JAR_FILES) $(BUILT_SOURCES)
+CLEANFILES = jvpp-registry.ok $(JAR_FILES) $(BUILT_SOURCES) *.files */*.h
 
 clean-local:
 	rm -rf $(CLEANDIRS)
diff --git a/src/vpp.am b/src/vpp.am
index 0b605ec5..37466c60 100644
--- a/src/vpp.am
+++ b/src/vpp.am
@@ -132,4 +132,6 @@ bin_vpp_get_metrics_LDADD = \
   libvppinfra.la \
   -lpthread -lm -lrt
 
+CLEANFILES += vpp/app/version.h
+
 # vi:syntax=automake
diff --git a/src/vppapigen.am b/src/vppapigen.am
index 3207c83a..5c25e1ec 100644
--- a/src/vppapigen.am
+++ b/src/vppapigen.am
@@ -27,4 +27,6 @@ vppapigen_SOURCES = tools/vppapigen/gram.y tools/vppapigen/lex_e.c tools/vppapig
 vppapigen_LDADD = libvppinfra.la
 vppapigen_LDFLAGS = -static
 
+CLEANFILES += tools/vppapigen/gram.c tools/vppapigen/gram.h
+CLEANFILES += tools/vppapigen/lex_e.c
 # vi:syntax=automake
-- 
cgit 1.2.3-korg


From c3a814be9dc769be942ff8029c7b6eccd4b3af05 Mon Sep 17 00:00:00 2001
From: Damjan Marion 
Date: Tue, 28 Feb 2017 19:22:22 +0100
Subject: dpdk: be a plugin

Change-Id: I238258cdeb77035adc5e88903d824593d0a1da90
Signed-off-by: Damjan Marion 
---
 src/Makefile.am                                    |   21 -
 src/plugins/Makefile.am                            |    5 +
 src/plugins/dpdk.am                                |   50 +
 src/plugins/dpdk/api/dpdk.api                      |  103 +
 src/plugins/dpdk/api/dpdk_all_api_h.h              |   19 +
 src/plugins/dpdk/api/dpdk_msg_enum.h               |   31 +
 src/plugins/dpdk/api/dpdk_test.c                   |  397 ++++
 src/plugins/dpdk/buffer.c                          |  588 ++++++
 src/plugins/dpdk/device/cli.c                      | 2079 ++++++++++++++++++++
 src/plugins/dpdk/device/device.c                   |  852 ++++++++
 src/plugins/dpdk/device/dpdk.h                     |  490 +++++
 src/plugins/dpdk/device/dpdk_priv.h                |  135 ++
 src/plugins/dpdk/device/format.c                   |  754 +++++++
 src/plugins/dpdk/device/node.c                     |  674 +++++++
 src/plugins/dpdk/dir.dox                           |   27 +
 src/plugins/dpdk/hqos/hqos.c                       |  775 ++++++++
 src/plugins/dpdk/hqos/qos_doc.md                   |  411 ++++
 src/plugins/dpdk/init.c                            | 2074 +++++++++++++++++++
 src/plugins/dpdk/ipsec/cli.c                       |  154 ++
 src/plugins/dpdk/ipsec/crypto_node.c               |  215 ++
 src/plugins/dpdk/ipsec/dir.dox                     |   18 +
 src/plugins/dpdk/ipsec/dpdk_crypto_ipsec_doc.md    |   86 +
 src/plugins/dpdk/ipsec/esp.h                       |  249 +++
 src/plugins/dpdk/ipsec/esp_decrypt.c               |  594 ++++++
 src/plugins/dpdk/ipsec/esp_encrypt.c               |  609 ++++++
 src/plugins/dpdk/ipsec/ipsec.c                     |  430 ++++
 src/plugins/dpdk/ipsec/ipsec.h                     |  227 +++
 src/plugins/dpdk/main.c                            |   95 +
 src/plugins/dpdk/thread.c                          |   85 +
 src/vat/api_format.c                               |  320 ---
 src/vnet.am                                        |   41 +-
 src/vnet/devices/dpdk/buffer.c                     |  588 ------
 src/vnet/devices/dpdk/cli.c                        | 2079 --------------------
 src/vnet/devices/dpdk/device.c                     |  852 --------
 src/vnet/devices/dpdk/dir.dox                      |   27 -
 src/vnet/devices/dpdk/dpdk.api                     |  103 -
 src/vnet/devices/dpdk/dpdk.h                       |  487 -----
 src/vnet/devices/dpdk/dpdk_api.c                   |  246 ---
 src/vnet/devices/dpdk/dpdk_priv.h                  |  135 --
 src/vnet/devices/dpdk/format.c                     |  754 -------
 src/vnet/devices/dpdk/hqos.c                       |  775 --------
 src/vnet/devices/dpdk/init.c                       | 1801 -----------------
 src/vnet/devices/dpdk/ipsec/cli.c                  |  154 --
 src/vnet/devices/dpdk/ipsec/crypto_node.c          |  215 --
 src/vnet/devices/dpdk/ipsec/dir.dox                |   18 -
 .../devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md    |   86 -
 src/vnet/devices/dpdk/ipsec/esp.h                  |  249 ---
 src/vnet/devices/dpdk/ipsec/esp_decrypt.c          |  594 ------
 src/vnet/devices/dpdk/ipsec/esp_encrypt.c          |  609 ------
 src/vnet/devices/dpdk/ipsec/ipsec.c                |  430 ----
 src/vnet/devices/dpdk/ipsec/ipsec.h                |  227 ---
 src/vnet/devices/dpdk/main.c                       |   85 -
 src/vnet/devices/dpdk/node.c                       |  674 -------
 src/vnet/devices/dpdk/qos_doc.md                   |  411 ----
 src/vnet/devices/dpdk/thread.c                     |   85 -
 src/vnet/devices/virtio/vhost-user.h               |   11 -
 src/vnet/ipsec/ipsec_api.c                         |    4 -
 src/vnet/pg/input.c                                |   11 +-
 src/vnet/pg/stream.c                               |    5 +-
 src/vnet/replication.c                             |    6 +-
 src/vnet/vnet_all_api_h.h                          |    3 -
 src/vpp/api/custom_dump.c                          |   64 -
 src/vpp/api/gmon.c                                 |    3 +-
 src/vpp/api/vpe.api                                |    1 -
 src/vpp/app/l2t.c                                  |  562 ------
 src/vpp/app/l2t_l2.c                               |  267 ---
 66 files changed, 12241 insertions(+), 12988 deletions(-)
 create mode 100644 src/plugins/dpdk.am
 create mode 100644 src/plugins/dpdk/api/dpdk.api
 create mode 100644 src/plugins/dpdk/api/dpdk_all_api_h.h
 create mode 100644 src/plugins/dpdk/api/dpdk_msg_enum.h
 create mode 100644 src/plugins/dpdk/api/dpdk_test.c
 create mode 100644 src/plugins/dpdk/buffer.c
 create mode 100644 src/plugins/dpdk/device/cli.c
 create mode 100644 src/plugins/dpdk/device/device.c
 create mode 100644 src/plugins/dpdk/device/dpdk.h
 create mode 100644 src/plugins/dpdk/device/dpdk_priv.h
 create mode 100644 src/plugins/dpdk/device/format.c
 create mode 100644 src/plugins/dpdk/device/node.c
 create mode 100644 src/plugins/dpdk/dir.dox
 create mode 100644 src/plugins/dpdk/hqos/hqos.c
 create mode 100644 src/plugins/dpdk/hqos/qos_doc.md
 create mode 100755 src/plugins/dpdk/init.c
 create mode 100644 src/plugins/dpdk/ipsec/cli.c
 create mode 100644 src/plugins/dpdk/ipsec/crypto_node.c
 create mode 100644 src/plugins/dpdk/ipsec/dir.dox
 create mode 100644 src/plugins/dpdk/ipsec/dpdk_crypto_ipsec_doc.md
 create mode 100644 src/plugins/dpdk/ipsec/esp.h
 create mode 100644 src/plugins/dpdk/ipsec/esp_decrypt.c
 create mode 100644 src/plugins/dpdk/ipsec/esp_encrypt.c
 create mode 100644 src/plugins/dpdk/ipsec/ipsec.c
 create mode 100644 src/plugins/dpdk/ipsec/ipsec.h
 create mode 100644 src/plugins/dpdk/main.c
 create mode 100644 src/plugins/dpdk/thread.c
 delete mode 100644 src/vnet/devices/dpdk/buffer.c
 delete mode 100644 src/vnet/devices/dpdk/cli.c
 delete mode 100644 src/vnet/devices/dpdk/device.c
 delete mode 100644 src/vnet/devices/dpdk/dir.dox
 delete mode 100644 src/vnet/devices/dpdk/dpdk.api
 delete mode 100644 src/vnet/devices/dpdk/dpdk.h
 delete mode 100644 src/vnet/devices/dpdk/dpdk_api.c
 delete mode 100644 src/vnet/devices/dpdk/dpdk_priv.h
 delete mode 100644 src/vnet/devices/dpdk/format.c
 delete mode 100644 src/vnet/devices/dpdk/hqos.c
 delete mode 100755 src/vnet/devices/dpdk/init.c
 delete mode 100644 src/vnet/devices/dpdk/ipsec/cli.c
 delete mode 100644 src/vnet/devices/dpdk/ipsec/crypto_node.c
 delete mode 100644 src/vnet/devices/dpdk/ipsec/dir.dox
 delete mode 100644 src/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md
 delete mode 100644 src/vnet/devices/dpdk/ipsec/esp.h
 delete mode 100644 src/vnet/devices/dpdk/ipsec/esp_decrypt.c
 delete mode 100644 src/vnet/devices/dpdk/ipsec/esp_encrypt.c
 delete mode 100644 src/vnet/devices/dpdk/ipsec/ipsec.c
 delete mode 100644 src/vnet/devices/dpdk/ipsec/ipsec.h
 delete mode 100644 src/vnet/devices/dpdk/main.c
 delete mode 100644 src/vnet/devices/dpdk/node.c
 delete mode 100644 src/vnet/devices/dpdk/qos_doc.md
 delete mode 100644 src/vnet/devices/dpdk/thread.c
 delete mode 100644 src/vpp/app/l2t.c
 delete mode 100644 src/vpp/app/l2t_l2.c

(limited to 'src/plugins/Makefile.am')

diff --git a/src/Makefile.am b/src/Makefile.am
index 641707ed..5daaa48e 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -38,27 +38,6 @@ install-data-local:
 	GREP_TIME=`echo $$GREP_TIME | awk '{print $$2}'` ; \
 	echo "Command list built, Time taken: $$GREP_TIME"
 
-###############################################################################
-# DPDK
-###############################################################################
-
-if WITH_DPDK
-if ENABLE_DPDK_SHARED
-DPDK_LD_FLAGS = -Wl,--whole-archive,-ldpdk,--no-whole-archive
-else
-DPDK_LD_FLAGS = -Wl,--whole-archive,-l:libdpdk.a,--no-whole-archive,-lm,-ldl
-endif
-if WITH_DPDK_CRYPTO_SW
-DPDK_LD_ADD = -lIPSec_MB -lisal_crypto
-endif
-if WITH_DPDK_MLX5_PMD
-DPDK_LD_FLAGS += -libverbs -lmlx5 -lnuma
-endif
-else
-DPDK_LD_FLAGS =
-DPDK_LD_ADD =
-endif
-
 ###############################################################################
 # Components
 ###############################################################################
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 06b575d1..c8877899 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -24,6 +24,7 @@ vppplugins_LTLIBRARIES =
 vppapitestplugins_LTLIBRARIES =
 noinst_HEADERS =
 nobase_apiinclude_HEADERS =
+nobase_include_HEADERS =
 
 vppapitestpluginsdir = ${libdir}/vpp_api_test_plugins
 vpppluginsdir = ${libdir}/vpp_plugins
@@ -32,6 +33,10 @@ if ENABLE_ACL_PLUGIN
 include acl.am
 endif
 
+if WITH_DPDK
+include dpdk.am
+endif
+
 if ENABLE_FLOWPERPKT_PLUGIN
 include flowperpkt.am
 endif
diff --git a/src/plugins/dpdk.am b/src/plugins/dpdk.am
new file mode 100644
index 00000000..212bbb73
--- /dev/null
+++ b/src/plugins/dpdk.am
@@ -0,0 +1,50 @@
+# Copyright (c) 2016 Cisco Systems, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppapitestplugins_LTLIBRARIES += dpdk_test_plugin.la
+vppplugins_LTLIBRARIES += dpdk_plugin.la
+
+dpdk_plugin_la_LDFLAGS = $(AM_LDFLAGS) -Wl,--whole-archive,-l:libdpdk.a,--no-whole-archive,-lm,-ldl
+
+dpdk_plugin_la_SOURCES =					\
+  dpdk/init.c							\
+  dpdk/main.c							\
+  dpdk/buffer.c							\
+  dpdk/thread.c							\
+  dpdk/device/cli.c						\
+  dpdk/device/dpdk_priv.h					\
+  dpdk/device/device.c						\
+  dpdk/device/format.c						\
+  dpdk/device/node.c						\
+  dpdk/hqos/hqos.c						\
+  dpdk/ipsec/esp_encrypt.c					\
+  dpdk/ipsec/esp_decrypt.c					\
+  dpdk/ipsec/crypto_node.c					\
+  dpdk/ipsec/cli.c						\
+  dpdk/ipsec/ipsec.c						\
+  dpdk/api/dpdk_plugin.api.h
+
+API_FILES += dpdk/api/dpdk.api
+
+nobase_include_HEADERS +=					\
+  dpdk/device/dpdk.h						\
+  dpdk/api/dpdk_all_api_h.h
+
+nobase_include_HEADERS +=			\
+ dpdk/ipsec/ipsec.h		\
+ dpdk/ipsec/esp.h
+
+dpdk_test_plugin_la_SOURCES = 					\
+  dpdk/api/dpdk_test.c dpdk/api/dpdk_plugin.api.h
+
+# vi:syntax=automake
diff --git a/src/plugins/dpdk/api/dpdk.api b/src/plugins/dpdk/api/dpdk.api
new file mode 100644
index 00000000..21215d45
--- /dev/null
+++ b/src/plugins/dpdk/api/dpdk.api
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2015-2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** \brief DPDK interface HQoS pipe profile set request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - the interface
+    @param subport - subport ID
+    @param pipe - pipe ID within its subport
+    @param profile - pipe profile ID
+*/
+define sw_interface_set_dpdk_hqos_pipe {
+    u32 client_index;
+    u32 context;
+    u32 sw_if_index;
+    u32 subport;
+    u32 pipe;
+    u32 profile;
+};
+
+/** \brief DPDK interface HQoS pipe profile set reply
+    @param context - sender context, to match reply w/ request
+    @param retval - request return code
+*/
+define sw_interface_set_dpdk_hqos_pipe_reply {
+    u32 context;
+    i32 retval;
+};
+
+/** \brief DPDK interface HQoS subport parameters set request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - the interface
+    @param subport - subport ID
+    @param tb_rate - subport token bucket rate (measured in bytes/second)
+    @param tb_size - subport token bucket size (measured in credits)
+    @param tc_rate - subport traffic class 0 .. 3 rates (measured in bytes/second)
+    @param tc_period - enforcement period for rates (measured in milliseconds)
+*/
+define sw_interface_set_dpdk_hqos_subport {
+    u32 client_index;
+    u32 context;
+    u32 sw_if_index;
+    u32 subport;
+    u32 tb_rate;
+    u32 tb_size;
+    u32 tc_rate[4];
+    u32 tc_period;
+};
+
+/** \brief DPDK interface HQoS subport parameters set reply
+    @param context - sender context, to match reply w/ request
+    @param retval - request return code
+*/
+define sw_interface_set_dpdk_hqos_subport_reply {
+    u32 context;
+    i32 retval;
+};
+
+/** \brief DPDK interface HQoS tctbl entry set request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - the interface
+    @param entry - entry index ID
+    @param tc - traffic class (0 .. 3)
+    @param queue - traffic class queue (0 .. 3)
+*/
+define sw_interface_set_dpdk_hqos_tctbl {
+    u32 client_index;
+    u32 context;
+    u32 sw_if_index;
+    u32 entry;
+    u32 tc;
+    u32 queue;
+};
+
+/** \brief DPDK interface HQoS tctbl entry set reply
+    @param context - sender context, to match reply w/ request
+    @param retval - request return code
+*/
+define sw_interface_set_dpdk_hqos_tctbl_reply {
+    u32 context;
+    i32 retval;
+};
+
+/*
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
+ 
\ No newline at end of file
diff --git a/src/plugins/dpdk/api/dpdk_all_api_h.h b/src/plugins/dpdk/api/dpdk_all_api_h.h
new file mode 100644
index 00000000..15eb98d6
--- /dev/null
+++ b/src/plugins/dpdk/api/dpdk_all_api_h.h
@@ -0,0 +1,19 @@
+
+/*
+ * dpdk_all_api_h.h - skeleton vpp engine plug-in api #include file
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include 
diff --git a/src/plugins/dpdk/api/dpdk_msg_enum.h b/src/plugins/dpdk/api/dpdk_msg_enum.h
new file mode 100644
index 00000000..952ce6ad
--- /dev/null
+++ b/src/plugins/dpdk/api/dpdk_msg_enum.h
@@ -0,0 +1,31 @@
+
+/*
+ * dpdk_msg_enum.h - skeleton vpp engine plug-in message enumeration
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_dpdk_msg_enum_h
+#define included_dpdk_msg_enum_h
+
+#include 
+
+#define vl_msg_id(n,h) n,
+typedef enum {
+#include 
+    /* We'll want to know how many messages IDs we need... */
+    VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_dpdk_msg_enum_h */
diff --git a/src/plugins/dpdk/api/dpdk_test.c b/src/plugins/dpdk/api/dpdk_test.c
new file mode 100644
index 00000000..9fe0f934
--- /dev/null
+++ b/src/plugins/dpdk/api/dpdk_test.c
@@ -0,0 +1,397 @@
+
+/*
+ * dpdk_test.c - skeleton vpp-api-test plug-in
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+uword unformat_sw_if_index (unformat_input_t * input, va_list * args);
+
+/* Declare message IDs */
+#include 
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+
+#define vl_endianfun             /* define message structures */
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+typedef struct {
+    /* API message ID base */
+    u16 msg_id_base;
+    vat_main_t *vat_main;
+} dpdk_test_main_t;
+
+dpdk_test_main_t dpdk_test_main;
+
+#define foreach_standard_reply_retval_handler         \
+_(sw_interface_set_dpdk_hqos_pipe_reply)              \
+_(sw_interface_set_dpdk_hqos_subport_reply)           \
+_(sw_interface_set_dpdk_hqos_tctbl_reply)
+
+#define _(n)                                          \
+    static void vl_api_##n##_t_handler                \
+    (vl_api_##n##_t * mp)                             \
+    {                                                 \
+        vat_main_t * vam = dpdk_test_main.vat_main;  \
+        i32 retval = ntohl(mp->retval);               \
+        if (vam->async_mode) {                        \
+            vam->async_errors += (retval < 0);        \
+        } else {                                      \
+            vam->retval = retval;                     \
+            vam->result_ready = 1;                    \
+        }                                             \
+    }
+foreach_standard_reply_retval_handler;
+#undef _
+
+/* 
+ * Table of message reply handlers, must include boilerplate handlers
+ * we just generated
+ */
+#define foreach_vpe_api_reply_msg                               \
+_(SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY,                        \
+  sw_interface_set_dpdk_hqos_pipe_reply)                        \
+_(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY,                     \
+  sw_interface_set_dpdk_hqos_subport_reply)                     \
+_(SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY,                       \
+  sw_interface_set_dpdk_hqos_tctbl_reply)
+
+/* M: construct, but don't yet send a message */
+#define M(T,t)                                                  \
+do {                                                            \
+    vam->result_ready = 0;                                      \
+    mp = vl_msg_api_alloc(sizeof(*mp));                         \
+    memset (mp, 0, sizeof (*mp));                               \
+    mp->_vl_msg_id = ntohs (VL_API_##T + dm->msg_id_base);      \
+    mp->client_index = vam->my_client_index;                    \
+} while(0);
+
+#define M2(T,t,n)                                               \
+do {                                                            \
+    vam->result_ready = 0;                                      \
+    mp = vl_msg_api_alloc(sizeof(*mp)+(n));                     \
+    memset (mp, 0, sizeof (*mp));                               \
+    mp->_vl_msg_id = ntohs (VL_API_##T + dm->msg_id_base);      \
+    mp->client_index = vam->my_client_index;                    \
+} while(0);
+
+/* S: send a message */
+#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp))
+
+/* W: wait for results, with timeout */
+#define W                                       \
+do {                                            \
+    timeout = vat_time_now (vam) + 1.0;         \
+                                                \
+    while (vat_time_now (vam) < timeout) {      \
+        if (vam->result_ready == 1) {           \
+            return (vam->retval);               \
+        }                                       \
+    }                                           \
+    return -99;                                 \
+} while(0);
+
+static int
+api_sw_interface_set_dpdk_hqos_pipe (vat_main_t * vam)
+{
+  dpdk_test_main_t * dm = &dpdk_test_main;
+  unformat_input_t *i = vam->input;
+  vl_api_sw_interface_set_dpdk_hqos_pipe_t *mp;
+  f64 timeout;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  u32 subport;
+  u8 subport_set = 0;
+  u32 pipe;
+  u8 pipe_set = 0;
+  u32 profile;
+  u8 profile_set = 0;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+        if (unformat (i, "rx sw_if_index %u", &sw_if_index))
+  sw_if_index_set = 1;
+      else if (unformat (i, "subport %u", &subport))
+  subport_set = 1;
+      else if (unformat (i, "pipe %u", &pipe))
+  pipe_set = 1;
+      else if (unformat (i, "profile %u", &profile))
+  profile_set = 1;
+      else
+  break;
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      errmsg ("missing interface name or sw_if_index");
+      return -99;
+    }
+
+  if (subport_set == 0)
+    {
+      errmsg ("missing subport ");
+      return -99;
+    }
+
+  if (pipe_set == 0)
+    {
+      errmsg ("missing pipe");
+      return -99;
+    }
+
+  if (profile_set == 0)
+    {
+      errmsg ("missing profile");
+      return -99;
+    }
+
+  M (SW_INTERFACE_SET_DPDK_HQOS_PIPE, sw_interface_set_dpdk_hqos_pipe);
+
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->subport = ntohl (subport);
+  mp->pipe = ntohl (pipe);
+  mp->profile = ntohl (profile);
+
+
+  S;
+  W;
+  /* NOTREACHED */
+  return 0;
+}
+
+static int
+api_sw_interface_set_dpdk_hqos_subport (vat_main_t * vam)
+{
+  dpdk_test_main_t * dm = &dpdk_test_main;
+  unformat_input_t *i = vam->input;
+  vl_api_sw_interface_set_dpdk_hqos_subport_t *mp;
+  f64 timeout;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  u32 subport;
+  u8 subport_set = 0;
+  u32 tb_rate = 1250000000; /* 10GbE */
+  u32 tb_size = 1000000;
+  u32 tc_rate[] = { 1250000000, 1250000000, 1250000000, 1250000000 };
+  u32 tc_period = 10;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "rx sw_if_index %u", &sw_if_index))
+  sw_if_index_set = 1;
+      else if (unformat (i, "subport %u", &subport))
+  subport_set = 1;
+      else if (unformat (i, "rate %u", &tb_rate))
+  {
+    u32 tc_id;
+
+    for (tc_id = 0; tc_id < (sizeof (tc_rate) / sizeof (tc_rate[0]));
+         tc_id++)
+      tc_rate[tc_id] = tb_rate;
+  }
+      else if (unformat (i, "bktsize %u", &tb_size))
+  ;
+      else if (unformat (i, "tc0 %u", &tc_rate[0]))
+  ;
+      else if (unformat (i, "tc1 %u", &tc_rate[1]))
+  ;
+      else if (unformat (i, "tc2 %u", &tc_rate[2]))
+  ;
+      else if (unformat (i, "tc3 %u", &tc_rate[3]))
+  ;
+      else if (unformat (i, "period %u", &tc_period))
+  ;
+      else
+  break;
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      errmsg ("missing interface name or sw_if_index");
+      return -99;
+    }
+
+  if (subport_set == 0)
+    {
+      errmsg ("missing subport ");
+      return -99;
+    }
+
+  M (SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, sw_interface_set_dpdk_hqos_subport);
+
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->subport = ntohl (subport);
+  mp->tb_rate = ntohl (tb_rate);
+  mp->tb_size = ntohl (tb_size);
+  mp->tc_rate[0] = ntohl (tc_rate[0]);
+  mp->tc_rate[1] = ntohl (tc_rate[1]);
+  mp->tc_rate[2] = ntohl (tc_rate[2]);
+  mp->tc_rate[3] = ntohl (tc_rate[3]);
+  mp->tc_period = ntohl (tc_period);
+
+  S;
+  W;
+  /* NOTREACHED */
+  return 0;
+}
+
+static int
+api_sw_interface_set_dpdk_hqos_tctbl (vat_main_t * vam)
+{
+  dpdk_test_main_t * dm = &dpdk_test_main;
+  unformat_input_t *i = vam->input;
+  vl_api_sw_interface_set_dpdk_hqos_tctbl_t *mp;
+  f64 timeout;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  u8 entry_set = 0;
+  u8 tc_set = 0;
+  u8 queue_set = 0;
+  u32 entry, tc, queue;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "rx sw_if_index %u", &sw_if_index))
+  sw_if_index_set = 1;
+      else if (unformat (i, "entry %d", &entry))
+  entry_set = 1;
+      else if (unformat (i, "tc %d", &tc))
+  tc_set = 1;
+      else if (unformat (i, "queue %d", &queue))
+  queue_set = 1;
+      else
+  break;
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      errmsg ("missing interface name or sw_if_index");
+      return -99;
+    }
+
+  if (entry_set == 0)
+    {
+      errmsg ("missing entry ");
+      return -99;
+    }
+
+  if (tc_set == 0)
+    {
+      errmsg ("missing traffic class ");
+      return -99;
+    }
+
+  if (queue_set == 0)
+    {
+      errmsg ("missing queue ");
+      return -99;
+    }
+
+  M (SW_INTERFACE_SET_DPDK_HQOS_TCTBL, sw_interface_set_dpdk_hqos_tctbl);
+
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->entry = ntohl (entry);
+  mp->tc = ntohl (tc);
+  mp->queue = ntohl (queue);
+
+  S;
+  W;
+  /* NOTREACHED */
+  return 0;
+}
+
+/* 
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg                                               \
+_(sw_interface_set_dpdk_hqos_pipe,                                        \
+  "rx sw_if_index  subport  pipe \n"   \
+  "profile \n")                                               \
+_(sw_interface_set_dpdk_hqos_subport,                                     \
+  "rx sw_if_index  subport  [rate ]\n"       \
+  "[bktsize ] [tc0 ] [tc1 ] [tc2 ] [tc3 ] [period ]\n") \
+_(sw_interface_set_dpdk_hqos_tctbl,                                       \
+  "rx sw_if_index  entry  tc  queue \n")
+
+void vat_api_hookup (vat_main_t *vam)
+{
+  dpdk_test_main_t * dm __attribute__((unused)) = &dpdk_test_main;
+  /* Hook up handlers for replies from the data plane plug-in */
+#define _(N,n)                                                  \
+  vl_msg_api_set_handlers((VL_API_##N + dm->msg_id_base),       \
+                          #n,                                   \
+                          vl_api_##n##_t_handler,               \
+                          vl_noop_handler,                      \
+                          vl_api_##n##_t_endian,                \
+                          vl_api_##n##_t_print,                 \
+                          sizeof(vl_api_##n##_t), 1); 
+  foreach_vpe_api_reply_msg;
+#undef _
+
+  /* API messages we can send */
+#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _    
+    
+  /* Help strings */
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t * vat_plugin_register (vat_main_t *vam)
+{
+  dpdk_test_main_t * dm = &dpdk_test_main;
+  u8 * name;
+
+  dm->vat_main = vam;
+
+  /* Ask the vpp engine for the first assigned message-id */
+  name = format (0, "dpdk_%08x%c", api_version, 0);
+  dm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (dm->msg_id_base != (u16) ~0)
+    vat_api_hookup (vam);
+  
+  vec_free(name);
+  
+  return 0;
+}
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
new file mode 100644
index 00000000..2765c292
--- /dev/null
+++ b/src/plugins/dpdk/buffer.c
@@ -0,0 +1,588 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * buffer.c: allocate/free network buffers.
+ *
+ * Copyright (c) 2008 Eliot Dresselhaus
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file
+ *
+ * Allocate/free network buffers.
+ */
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+
+STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
+	       "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
+
+static_always_inline void
+dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
+{
+  vlib_buffer_t *hb = b;
+  struct rte_mbuf *mb;
+  u32 next, flags;
+  mb = rte_mbuf_from_vlib_buffer (hb);
+
+next:
+  flags = b->flags;
+  next = b->next_buffer;
+  mb = rte_mbuf_from_vlib_buffer (b);
+
+  if (PREDICT_FALSE (b->n_add_refs))
+    {
+      rte_mbuf_refcnt_update (mb, b->n_add_refs);
+      b->n_add_refs = 0;
+    }
+
+  rte_pktmbuf_free_seg (mb);
+
+  if (flags & VLIB_BUFFER_NEXT_PRESENT)
+    {
+      b = vlib_get_buffer (vm, next);
+      goto next;
+    }
+}
+
+static void
+del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
+{
+  u32 i;
+  vlib_buffer_t *b;
+
+  for (i = 0; i < vec_len (f->buffers); i++)
+    {
+      b = vlib_get_buffer (vm, f->buffers[i]);
+      dpdk_rte_pktmbuf_free (vm, b);
+    }
+
+  vec_free (f->name);
+  vec_free (f->buffers);
+}
+
+/* Add buffer free list. */
+static void
+dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  vlib_buffer_free_list_t *f;
+  u32 merge_index;
+  int i;
+
+  ASSERT (os_get_cpu_number () == 0);
+
+  f = vlib_buffer_get_free_list (vm, free_list_index);
+
+  merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
+  if (merge_index != ~0 && merge_index != free_list_index)
+    {
+      vlib_buffer_merge_free_lists (pool_elt_at_index
+				    (bm->buffer_free_list_pool, merge_index),
+				    f);
+    }
+
+  del_free_list (vm, f);
+
+  /* Poison it. */
+  memset (f, 0xab, sizeof (f[0]));
+
+  pool_put (bm->buffer_free_list_pool, f);
+
+  for (i = 1; i < vec_len (vlib_mains); i++)
+    {
+      bm = vlib_mains[i]->buffer_main;
+      f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);;
+      memset (f, 0xab, sizeof (f[0]));
+      pool_put (bm->buffer_free_list_pool, f);
+    }
+}
+
+/* Make sure free list has at least given number of free buffers. */
+static uword
+fill_free_list (vlib_main_t * vm,
+		vlib_buffer_free_list_t * fl, uword min_free_buffers)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  vlib_buffer_t *b0, *b1, *b2, *b3;
+  int n, i;
+  u32 bi0, bi1, bi2, bi3;
+  unsigned socket_id = rte_socket_id ();
+  struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
+  struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
+
+  /* Too early? */
+  if (PREDICT_FALSE (rmp == 0))
+    return 0;
+
+  /* Already have enough free buffers on free list? */
+  n = min_free_buffers - vec_len (fl->buffers);
+  if (n <= 0)
+    return min_free_buffers;
+
+  /* Always allocate round number of buffers. */
+  n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
+
+  /* Always allocate new buffers in reasonably large sized chunks. */
+  n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
+
+  vec_validate (vm->mbuf_alloc_list, n - 1);
+
+  if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0)
+    return 0;
+
+  _vec_len (vm->mbuf_alloc_list) = n;
+
+  i = 0;
+
+  while (i < (n - 7))
+    {
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+				   (vm->mbuf_alloc_list[i + 4]), STORE);
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+				   (vm->mbuf_alloc_list[i + 5]), STORE);
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+				   (vm->mbuf_alloc_list[i + 6]), STORE);
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+				   (vm->mbuf_alloc_list[i + 7]), STORE);
+
+      mb0 = vm->mbuf_alloc_list[i];
+      mb1 = vm->mbuf_alloc_list[i + 1];
+      mb2 = vm->mbuf_alloc_list[i + 2];
+      mb3 = vm->mbuf_alloc_list[i + 3];
+
+      ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
+      ASSERT (rte_mbuf_refcnt_read (mb1) == 0);
+      ASSERT (rte_mbuf_refcnt_read (mb2) == 0);
+      ASSERT (rte_mbuf_refcnt_read (mb3) == 0);
+
+      rte_mbuf_refcnt_set (mb0, 1);
+      rte_mbuf_refcnt_set (mb1, 1);
+      rte_mbuf_refcnt_set (mb2, 1);
+      rte_mbuf_refcnt_set (mb3, 1);
+
+      b0 = vlib_buffer_from_rte_mbuf (mb0);
+      b1 = vlib_buffer_from_rte_mbuf (mb1);
+      b2 = vlib_buffer_from_rte_mbuf (mb2);
+      b3 = vlib_buffer_from_rte_mbuf (mb3);
+
+      bi0 = vlib_get_buffer_index (vm, b0);
+      bi1 = vlib_get_buffer_index (vm, b1);
+      bi2 = vlib_get_buffer_index (vm, b2);
+      bi3 = vlib_get_buffer_index (vm, b3);
+
+      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
+      vec_add1_aligned (fl->buffers, bi1, CLIB_CACHE_LINE_BYTES);
+      vec_add1_aligned (fl->buffers, bi2, CLIB_CACHE_LINE_BYTES);
+      vec_add1_aligned (fl->buffers, bi3, CLIB_CACHE_LINE_BYTES);
+
+      vlib_buffer_init_for_free_list (b0, fl);
+      vlib_buffer_init_for_free_list (b1, fl);
+      vlib_buffer_init_for_free_list (b2, fl);
+      vlib_buffer_init_for_free_list (b3, fl);
+
+      if (fl->buffer_init_function)
+	{
+	  fl->buffer_init_function (vm, fl, &bi0, 1);
+	  fl->buffer_init_function (vm, fl, &bi1, 1);
+	  fl->buffer_init_function (vm, fl, &bi2, 1);
+	  fl->buffer_init_function (vm, fl, &bi3, 1);
+	}
+      i += 4;
+    }
+
+  while (i < n)
+    {
+      mb0 = vm->mbuf_alloc_list[i];
+
+      ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
+      rte_mbuf_refcnt_set (mb0, 1);
+
+      b0 = vlib_buffer_from_rte_mbuf (mb0);
+      bi0 = vlib_get_buffer_index (vm, b0);
+
+      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
+
+      vlib_buffer_init_for_free_list (b0, fl);
+
+      if (fl->buffer_init_function)
+	fl->buffer_init_function (vm, fl, &bi0, 1);
+      i++;
+    }
+
+  fl->n_alloc += n;
+
+  return n;
+}
+
+static u32
+alloc_from_free_list (vlib_main_t * vm,
+		      vlib_buffer_free_list_t * free_list,
+		      u32 * alloc_buffers, u32 n_alloc_buffers)
+{
+  u32 *dst, *src;
+  uword len, n_filled;
+
+  dst = alloc_buffers;
+
+  n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
+  if (n_filled == 0)
+    return 0;
+
+  len = vec_len (free_list->buffers);
+  ASSERT (len >= n_alloc_buffers);
+
+  src = free_list->buffers + len - n_alloc_buffers;
+  clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
+
+  _vec_len (free_list->buffers) -= n_alloc_buffers;
+
+  return n_alloc_buffers;
+}
+
+/* Allocate a given number of buffers into given array.
+   Returns number actually allocated which will be either zero or
+   number requested. */
+u32
+dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+
+  return alloc_from_free_list
+    (vm,
+     pool_elt_at_index (bm->buffer_free_list_pool,
+			VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
+     buffers, n_buffers);
+}
+
+
+u32
+dpdk_buffer_alloc_from_free_list (vlib_main_t * vm,
+				  u32 * buffers,
+				  u32 n_buffers, u32 free_list_index)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  vlib_buffer_free_list_t *f;
+  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
+  return alloc_from_free_list (vm, f, buffers, n_buffers);
+}
+
+static_always_inline void
+vlib_buffer_free_inline (vlib_main_t * vm,
+			 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  vlib_buffer_free_list_t *fl;
+  u32 fi;
+  int i;
+  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
+	     u32 follow_buffer_next);
+
+  cb = bm->buffer_free_callback;
+
+  if (PREDICT_FALSE (cb != 0))
+    n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
+
+  if (!n_buffers)
+    return;
+
+  for (i = 0; i < n_buffers; i++)
+    {
+      vlib_buffer_t *b;
+
+      b = vlib_get_buffer (vm, buffers[i]);
+
+      fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
+
+      /* The only current use of this callback: multicast recycle */
+      if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
+	{
+	  int j;
+
+	  vlib_buffer_add_to_free_list
+	    (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
+
+	  for (j = 0; j < vec_len (bm->announce_list); j++)
+	    {
+	      if (fl == bm->announce_list[j])
+		goto already_announced;
+	    }
+	  vec_add1 (bm->announce_list, fl);
+	already_announced:
+	  ;
+	}
+      else
+	{
+	  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+	    dpdk_rte_pktmbuf_free (vm, b);
+	}
+    }
+  if (vec_len (bm->announce_list))
+    {
+      vlib_buffer_free_list_t *fl;
+      for (i = 0; i < vec_len (bm->announce_list); i++)
+	{
+	  fl = bm->announce_list[i];
+	  fl->buffers_added_to_freelist_function (vm, fl);
+	}
+      _vec_len (bm->announce_list) = 0;
+    }
+}
+
+static void
+dpdk_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+  vlib_buffer_free_inline (vm, buffers, n_buffers,	/* follow_buffer_next */
+			   1);
+}
+
+static void
+dpdk_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+  vlib_buffer_free_inline (vm, buffers, n_buffers,	/* follow_buffer_next */
+			   0);
+}
+
+static void
+dpdk_packet_template_init (vlib_main_t * vm,
+			   void *vt,
+			   void *packet_data,
+			   uword n_packet_data_bytes,
+			   uword min_n_buffers_each_physmem_alloc, u8 * name)
+{
+  vlib_packet_template_t *t = (vlib_packet_template_t *) vt;
+
+  vlib_worker_thread_barrier_sync (vm);
+  memset (t, 0, sizeof (t[0]));
+
+  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
+
+  vlib_worker_thread_barrier_release (vm);
+}
+
+clib_error_t *
+vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
+			 unsigned socket_id)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  vlib_physmem_main_t *vpm = &vm->physmem_main;
+  struct rte_mempool *rmp;
+  int i;
+
+  vec_validate_aligned (dm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
+
+  /* pool already exists, nothing to do */
+  if (dm->pktmbuf_pools[socket_id])
+    return 0;
+
+  u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0);
+
+  rmp = rte_pktmbuf_pool_create ((char *) pool_name,	/* pool name */
+				 num_mbufs,	/* number of mbufs */
+				 512,	/* cache size */
+				 VLIB_BUFFER_HDR_SIZE,	/* priv size */
+				 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE,	/* dataroom size */
+				 socket_id);	/* cpu socket */
+
+  if (rmp)
+    {
+      {
+	uword this_pool_end;
+	uword this_pool_start;
+	uword this_pool_size;
+	uword save_vpm_start, save_vpm_end, save_vpm_size;
+	struct rte_mempool_memhdr *memhdr;
+
+	this_pool_start = ~0ULL;
+	this_pool_end = 0LL;
+
+	STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
+	{
+	  if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
+	    this_pool_end = (uword) (memhdr->addr + memhdr->len);
+	  if (((uword) memhdr->addr) < this_pool_start)
+	    this_pool_start = (uword) (memhdr->addr);
+	}
+	ASSERT (this_pool_start < ~0ULL && this_pool_end > 0);
+	this_pool_size = this_pool_end - this_pool_start;
+
+	if (CLIB_DEBUG > 1)
+	  {
+	    clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
+			  pool_name, this_pool_start, this_pool_end,
+			  this_pool_size);
+	    clib_warning
+	      ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
+	       vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
+	  }
+
+	save_vpm_start = vpm->virtual.start;
+	save_vpm_end = vpm->virtual.end;
+	save_vpm_size = vpm->virtual.size;
+
+	if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
+	  vpm->virtual.start = this_pool_start;
+	if (this_pool_end > vpm->virtual.end)
+	  vpm->virtual.end = this_pool_end;
+
+	vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
+
+	if (CLIB_DEBUG > 1)
+	  {
+	    clib_warning
+	      ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
+	       vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
+	  }
+
+	/* check if fits into buffer index range */
+	if ((u64) vpm->virtual.size >
+	    ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
+	  {
+	    clib_warning ("physmem: virtual size out of range!");
+	    vpm->virtual.start = save_vpm_start;
+	    vpm->virtual.end = save_vpm_end;
+	    vpm->virtual.size = save_vpm_size;
+	    rmp = 0;
+	  }
+      }
+      if (rmp)
+	{
+	  dm->pktmbuf_pools[socket_id] = rmp;
+	  vec_free (pool_name);
+	  return 0;
+	}
+    }
+
+  vec_free (pool_name);
+
+  /* no usable pool for this socket, try to use pool from another one */
+  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
+    {
+      if (dm->pktmbuf_pools[i])
+	{
+	  clib_warning
+	    ("WARNING: Failed to allocate mempool for CPU socket %u. "
+	     "Threads running on socket %u will use socket %u mempool.",
+	     socket_id, socket_id, i);
+	  dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i];
+	  return 0;
+	}
+    }
+
+  return clib_error_return (0, "failed to allocate mempool on socket %u",
+			    socket_id);
+}
+
+#if CLIB_DEBUG > 0
+
+u32 *vlib_buffer_state_validation_lock;
+uword *vlib_buffer_state_validation_hash;
+void *vlib_buffer_state_heap;
+
+static clib_error_t *
+buffer_state_validation_init (vlib_main_t * vm)
+{
+  void *oldheap;
+
+  vlib_buffer_state_heap = mheap_alloc (0, 10 << 20);
+
+  oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
+
+  vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword));
+  vec_validate_aligned (vlib_buffer_state_validation_lock, 0,
+			CLIB_CACHE_LINE_BYTES);
+  clib_mem_set_heap (oldheap);
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (buffer_state_validation_init);
+#endif
+
+static vlib_buffer_callbacks_t callbacks = {
+  .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
+  .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
+  .vlib_buffer_free_cb = &dpdk_buffer_free,
+  .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
+  .vlib_packet_template_init_cb = &dpdk_packet_template_init,
+  .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
+};
+
+static clib_error_t *
+dpdk_buffer_init (vlib_main_t * vm)
+{
+  vlib_buffer_cb_register (vm, &callbacks);
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (dpdk_buffer_init);
+
+/** @endcond */
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c
new file mode 100644
index 00000000..d2def2fc
--- /dev/null
+++ b/src/plugins/dpdk/device/cli.c
@@ -0,0 +1,2079 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+/**
+ * @file
+ * @brief CLI for DPDK Abstraction Layer and pcap Tx Trace.
+ *
+ * This file contains the source code for CLI for DPDK
+ * Abstraction Layer and pcap Tx Trace.
+ */
+
+
+static clib_error_t *
+get_hqos (u32 hw_if_index, u32 subport_id, dpdk_device_t ** xd,
+	  dpdk_device_config_t ** devconf)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_hw_interface_t *hw;
+  struct rte_eth_dev_info dev_info;
+  uword *p = 0;
+  clib_error_t *error = NULL;
+
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
+
+  if (subport_id != 0)
+    {
+      error = clib_error_return (0, "Invalid subport");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  *xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rte_eth_dev_info_get ((*xd)->device_index, &dev_info);
+  if (dev_info.pci_dev)
+    {				/* bonded interface has no pci info */
+      vlib_pci_addr_t pci_addr;
+
+      pci_addr.domain = dev_info.pci_dev->addr.domain;
+      pci_addr.bus = dev_info.pci_dev->addr.bus;
+      pci_addr.slot = dev_info.pci_dev->addr.devid;
+      pci_addr.function = dev_info.pci_dev->addr.function;
+
+      p =
+	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
+    }
+
+  if (p)
+    (*devconf) = pool_elt_at_index (dm->conf->dev_confs, p[0]);
+  else
+    (*devconf) = &dm->conf->default_devconf;
+
+done:
+  return error;
+}
+
+static clib_error_t *
+pcap_trace_command_fn (vlib_main_t * vm,
+		       unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+#define PCAP_DEF_PKT_TO_CAPTURE (100)
+
+  unformat_input_t _line_input, *line_input = &_line_input;
+  dpdk_main_t *dm = &dpdk_main;
+  u8 *filename;
+  u8 *chroot_filename = 0;
+  u32 max = 0;
+  int enabled = 0;
+  int errorFlag = 0;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "on"))
+	{
+	  if (dm->tx_pcap_enable == 0)
+	    {
+	      enabled = 1;
+	    }
+	  else
+	    {
+	      vlib_cli_output (vm, "pcap tx capture already on...");
+	      errorFlag = 1;
+	      break;
+	    }
+	}
+      else if (unformat (line_input, "off"))
+	{
+	  if (dm->tx_pcap_enable)
+	    {
+	      vlib_cli_output (vm, "captured %d pkts...",
+			       dm->pcap_main.n_packets_captured + 1);
+	      if (dm->pcap_main.n_packets_captured)
+		{
+		  dm->pcap_main.n_packets_to_capture =
+		    dm->pcap_main.n_packets_captured;
+		  error = pcap_write (&dm->pcap_main);
+		  if (error)
+		    clib_error_report (error);
+		  else
+		    vlib_cli_output (vm, "saved to %s...", dm->pcap_filename);
+		}
+
+	      dm->tx_pcap_enable = 0;
+	    }
+	  else
+	    {
+	      vlib_cli_output (vm, "pcap tx capture already off...");
+	      errorFlag = 1;
+	      break;
+	    }
+	}
+      else if (unformat (line_input, "max %d", &max))
+	{
+	  if (dm->tx_pcap_enable)
+	    {
+	      vlib_cli_output (vm,
+			       "can't change max value while pcap tx capture active...");
+	      errorFlag = 1;
+	      break;
+	    }
+	}
+      else if (unformat (line_input, "intfc %U",
+			 unformat_vnet_sw_interface, dm->vnet_main,
+			 &dm->pcap_sw_if_index))
+	;
+
+      else if (unformat (line_input, "intfc any"))
+	{
+	  dm->pcap_sw_if_index = 0;
+	}
+      else if (unformat (line_input, "file %s", &filename))
+	{
+	  if (dm->tx_pcap_enable)
+	    {
+	      vlib_cli_output (vm,
+			       "can't change file while pcap tx capture active...");
+	      errorFlag = 1;
+	      break;
+	    }
+
+	  /* Brain-police user path input */
+	  if (strstr ((char *) filename, "..")
+	      || index ((char *) filename, '/'))
+	    {
+	      vlib_cli_output (vm, "illegal characters in filename '%s'",
+			       filename);
+	      vlib_cli_output (vm,
+			       "Hint: Only filename, do not enter directory structure.");
+	      vec_free (filename);
+	      errorFlag = 1;
+	      break;
+	    }
+
+	  chroot_filename = format (0, "/tmp/%s%c", filename, 0);
+	  vec_free (filename);
+	}
+      else if (unformat (line_input, "status"))
+	{
+	  if (dm->pcap_sw_if_index == 0)
+	    {
+	      vlib_cli_output (vm, "max is %d for any interface to file %s",
+			       dm->
+			       pcap_pkts_to_capture ? dm->pcap_pkts_to_capture
+			       : PCAP_DEF_PKT_TO_CAPTURE,
+			       dm->
+			       pcap_filename ? dm->pcap_filename : (u8 *)
+			       "/tmp/vpe.pcap");
+	    }
+	  else
+	    {
+	      vlib_cli_output (vm, "max is %d for interface %U to file %s",
+			       dm->
+			       pcap_pkts_to_capture ? dm->pcap_pkts_to_capture
+			       : PCAP_DEF_PKT_TO_CAPTURE,
+			       format_vnet_sw_if_index_name, dm->vnet_main,
+			       dm->pcap_sw_if_index,
+			       dm->
+			       pcap_filename ? dm->pcap_filename : (u8 *)
+			       "/tmp/vpe.pcap");
+	    }
+
+	  if (dm->tx_pcap_enable == 0)
+	    {
+	      vlib_cli_output (vm, "pcap tx capture is off...");
+	    }
+	  else
+	    {
+	      vlib_cli_output (vm, "pcap tx capture is on: %d of %d pkts...",
+			       dm->pcap_main.n_packets_captured,
+			       dm->pcap_main.n_packets_to_capture);
+	    }
+	  break;
+	}
+
+      else
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  errorFlag = 1;
+	  break;
+	}
+    }
+  unformat_free (line_input);
+
+
+  if (errorFlag == 0)
+    {
+      /* Since no error, save configured values. */
+      if (chroot_filename)
+	{
+	  if (dm->pcap_filename)
+	    vec_free (dm->pcap_filename);
+	  vec_add1 (chroot_filename, 0);
+	  dm->pcap_filename = chroot_filename;
+	}
+
+      if (max)
+	dm->pcap_pkts_to_capture = max;
+
+
+      if (enabled)
+	{
+	  if (dm->pcap_filename == 0)
+	    dm->pcap_filename = format (0, "/tmp/vpe.pcap%c", 0);
+
+	  memset (&dm->pcap_main, 0, sizeof (dm->pcap_main));
+	  dm->pcap_main.file_name = (char *) dm->pcap_filename;
+	  dm->pcap_main.n_packets_to_capture = PCAP_DEF_PKT_TO_CAPTURE;
+	  if (dm->pcap_pkts_to_capture)
+	    dm->pcap_main.n_packets_to_capture = dm->pcap_pkts_to_capture;
+
+	  dm->pcap_main.packet_type = PCAP_PACKET_TYPE_ethernet;
+	  dm->tx_pcap_enable = 1;
+	  vlib_cli_output (vm, "pcap tx capture on...");
+	}
+    }
+  else if (chroot_filename)
+    vec_free (chroot_filename);
+
+
+  return error;
+}
+
+/*?
+ * This command is used to start or stop a packet capture, or show
+ * the status of packet capture.
+ *
+ * This command has the following optional parameters:
+ *
+ * - on|off - Used to start or stop a packet capture.
+ *
+ * - max  - Depth of local buffer. Once 'nn' number
+ *   of packets have been received, buffer is flushed to file. Once another
+ *   'nn' number of packets have been received, buffer is flushed
+ *   to file, overwriting previous write. If not entered, value defaults
+ *   to 100. Can only be updated if packet capture is off.
+ *
+ * - intfc |any - Used to specify a given interface,
+ *   or use 'any' to run packet capture on all interfaces.
+ *   'any' is the default if not provided. Settings from a previous
+ *   packet capture are preserved, so 'any' can be used to reset
+ *   the interface setting.
+ *
+ * - file  - Used to specify the output filename. The file will
+ *   be placed in the '/tmp' directory, so only the filename is
+ *   supported. Directory should not be entered. If file already exists, file
+ *   will be overwritten. If no filename is provided, '/tmp/vpe.pcap'
+ *   will be used. Can only be updated if packet capture is off.
+ *
+ * - status - Displays the current status and configured attributes
+ *   associated with a packet capture. If packet capture is in progress,
+ *   'status' also will return the number of packets currently in
+ *   the local buffer. All additional attributes entered on command line
+ *   with 'status' will be ingnored and not applied.
+ *
+ * @cliexpar
+ * Example of how to display the status of a tx packet capture when off:
+ * @cliexstart{pcap tx trace status}
+ * max is 100, for any interface to file /tmp/vpe.pcap
+ * pcap tx capture is off...
+ * @cliexend
+ * Example of how to start a tx packet capture:
+ * @cliexstart{pcap tx trace on max 35 intfc GigabitEthernet0/8/0 file vppTest.pcap}
+ * pcap tx capture on...
+ * @cliexend
+ * Example of how to display the status of a tx packet capture in progress:
+ * @cliexstart{pcap tx trace status}
+ * max is 35, for interface GigabitEthernet0/8/0 to file /tmp/vppTest.pcap
+ * pcap tx capture is on: 20 of 35 pkts...
+ * @cliexend
+ * Example of how to stop a tx packet capture:
+ * @cliexstart{vppctl pcap tx trace off}
+ * captured 21 pkts...
+ * saved to /tmp/vppTest.pcap...
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (pcap_trace_command, static) = {
+    .path = "pcap tx trace",
+    .short_help =
+    "pcap tx trace [on|off] [max ] [intfc |any] [file ] [status]",
+    .function = pcap_trace_command_fn,
+};
+/* *INDENT-ON* */
+
+
+static clib_error_t *
+show_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input,
+		  vlib_cli_command_t * cmd)
+{
+  struct rte_mempool *rmp;
+  int i;
+
+  for (i = 0; i < vec_len (dpdk_main.pktmbuf_pools); i++)
+    {
+      rmp = dpdk_main.pktmbuf_pools[i];
+      if (rmp)
+	{
+	  unsigned count = rte_mempool_avail_count (rmp);
+	  unsigned free_count = rte_mempool_in_use_count (rmp);
+
+	  vlib_cli_output (vm,
+			   "name=\"%s\"  available = %7d allocated = %7d total = %7d\n",
+			   rmp->name, (u32) count, (u32) free_count,
+			   (u32) (count + free_count));
+	}
+      else
+	{
+	  vlib_cli_output (vm, "rte_mempool is NULL (!)\n");
+	}
+    }
+  return 0;
+}
+
+/*?
+ * This command displays statistics of each DPDK mempool.
+ *
+ * @cliexpar
+ * Example of how to display DPDK buffer data:
+ * @cliexstart{show dpdk buffer}
+ * name="mbuf_pool_socket0"  available =   15104 allocated =    1280 total =   16384
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_show_dpdk_bufferr,static) = {
+    .path = "show dpdk buffer",
+    .short_help = "show dpdk buffer",
+    .function = show_dpdk_buffer,
+    .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+test_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input,
+		  vlib_cli_command_t * cmd)
+{
+  static u32 *allocated_buffers;
+  u32 n_alloc = 0;
+  u32 n_free = 0;
+  u32 first, actual_alloc;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "allocate %d", &n_alloc))
+	;
+      else if (unformat (input, "free %d", &n_free))
+	;
+      else
+	break;
+    }
+
+  if (n_free)
+    {
+      if (vec_len (allocated_buffers) < n_free)
+	return clib_error_return (0, "Can't free %d, only %d allocated",
+				  n_free, vec_len (allocated_buffers));
+
+      first = vec_len (allocated_buffers) - n_free;
+      vlib_buffer_free (vm, allocated_buffers + first, n_free);
+      _vec_len (allocated_buffers) = first;
+    }
+  if (n_alloc)
+    {
+      first = vec_len (allocated_buffers);
+      vec_validate (allocated_buffers,
+		    vec_len (allocated_buffers) + n_alloc - 1);
+
+      actual_alloc = vlib_buffer_alloc (vm, allocated_buffers + first,
+					n_alloc);
+      _vec_len (allocated_buffers) = first + actual_alloc;
+
+      if (actual_alloc < n_alloc)
+	vlib_cli_output (vm, "WARNING: only allocated %d buffers",
+			 actual_alloc);
+    }
+
+  vlib_cli_output (vm, "Currently %d buffers allocated",
+		   vec_len (allocated_buffers));
+
+  if (allocated_buffers && vec_len (allocated_buffers) == 0)
+    vec_free (allocated_buffers);
+
+  return 0;
+}
+
+/*?
+ * This command tests the allocation and freeing of DPDK buffers.
+ * If both 'allocate' and 'free' are entered on the
+ * same command, the 'free' is executed first. If no
+ * parameters are provided, this command display how many DPDK buffers
+ * the test command has allocated.
+ *
+ * @cliexpar
+ * @parblock
+ *
+ * Example of how to display how many DPDK buffer test command has allcoated:
+ * @cliexstart{test dpdk buffer}
+ * Currently 0 buffers allocated
+ * @cliexend
+ *
+ * Example of how to allocate DPDK buffers using the test command:
+ * @cliexstart{test dpdk buffer allocate 10}
+ * Currently 10 buffers allocated
+ * @cliexend
+ *
+ * Example of how to free DPDK buffers allocated by the test command:
+ * @cliexstart{test dpdk buffer free 10}
+ * Currently 0 buffers allocated
+ * @cliexend
+ * @endparblock
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_test_dpdk_buffer,static) = {
+    .path = "test dpdk buffer",
+    .short_help = "test dpdk buffer [allocate ] [free ]",
+    .function = test_dpdk_buffer,
+    .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input,
+		  vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 nb_rx_desc = (u32) ~ 0;
+  u32 nb_tx_desc = (u32) ~ 0;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "tx %d", &nb_tx_desc))
+	;
+      else if (unformat (line_input, "rx %d", &nb_rx_desc))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
+    {
+      error =
+	clib_error_return (0,
+			   "number of descriptors can be set only for "
+			   "physical devices");
+      goto done;
+    }
+
+  if ((nb_rx_desc == (u32) ~ 0 || nb_rx_desc == xd->nb_rx_desc) &&
+      (nb_tx_desc == (u32) ~ 0 || nb_tx_desc == xd->nb_tx_desc))
+    {
+      error = clib_error_return (0, "nothing changed");
+      goto done;
+    }
+
+  if (nb_rx_desc != (u32) ~ 0)
+    xd->nb_rx_desc = nb_rx_desc;
+
+  if (nb_tx_desc != (u32) ~ 0)
+    xd->nb_tx_desc = nb_tx_desc;
+
+  error = dpdk_port_setup (dm, xd);
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command sets the number of DPDK 'rx' and
+ * 'tx' descriptors for the given physical interface. Use
+ * the command 'show hardware-interface' to display the
+ * current descriptor allocation.
+ *
+ * @cliexpar
+ * Example of how to set the DPDK interface descriptors:
+ * @cliexcmd{set dpdk interface descriptors GigabitEthernet0/8/0 rx 512 tx 512}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_desc,static) = {
+    .path = "set dpdk interface descriptors",
+    .short_help = "set dpdk interface descriptors  [rx ] [tx ]",
+    .function = set_dpdk_if_desc,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
+			vlib_cli_command_t * cmd)
+{
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_and_queue_t *dq;
+  int cpu;
+
+  if (tm->n_vlib_mains == 1)
+    vlib_cli_output (vm, "All interfaces are handled by main thread");
+
+  for (cpu = 0; cpu < vec_len (dm->devices_by_cpu); cpu++)
+    {
+      if (cpu >= dm->input_cpu_first_index &&
+	  cpu < (dm->input_cpu_first_index + dm->input_cpu_count))
+	vlib_cli_output (vm, "Thread %u (%s at lcore %u):", cpu,
+			 vlib_worker_threads[cpu].name,
+			 vlib_worker_threads[cpu].lcore_id);
+
+      /* *INDENT-OFF* */
+      vec_foreach(dq, dm->devices_by_cpu[cpu])
+        {
+          u32 hw_if_index = dm->devices[dq->device].vlib_hw_if_index;
+          vnet_hw_interface_t * hi =  vnet_get_hw_interface(dm->vnet_main, hw_if_index);
+          vlib_cli_output(vm, "  %v queue %u", hi->name, dq->queue_id);
+        }
+      /* *INDENT-ON* */
+    }
+  return 0;
+}
+
+/*?
+ * This command is used to display the thread and core each
+ * DPDK interface and queue is assigned too.
+ *
+ * @cliexpar
+ * Example of how to display the DPDK interface placement:
+ * @cliexstart{show dpdk interface placement}
+ * Thread 1 (vpp_wk_0 at lcore 1):
+ *   GigabitEthernet0/8/0 queue 0
+ *   GigabitEthernet0/9/0 queue 0
+ * Thread 2 (vpp_wk_1 at lcore 2):
+ *   GigabitEthernet0/8/0 queue 1
+ *   GigabitEthernet0/9/0 queue 1
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_show_dpdk_if_placement,static) = {
+    .path = "show dpdk interface placement",
+    .short_help = "show dpdk interface placement",
+    .function = show_dpdk_if_placement,
+};
+/* *INDENT-ON* */
+
+static int
+dpdk_device_queue_sort (void *a1, void *a2)
+{
+  dpdk_device_and_queue_t *dq1 = a1;
+  dpdk_device_and_queue_t *dq2 = a2;
+
+  if (dq1->device > dq2->device)
+    return 1;
+  else if (dq1->device < dq2->device)
+    return -1;
+  else if (dq1->queue_id > dq2->queue_id)
+    return 1;
+  else if (dq1->queue_id < dq2->queue_id)
+    return -1;
+  else
+    return 0;
+}
+
+static clib_error_t *
+set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
+		       vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_and_queue_t *dq;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 queue = (u32) 0;
+  u32 cpu = (u32) ~ 0;
+  int i;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "queue %d", &queue))
+	;
+      else if (unformat (line_input, "thread %d", &cpu))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
+
+  if (cpu < dm->input_cpu_first_index ||
+      cpu >= (dm->input_cpu_first_index + dm->input_cpu_count))
+    {
+      error = clib_error_return (0, "please specify valid thread id");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  for (i = 0; i < vec_len (dm->devices_by_cpu); i++)
+    {
+      /* *INDENT-OFF* */
+      vec_foreach(dq, dm->devices_by_cpu[i])
+        {
+          if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index &&
+              queue == dq->queue_id)
+            {
+              if (cpu == i) /* nothing to do */
+                goto done;
+
+              vec_del1(dm->devices_by_cpu[i], dq - dm->devices_by_cpu[i]);
+              vec_add2(dm->devices_by_cpu[cpu], dq, 1);
+              dq->queue_id = queue;
+              dq->device = xd->device_index;
+              xd->cpu_socket_id_by_queue[queue] =
+                rte_lcore_to_socket_id(vlib_worker_threads[cpu].lcore_id);
+
+              vec_sort_with_function(dm->devices_by_cpu[i],
+		                     dpdk_device_queue_sort);
+
+              vec_sort_with_function(dm->devices_by_cpu[cpu],
+		                     dpdk_device_queue_sort);
+
+              if (vec_len(dm->devices_by_cpu[i]) == 0)
+                vlib_node_set_state (vlib_mains[i], dpdk_input_node.index,
+		                     VLIB_NODE_STATE_DISABLED);
+
+              if (vec_len(dm->devices_by_cpu[cpu]) == 1)
+                vlib_node_set_state (vlib_mains[cpu], dpdk_input_node.index,
+		                     VLIB_NODE_STATE_POLLING);
+
+              goto done;
+            }
+        }
+      /* *INDENT-ON* */
+    }
+
+  error = clib_error_return (0, "not found");
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to assign a given interface, and optionally a
+ * given queue, to a different thread. This will not create a thread,
+ * so the thread must already exist. Use '/etc/vpp/startup.conf'
+ * for the initial thread creation. If the 'queue' is not provided,
+ * it defaults to 0.
+ *
+ * @cliexpar
+ * Example of how to display the DPDK interface placement:
+ * @cliexstart{show dpdk interface placement}
+ * Thread 1 (vpp_wk_0 at lcore 1):
+ *   GigabitEthernet0/8/0 queue 0
+ *   GigabitEthernet0/9/0 queue 0
+ * Thread 2 (vpp_wk_1 at lcore 2):
+ *   GigabitEthernet0/8/0 queue 1
+ *   GigabitEthernet0/9/0 queue 1
+ * @cliexend
+ * Example of how to assign a DPDK interface and queue to a thread:
+ * @cliexcmd{set dpdk interface placement GigabitEthernet0/8/0 queue 1 thread 1}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_placement,static) = {
+    .path = "set dpdk interface placement",
+    .short_help = "set dpdk interface placement  [queue ] thread ",
+    .function = set_dpdk_if_placement,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
+			     vlib_cli_command_t * cmd)
+{
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_and_queue_t *dq;
+  int cpu;
+
+  if (tm->n_vlib_mains == 1)
+    vlib_cli_output (vm, "All interfaces are handled by main thread");
+
+  for (cpu = 0; cpu < vec_len (dm->devices_by_hqos_cpu); cpu++)
+    {
+      if (cpu >= dm->hqos_cpu_first_index &&
+	  cpu < (dm->hqos_cpu_first_index + dm->hqos_cpu_count))
+	vlib_cli_output (vm, "Thread %u (%s at lcore %u):", cpu,
+			 vlib_worker_threads[cpu].name,
+			 vlib_worker_threads[cpu].lcore_id);
+
+      vec_foreach (dq, dm->devices_by_hqos_cpu[cpu])
+      {
+	u32 hw_if_index = dm->devices[dq->device].vlib_hw_if_index;
+	vnet_hw_interface_t *hi =
+	  vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+	vlib_cli_output (vm, "  %v queue %u", hi->name, dq->queue_id);
+      }
+    }
+  return 0;
+}
+
+/*?
+ * This command is used to display the thread and core each
+ * DPDK output interface and HQoS queue is assigned too.
+ *
+ * @cliexpar
+ * Example of how to display the DPDK output interface and HQoS queue placement:
+ * @cliexstart{show dpdk interface hqos placement}
+ * Thread 1 (vpp_hqos-threads_0 at lcore 3):
+ *   GigabitEthernet0/8/0 queue 0
+ * Thread 2 (vpp_hqos-threads_1 at lcore 4):
+ *   GigabitEthernet0/9/0 queue 0
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_show_dpdk_if_hqos_placement, static) = {
+  .path = "show dpdk interface hqos placement",
+  .short_help = "show dpdk interface hqos placement",
+  .function = show_dpdk_if_hqos_placement,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_and_queue_t *dq;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 cpu = (u32) ~ 0;
+  int i;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "thread %d", &cpu))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    return clib_error_return (0, "please specify valid interface name");
+
+  if (cpu < dm->hqos_cpu_first_index ||
+      cpu >= (dm->hqos_cpu_first_index + dm->hqos_cpu_count))
+    {
+      error = clib_error_return (0, "please specify valid thread id");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  for (i = 0; i < vec_len (dm->devices_by_hqos_cpu); i++)
+    {
+      vec_foreach (dq, dm->devices_by_hqos_cpu[i])
+      {
+	if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index)
+	  {
+	    if (cpu == i)	/* nothing to do */
+	      goto done;
+
+	    vec_del1 (dm->devices_by_hqos_cpu[i],
+		      dq - dm->devices_by_hqos_cpu[i]);
+	    vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
+	    dq->queue_id = 0;
+	    dq->device = xd->device_index;
+
+	    vec_sort_with_function (dm->devices_by_hqos_cpu[i],
+				    dpdk_device_queue_sort);
+
+	    vec_sort_with_function (dm->devices_by_hqos_cpu[cpu],
+				    dpdk_device_queue_sort);
+
+	    goto done;
+	  }
+      }
+    }
+
+  error = clib_error_return (0, "not found");
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to assign a given DPDK output interface and
+ * HQoS queue to a different thread. This will not create a thread,
+ * so the thread must already exist. Use '/etc/vpp/startup.conf'
+ * for the initial thread creation. See @ref qos_doc for more details.
+ *
+ * @cliexpar
+ * Example of how to display the DPDK output interface and HQoS queue placement:
+ * @cliexstart{show dpdk interface hqos placement}
+ * Thread 1 (vpp_hqos-threads_0 at lcore 3):
+ *   GigabitEthernet0/8/0 queue 0
+ * Thread 2 (vpp_hqos-threads_1 at lcore 4):
+ *   GigabitEthernet0/9/0 queue 0
+ * @cliexend
+ * Example of how to assign a DPDK output interface and HQoS queue to a thread:
+ * @cliexcmd{set dpdk interface hqos placement GigabitEthernet0/8/0 thread 2}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_placement, static) = {
+  .path = "set dpdk interface hqos placement",
+  .short_help = "set dpdk interface hqos placement  thread ",
+  .function = set_dpdk_if_hqos_placement,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input,
+		       vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 subport_id = (u32) ~ 0;
+  u32 pipe_id = (u32) ~ 0;
+  u32 profile_id = (u32) ~ 0;
+  int rv;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "subport %d", &subport_id))
+	;
+      else if (unformat (line_input, "pipe %d", &pipe_id))
+	;
+      else if (unformat (line_input, "profile %d", &profile_id))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rv =
+    rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
+			   profile_id);
+  if (rv)
+    {
+      error = clib_error_return (0, "pipe configuration failed");
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to change the profile associate with a HQoS pipe. The
+ * '' is zero based. Use the command
+ * 'show dpdk interface hqos' to display the content of each profile.
+ * See @ref qos_doc for more details.
+ *
+ * @note
+ * Currently there is not an API to create a new HQoS pipe profile. One is
+ * created by default in the code (search for 'hqos_pipe_params_default'').
+ * Additional profiles can be created in code and code recompiled. Then use this
+ * command to assign it.
+ *
+ * @cliexpar
+ * Example of how to assign a new profile to a HQoS pipe:
+ * @cliexcmd{set dpdk interface hqos pipe GigabitEthernet0/8/0 subport 0 pipe 2 profile 1}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_pipe, static) =
+{
+  .path = "set dpdk interface hqos pipe",
+  .short_help = "set dpdk interface hqos pipe  subport  pipe  "
+                  "profile ",
+  .function = set_dpdk_if_hqos_pipe,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input,
+			  vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = NULL;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 subport_id = (u32) ~ 0;
+  struct rte_sched_subport_params p;
+  int rv;
+  clib_error_t *error = NULL;
+  u32 tb_rate = (u32) ~ 0;
+  u32 tb_size = (u32) ~ 0;
+  u32 tc_rate[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] =
+    { (u32) ~ 0, (u32) ~ 0, (u32) ~ 0, (u32) ~ 0 };
+  u32 tc_period = (u32) ~ 0;
+  dpdk_device_config_t *devconf = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "subport %d", &subport_id))
+	;
+      else if (unformat (line_input, "rate %d", &tb_rate))
+	;
+      else if (unformat (line_input, "bktsize %d", &tb_size))
+	;
+      else if (unformat (line_input, "tc0 %d", &tc_rate[0]))
+	;
+      else if (unformat (line_input, "tc1 %d", &tc_rate[1]))
+	;
+      else if (unformat (line_input, "tc2 %d", &tc_rate[2]))
+	;
+      else if (unformat (line_input, "tc3 %d", &tc_rate[3]))
+	;
+      else if (unformat (line_input, "period %d", &tc_period))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  error = get_hqos (hw_if_index, subport_id, &xd, &devconf);
+
+  if (error == NULL)
+    {
+      /* Copy the current values over to local structure. */
+      memcpy (&p, &devconf->hqos.subport[subport_id], sizeof (p));
+
+      /* Update local structure with input values. */
+      if (tb_rate != (u32) ~ 0)
+	{
+	  p.tb_rate = tb_rate;
+	  p.tc_rate[0] = tb_rate;
+	  p.tc_rate[1] = tb_rate;
+	  p.tc_rate[2] = tb_rate;
+	  p.tc_rate[3] = tb_rate;
+	}
+      if (tb_size != (u32) ~ 0)
+	{
+	  p.tb_size = tb_size;
+	}
+      if (tc_rate[0] != (u32) ~ 0)
+	{
+	  p.tc_rate[0] = tc_rate[0];
+	}
+      if (tc_rate[1] != (u32) ~ 0)
+	{
+	  p.tc_rate[1] = tc_rate[1];
+	}
+      if (tc_rate[2] != (u32) ~ 0)
+	{
+	  p.tc_rate[2] = tc_rate[2];
+	}
+      if (tc_rate[3] != (u32) ~ 0)
+	{
+	  p.tc_rate[3] = tc_rate[3];
+	}
+      if (tc_period != (u32) ~ 0)
+	{
+	  p.tc_period = tc_period;
+	}
+
+      /* Apply changes. */
+      rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, &p);
+      if (rv)
+	{
+	  error = clib_error_return (0, "subport configuration failed");
+	  goto done;
+	}
+      else
+	{
+	  /* Successfully applied, so save of the input values. */
+	  memcpy (&devconf->hqos.subport[subport_id], &p, sizeof (p));
+	}
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to set the subport level parameters such as token
+ * bucket rate (bytes per seconds), token bucket size (bytes), traffic class
+ * rates (bytes per seconds) and token update period (Milliseconds).
+ *
+ * By default, the 'rate' is set to 1250000000 bytes/second (10GbE
+ * rate) and each of the four traffic classes is set to 100% of the port rate.
+ * If the 'rate' is updated by this command, all four traffic classes
+ * are assigned the same value. Each of the four traffic classes can be updated
+ * individually.
+ *
+ * @cliexpar
+ * Example of how modify the subport attributes for a 1GbE link:
+ * @cliexcmd{set dpdk interface hqos subport GigabitEthernet0/8/0 subport 0 rate 125000000}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_subport, static) = {
+  .path = "set dpdk interface hqos subport",
+  .short_help = "set dpdk interface hqos subport  subport  "
+                 "[rate ] [bktsize ] [tc0 ] [tc1 ] [tc2 ] [tc3 ] "
+                 "[period ]",
+  .function = set_dpdk_if_hqos_subport,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input,
+			vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 tc = (u32) ~ 0;
+  u32 queue = (u32) ~ 0;
+  u32 entry = (u32) ~ 0;
+  u32 val, i;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "entry %d", &entry))
+	;
+      else if (unformat (line_input, "tc %d", &tc))
+	;
+      else if (unformat (line_input, "queue %d", &queue))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
+  if (entry >= 64)
+    {
+      error = clib_error_return (0, "invalid entry");
+      goto done;
+    }
+  if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
+    {
+      error = clib_error_return (0, "invalid traffic class");
+      goto done;
+    }
+  if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS)
+    {
+      error = clib_error_return (0, "invalid traffic class queue");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  /* Detect the set of worker threads */
+  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  /* Should never happen, shut up Coverity warning */
+  if (p == 0)
+    {
+      error = clib_error_return (0, "no worker registrations?");
+      goto done;
+    }
+
+  vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
+  int worker_thread_first = tr->first_index;
+  int worker_thread_count = tr->count;
+
+  val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue;
+  for (i = 0; i < worker_thread_count; i++)
+    xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val;
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to set the traffic class translation table. The
+ * traffic class translation table is used to map 64 values (0-63) to one of
+ * four traffic class and one of four HQoS input queue. Use the 'show
+ * dpdk interface hqos' command to display the traffic class translation
+ * table. See @ref qos_doc for more details.
+ *
+ * This command has the following parameters:
+ *
+ * -  - Used to specify the output interface.
+ *
+ * - entry  - Mapped value (0-63) to assign traffic class and queue to.
+ *
+ * - tc  - Traffic class (0-3) to be used by the provided mapped value.
+ *
+ * - queue  - HQoS input queue (0-3) to be used by the provided mapped value.
+ *
+ * @cliexpar
+ * Example of how modify the traffic class translation table:
+ * @cliexcmd{set dpdk interface hqos tctbl GigabitEthernet0/8/0 entry 16 tc 2 queue 2}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_tctbl, static) = {
+  .path = "set dpdk interface hqos tctbl",
+  .short_help = "set dpdk interface hqos tctbl  entry  tc  queue ",
+  .function = set_dpdk_if_hqos_tctbl,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
+			   vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_main_t *dm = &dpdk_main;
+  clib_error_t *error = NULL;
+
+  /* Device specific data */
+  struct rte_eth_dev_info dev_info;
+  dpdk_device_config_t *devconf = 0;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  u32 hw_if_index = (u32) ~ 0;
+
+  /* Detect the set of worker threads */
+  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  /* Should never happen, shut up Coverity warning */
+  if (p == 0)
+    return clib_error_return (0, "no worker registrations?");
+
+  vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
+  int worker_thread_first = tr->first_index;
+  int worker_thread_count = tr->count;
+
+  /* Packet field configuration */
+  u64 mask = (u64) ~ 0;
+  u32 id = (u32) ~ 0;
+  u32 offset = (u32) ~ 0;
+
+  /* HQoS params */
+  u32 n_subports_per_port, n_pipes_per_subport, tctbl_size;
+
+  u32 i;
+
+  /* Parse input arguments */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else if (unformat (line_input, "id subport"))
+	id = 0;
+      else if (unformat (line_input, "id pipe"))
+	id = 1;
+      else if (unformat (line_input, "id tc"))
+	id = 2;
+      else if (unformat (line_input, "id %d", &id))
+	;
+      else if (unformat (line_input, "offset %d", &offset))
+	;
+      else if (unformat (line_input, "mask %llx", &mask))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  /* Get interface */
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rte_eth_dev_info_get (xd->device_index, &dev_info);
+  if (dev_info.pci_dev)
+    {				/* bonded interface has no pci info */
+      vlib_pci_addr_t pci_addr;
+
+      pci_addr.domain = dev_info.pci_dev->addr.domain;
+      pci_addr.bus = dev_info.pci_dev->addr.bus;
+      pci_addr.slot = dev_info.pci_dev->addr.devid;
+      pci_addr.function = dev_info.pci_dev->addr.function;
+
+      p =
+	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
+    }
+
+  if (p)
+    devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
+  else
+    devconf = &dm->conf->default_devconf;
+
+  if (devconf->hqos_enabled == 0)
+    {
+      vlib_cli_output (vm, "HQoS disabled for this interface");
+      goto done;
+    }
+
+  n_subports_per_port = devconf->hqos.port.n_subports_per_port;
+  n_pipes_per_subport = devconf->hqos.port.n_pipes_per_subport;
+  tctbl_size = RTE_DIM (devconf->hqos.tc_table);
+
+  /* Validate packet field configuration: id, offset and mask */
+  if (id >= 3)
+    {
+      error = clib_error_return (0, "invalid packet field id");
+      goto done;
+    }
+
+  switch (id)
+    {
+    case 0:
+      if (dpdk_hqos_validate_mask (mask, n_subports_per_port) != 0)
+	{
+	  error = clib_error_return (0, "invalid subport ID mask "
+				     "(n_subports_per_port = %u)",
+				     n_subports_per_port);
+	  goto done;
+	}
+      break;
+    case 1:
+      if (dpdk_hqos_validate_mask (mask, n_pipes_per_subport) != 0)
+	{
+	  error = clib_error_return (0, "invalid pipe ID mask "
+				     "(n_pipes_per_subport = %u)",
+				     n_pipes_per_subport);
+	  goto done;
+	}
+      break;
+    case 2:
+    default:
+      if (dpdk_hqos_validate_mask (mask, tctbl_size) != 0)
+	{
+	  error = clib_error_return (0, "invalid TC table index mask "
+				     "(TC table size = %u)", tctbl_size);
+	  goto done;
+	}
+    }
+
+  /* Propagate packet field configuration to all workers */
+  for (i = 0; i < worker_thread_count; i++)
+    switch (id)
+      {
+      case 0:
+	xd->hqos_wt[worker_thread_first + i].hqos_field0_slabpos = offset;
+	xd->hqos_wt[worker_thread_first + i].hqos_field0_slabmask = mask;
+	xd->hqos_wt[worker_thread_first + i].hqos_field0_slabshr =
+	  __builtin_ctzll (mask);
+	break;
+      case 1:
+	xd->hqos_wt[worker_thread_first + i].hqos_field1_slabpos = offset;
+	xd->hqos_wt[worker_thread_first + i].hqos_field1_slabmask = mask;
+	xd->hqos_wt[worker_thread_first + i].hqos_field1_slabshr =
+	  __builtin_ctzll (mask);
+	break;
+      case 2:
+      default:
+	xd->hqos_wt[worker_thread_first + i].hqos_field2_slabpos = offset;
+	xd->hqos_wt[worker_thread_first + i].hqos_field2_slabmask = mask;
+	xd->hqos_wt[worker_thread_first + i].hqos_field2_slabshr =
+	  __builtin_ctzll (mask);
+      }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to set the packet fields required for classifiying the
+ * incoming packet. As a result of classification process, packet field
+ * information will be mapped to 5 tuples (subport, pipe, traffic class, pipe,
+ * color) and stored in packet mbuf.
+ *
+ * This command has the following parameters:
+ *
+ * -  - Used to specify the output interface.
+ *
+ * - id subport|pipe|tc - Classification occurs across three fields.
+ * This parameter indicates which of the three masks are being configured. Legacy
+ * code used 0-2 to represent these three fields, so 0-2 is still accepted.
+ *   - subport|0 - Currently only one subport is supported, so only
+ * an empty mask is supported for the subport classification.
+ *   - pipe|1 - Currently, 4096 pipes per subport are supported, so a
+ * 12-bit mask should be configure to map to the 0-4095 pipes.
+ *   - tc|2 - The translation table (see 'set dpdk interface hqos
+ * tctbl' command) maps each value (0-63) into one of the 4 traffic classes
+ * per pipe. A 6-bit mask should be configure to map this field to a traffic class.
+ *
+ * - offset  - Offset in the packet to apply the 64-bit mask for classification.
+ * The offset should be on an 8-byte boundary (0,8,16,24..).
+ *
+ * - mask  - 64-bit mask to apply to packet at the given 'offset'.
+ * Bits must be contiguous and should not include '0x'.
+ *
+ * The default values for the 'pktfield' assumes Ethernet/IPv4/UDP packets with
+ * no VLAN. Adjust based on expected packet format and desired classification field.
+ * - 'subport' is always empty (offset 0 mask 0000000000000000)
+ * - By default, 'pipe' maps to the UDP payload bits 12 .. 23 (offset 40
+ * mask 0000000fff000000)
+ * - By default, 'tc' maps to the DSCP field in IP header (offset 48 mask
+ * 00000000000000fc)
+ *
+ * @cliexpar
+ * Example of how modify the 'pipe' classification filter to match VLAN:
+ * @cliexcmd{set dpdk interface hqos pktfield GigabitEthernet0/8/0 id pipe offset 8 mask 0000000000000FFF}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_pktfield, static) = {
+  .path = "set dpdk interface hqos pktfield",
+  .short_help = "set dpdk interface hqos pktfield  id subport|pipe|tc offset  "
+                 "mask ",
+  .function = set_dpdk_if_hqos_pktfield,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
+		   vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  dpdk_device_config_hqos_t *cfg;
+  dpdk_device_hqos_per_hqos_thread_t *ht;
+  dpdk_device_hqos_per_worker_thread_t *wk;
+  u32 *tctbl;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 profile_id, subport_id, i;
+  struct rte_eth_dev_info dev_info;
+  dpdk_device_config_t *devconf = 0;
+  vlib_thread_registration_t *tr;
+  uword *p = 0;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify interface name!!");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rte_eth_dev_info_get (xd->device_index, &dev_info);
+  if (dev_info.pci_dev)
+    {				/* bonded interface has no pci info */
+      vlib_pci_addr_t pci_addr;
+
+      pci_addr.domain = dev_info.pci_dev->addr.domain;
+      pci_addr.bus = dev_info.pci_dev->addr.bus;
+      pci_addr.slot = dev_info.pci_dev->addr.devid;
+      pci_addr.function = dev_info.pci_dev->addr.function;
+
+      p =
+	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
+    }
+
+  if (p)
+    devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
+  else
+    devconf = &dm->conf->default_devconf;
+
+  if (devconf->hqos_enabled == 0)
+    {
+      vlib_cli_output (vm, "HQoS disabled for this interface");
+      goto done;
+    }
+
+  /* Detect the set of worker threads */
+  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+
+  /* Should never happen, shut up Coverity warning */
+  if (p == 0)
+    {
+      error = clib_error_return (0, "no worker registrations?");
+      goto done;
+    }
+
+  tr = (vlib_thread_registration_t *) p[0];
+
+  cfg = &devconf->hqos;
+  ht = xd->hqos_ht;
+  wk = &xd->hqos_wt[tr->first_index];
+  tctbl = wk->hqos_tc_table;
+
+  vlib_cli_output (vm, " Thread:");
+  vlib_cli_output (vm, "   Input SWQ size = %u packets", cfg->swq_size);
+  vlib_cli_output (vm, "   Enqueue burst size = %u packets",
+		   ht->hqos_burst_enq);
+  vlib_cli_output (vm, "   Dequeue burst size = %u packets",
+		   ht->hqos_burst_deq);
+
+  vlib_cli_output (vm,
+		   "   Packet field 0: slab position = %4u, slab bitmask = 0x%016llx   (subport)",
+		   wk->hqos_field0_slabpos, wk->hqos_field0_slabmask);
+  vlib_cli_output (vm,
+		   "   Packet field 1: slab position = %4u, slab bitmask = 0x%016llx   (pipe)",
+		   wk->hqos_field1_slabpos, wk->hqos_field1_slabmask);
+  vlib_cli_output (vm,
+		   "   Packet field 2: slab position = %4u, slab bitmask = 0x%016llx   (tc)",
+		   wk->hqos_field2_slabpos, wk->hqos_field2_slabmask);
+  vlib_cli_output (vm,
+		   "   Packet field 2  tc translation table: ([Mapped Value Range]: tc/queue tc/queue ...)");
+  vlib_cli_output (vm,
+		   "     [ 0 .. 15]: "
+		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
+		   tctbl[0] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[0] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[1] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[1] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[2] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[2] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[3] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[3] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[4] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[4] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[5] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[5] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[6] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[6] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[7] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[7] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[8] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[8] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[9] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[9] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[10] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[10] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[11] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[11] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[12] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[12] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[13] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[13] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[14] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[14] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[15] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[15] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
+  vlib_cli_output (vm,
+		   "     [16 .. 31]: "
+		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
+		   tctbl[16] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[16] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[17] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[17] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[18] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[18] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[19] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[19] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[20] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[20] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[21] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[21] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[22] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[22] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[23] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[23] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[24] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[24] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[25] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[25] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[26] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[26] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[27] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[27] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[28] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[28] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[29] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[29] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[30] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[30] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[31] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[31] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
+  vlib_cli_output (vm,
+		   "     [32 .. 47]: "
+		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
+		   tctbl[32] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[32] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[33] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[33] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[34] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[34] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[35] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[35] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[36] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[36] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[37] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[37] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[38] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[38] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[39] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[39] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[40] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[40] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[41] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[41] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[42] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[42] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[43] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[43] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[44] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[44] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[45] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[45] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[46] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[46] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[47] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[47] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
+  vlib_cli_output (vm,
+		   "     [48 .. 63]: "
+		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
+		   tctbl[48] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[48] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[49] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[49] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[50] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[50] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[51] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[51] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[52] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[52] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[53] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[53] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[54] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[54] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[55] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[55] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[56] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[56] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[57] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[57] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[58] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[58] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[59] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[59] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[60] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[60] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[61] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[61] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[62] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[62] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[63] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
+		   tctbl[63] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
+  vlib_cli_output (vm, " Port:");
+  vlib_cli_output (vm, "   Rate = %u bytes/second", cfg->port.rate);
+  vlib_cli_output (vm, "   MTU = %u bytes", cfg->port.mtu);
+  vlib_cli_output (vm, "   Frame overhead = %u bytes",
+		   cfg->port.frame_overhead);
+  vlib_cli_output (vm, "   Number of subports = %u",
+		   cfg->port.n_subports_per_port);
+  vlib_cli_output (vm, "   Number of pipes per subport = %u",
+		   cfg->port.n_pipes_per_subport);
+  vlib_cli_output (vm,
+		   "   Packet queue size: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u packets",
+		   cfg->port.qsize[0], cfg->port.qsize[1], cfg->port.qsize[2],
+		   cfg->port.qsize[3]);
+  vlib_cli_output (vm, "   Number of pipe profiles = %u",
+		   cfg->port.n_pipe_profiles);
+
+  for (subport_id = 0; subport_id < vec_len (cfg->subport); subport_id++)
+    {
+      vlib_cli_output (vm, " Subport %u:", subport_id);
+      vlib_cli_output (vm, "   Rate = %u bytes/second",
+		       cfg->subport[subport_id].tb_rate);
+      vlib_cli_output (vm, "   Token bucket size = %u bytes",
+		       cfg->subport[subport_id].tb_size);
+      vlib_cli_output (vm,
+		       "   Traffic class rate: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u bytes/second",
+		       cfg->subport[subport_id].tc_rate[0],
+		       cfg->subport[subport_id].tc_rate[1],
+		       cfg->subport[subport_id].tc_rate[2],
+		       cfg->subport[subport_id].tc_rate[3]);
+      vlib_cli_output (vm, "   TC period = %u milliseconds",
+		       cfg->subport[subport_id].tc_period);
+    }
+
+  for (profile_id = 0; profile_id < vec_len (cfg->pipe); profile_id++)
+    {
+      vlib_cli_output (vm, " Pipe profile %u:", profile_id);
+      vlib_cli_output (vm, "   Rate = %u bytes/second",
+		       cfg->pipe[profile_id].tb_rate);
+      vlib_cli_output (vm, "   Token bucket size = %u bytes",
+		       cfg->pipe[profile_id].tb_size);
+      vlib_cli_output (vm,
+		       "   Traffic class rate: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u bytes/second",
+		       cfg->pipe[profile_id].tc_rate[0],
+		       cfg->pipe[profile_id].tc_rate[1],
+		       cfg->pipe[profile_id].tc_rate[2],
+		       cfg->pipe[profile_id].tc_rate[3]);
+      vlib_cli_output (vm, "   TC period = %u milliseconds",
+		       cfg->pipe[profile_id].tc_period);
+#ifdef RTE_SCHED_SUBPORT_TC_OV
+      vlib_cli_output (vm, "   TC3 oversubscription_weight = %u",
+		       cfg->pipe[profile_id].tc_ov_weight);
+#endif
+
+      for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
+	{
+	  vlib_cli_output (vm,
+			   "   TC%u WRR weights: Q0 = %u, Q1 = %u, Q2 = %u, Q3 = %u",
+			   i, cfg->pipe[profile_id].wrr_weights[i * 4],
+			   cfg->pipe[profile_id].wrr_weights[i * 4 + 1],
+			   cfg->pipe[profile_id].wrr_weights[i * 4 + 2],
+			   cfg->pipe[profile_id].wrr_weights[i * 4 + 3]);
+	}
+    }
+
+#ifdef RTE_SCHED_RED
+  vlib_cli_output (vm, " Weighted Random Early Detection (WRED):");
+  for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
+    {
+      vlib_cli_output (vm, "   TC%u min: G = %u, Y = %u, R = %u", i,
+		       cfg->port.red_params[i][e_RTE_METER_GREEN].min_th,
+		       cfg->port.red_params[i][e_RTE_METER_YELLOW].min_th,
+		       cfg->port.red_params[i][e_RTE_METER_RED].min_th);
+
+      vlib_cli_output (vm, "   TC%u max: G = %u, Y = %u, R = %u", i,
+		       cfg->port.red_params[i][e_RTE_METER_GREEN].max_th,
+		       cfg->port.red_params[i][e_RTE_METER_YELLOW].max_th,
+		       cfg->port.red_params[i][e_RTE_METER_RED].max_th);
+
+      vlib_cli_output (vm,
+		       "   TC%u inverted probability: G = %u, Y = %u, R = %u",
+		       i, cfg->port.red_params[i][e_RTE_METER_GREEN].maxp_inv,
+		       cfg->port.red_params[i][e_RTE_METER_YELLOW].maxp_inv,
+		       cfg->port.red_params[i][e_RTE_METER_RED].maxp_inv);
+
+      vlib_cli_output (vm, "   TC%u weight: R = %u, Y = %u, R = %u", i,
+		       cfg->port.red_params[i][e_RTE_METER_GREEN].wq_log2,
+		       cfg->port.red_params[i][e_RTE_METER_YELLOW].wq_log2,
+		       cfg->port.red_params[i][e_RTE_METER_RED].wq_log2);
+    }
+#endif
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to display details of an output interface's HQoS
+ * settings.
+ *
+ * @cliexpar
+ * Example of how to display HQoS settings for an interfaces:
+ * @cliexstart{show dpdk interface hqos GigabitEthernet0/8/0}
+ *  Thread:
+ *    Input SWQ size = 4096 packets
+ *    Enqueue burst size = 256 packets
+ *    Dequeue burst size = 220 packets
+ *    Packet field 0: slab position =    0, slab bitmask = 0x0000000000000000   (subport)
+ *    Packet field 1: slab position =   40, slab bitmask = 0x0000000fff000000   (pipe)
+ *    Packet field 2: slab position =    8, slab bitmask = 0x00000000000000fc   (tc)
+ *    Packet field 2  tc translation table: ([Mapped Value Range]: tc/queue tc/queue ...)
+ *      [ 0 .. 15]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+ *      [16 .. 31]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+ *      [32 .. 47]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+ *      [48 .. 63]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+ *  Port:
+ *    Rate = 1250000000 bytes/second
+ *    MTU = 1514 bytes
+ *    Frame overhead = 24 bytes
+ *    Number of subports = 1
+ *    Number of pipes per subport = 4096
+ *    Packet queue size: TC0 = 64, TC1 = 64, TC2 = 64, TC3 = 64 packets
+ *    Number of pipe profiles = 2
+ *  Subport 0:
+ *    Rate = 1250000000 bytes/second
+ *    Token bucket size = 1000000 bytes
+ *    Traffic class rate: TC0 = 1250000000, TC1 = 1250000000, TC2 = 1250000000, TC3 = 1250000000 bytes/second
+ *    TC period = 10 milliseconds
+ *  Pipe profile 0:
+ *    Rate = 305175 bytes/second
+ *    Token bucket size = 1000000 bytes
+ *    Traffic class rate: TC0 = 305175, TC1 = 305175, TC2 = 305175, TC3 = 305175 bytes/second
+ *    TC period = 40 milliseconds
+ *    TC0 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+ *    TC1 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+ *    TC2 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+ *    TC3 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_show_dpdk_if_hqos, static) = {
+  .path = "show dpdk interface hqos",
+  .short_help = "show dpdk interface hqos ",
+  .function = show_dpdk_if_hqos,
+};
+
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = NULL;
+#ifdef RTE_SCHED_COLLECT_STATS
+  dpdk_main_t *dm = &dpdk_main;
+  u32 hw_if_index = (u32) ~ 0;
+  u32 subport = (u32) ~ 0;
+  u32 pipe = (u32) ~ 0;
+  u32 tc = (u32) ~ 0;
+  u32 tc_q = (u32) ~ 0;
+  vnet_hw_interface_t *hw;
+  dpdk_device_t *xd;
+  uword *p = 0;
+  struct rte_eth_dev_info dev_info;
+  dpdk_device_config_t *devconf = 0;
+  u32 qindex;
+  struct rte_sched_queue_stats stats;
+  u16 qlen;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
+	   &hw_if_index))
+	;
+
+      else if (unformat (line_input, "subport %d", &subport))
+	;
+
+      else if (unformat (line_input, "pipe %d", &pipe))
+	;
+
+      else if (unformat (line_input, "tc %d", &tc))
+	;
+
+      else if (unformat (line_input, "tc_q %d", &tc_q))
+	;
+
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (hw_if_index == (u32) ~ 0)
+    {
+      error = clib_error_return (0, "please specify interface name!!");
+      goto done;
+    }
+
+  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rte_eth_dev_info_get (xd->device_index, &dev_info);
+  if (dev_info.pci_dev)
+    {				/* bonded interface has no pci info */
+      vlib_pci_addr_t pci_addr;
+
+      pci_addr.domain = dev_info.pci_dev->addr.domain;
+      pci_addr.bus = dev_info.pci_dev->addr.bus;
+      pci_addr.slot = dev_info.pci_dev->addr.devid;
+      pci_addr.function = dev_info.pci_dev->addr.function;
+
+      p =
+	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
+    }
+
+  if (p)
+    devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
+  else
+    devconf = &dm->conf->default_devconf;
+
+  if (devconf->hqos_enabled == 0)
+    {
+      vlib_cli_output (vm, "HQoS disabled for this interface");
+      goto done;
+    }
+
+  /*
+   * Figure out which queue to query.  cf rte_sched_port_qindex.  (Not sure why
+   * that method isn't made public by DPDK - how _should_ we get the queue ID?)
+   */
+  qindex = subport * devconf->hqos.port.n_pipes_per_subport + pipe;
+  qindex = qindex * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE + tc;
+  qindex = qindex * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + tc_q;
+
+  if (rte_sched_queue_read_stats (xd->hqos_ht->hqos, qindex, &stats, &qlen) !=
+      0)
+    {
+      error = clib_error_return (0, "failed to read stats");
+      goto done;
+    }
+
+  vlib_cli_output (vm, "%=24s%=16s", "Stats Parameter", "Value");
+  vlib_cli_output (vm, "%=24s%=16d", "Packets", stats.n_pkts);
+  vlib_cli_output (vm, "%=24s%=16d", "Packets dropped", stats.n_pkts_dropped);
+#ifdef RTE_SCHED_RED
+  vlib_cli_output (vm, "%=24s%=16d", "Packets dropped (RED)",
+		   stats.n_pkts_red_dropped);
+#endif
+  vlib_cli_output (vm, "%=24s%=16d", "Bytes", stats.n_bytes);
+  vlib_cli_output (vm, "%=24s%=16d", "Bytes dropped", stats.n_bytes_dropped);
+
+#else
+
+  /* Get a line of input */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  vlib_cli_output (vm, "RTE_SCHED_COLLECT_STATS disabled in DPDK");
+  goto done;
+
+#endif
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * This command is used to display statistics associated with a HQoS traffic class
+ * queue.
+ *
+ * @note
+ * Statistic collection by the scheduler is disabled by default in DPDK. In order to
+ * turn it on, add the following line to '../vpp/dpdk/Makefile':
+ * - $(call set,RTE_SCHED_COLLECT_STATS,y)
+ *
+ * @cliexpar
+ * Example of how to display statistics of HQoS a HQoS traffic class queue:
+ * @cliexstart{show dpdk hqos queue GigabitEthernet0/9/0 subport 0 pipe 3181 tc 0 tc_q 0}
+ *      Stats Parameter          Value
+ *          Packets               140
+ *      Packets dropped            0
+ *           Bytes               8400
+ *       Bytes dropped             0
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cmd_show_dpdk_hqos_queue_stats, static) = {
+  .path = "show dpdk hqos queue",
+  .short_help = "show dpdk hqos queue  subport  pipe  tc  tc_q ",
+  .function = show_dpdk_hqos_queue_stats,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_dpdk_version_command_fn (vlib_main_t * vm,
+			      unformat_input_t * input,
+			      vlib_cli_command_t * cmd)
+{
+#define _(a,b,c) vlib_cli_output (vm, "%-25s " b, a ":", c);
+  _("DPDK Version", "%s", rte_version ());
+  _("DPDK EAL init args", "%s", dpdk_config_main.eal_init_args_str);
+#undef _
+  return 0;
+}
+
+/*?
+ * This command is used to display the current DPDK version and
+ * the list of arguments passed to DPDK when started.
+ *
+ * @cliexpar
+ * Example of how to display how many DPDK buffer test command has allcoated:
+ * @cliexstart{show dpdk version}
+ * DPDK Version:        DPDK 16.11.0
+ * DPDK EAL init args:  -c 1 -n 4 --huge-dir /run/vpp/hugepages --file-prefix vpp -w 0000:00:08.0 -w 0000:00:09.0 --master-lcore 0 --socket-mem 256
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_vpe_version_command, static) = {
+  .path = "show dpdk version",
+  .short_help = "show dpdk version",
+  .function = show_dpdk_version_command_fn,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+dpdk_cli_init (vlib_main_t * vm)
+{
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (dpdk_cli_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
new file mode 100644
index 00000000..50b26689
--- /dev/null
+++ b/src/plugins/dpdk/device/device.c
@@ -0,0 +1,852 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#define foreach_dpdk_tx_func_error			\
+  _(BAD_RETVAL, "DPDK tx function returned an error")	\
+  _(RING_FULL, "Tx packet drops (ring full)")	        \
+  _(PKT_DROP, "Tx packet drops (dpdk tx failure)")	\
+  _(REPL_FAIL, "Tx packet drops (replication failure)")
+
+typedef enum
+{
+#define _(f,s) DPDK_TX_FUNC_ERROR_##f,
+  foreach_dpdk_tx_func_error
+#undef _
+    DPDK_TX_FUNC_N_ERROR,
+} dpdk_tx_func_error_t;
+
+static char *dpdk_tx_func_error_strings[] = {
+#define _(n,s) s,
+  foreach_dpdk_tx_func_error
+#undef _
+};
+
+clib_error_t *
+dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address)
+{
+  int error;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
+
+  error = rte_eth_dev_default_mac_addr_set (xd->device_index,
+					    (struct ether_addr *) address);
+
+  if (error)
+    {
+      return clib_error_return (0, "mac address set failed: %d", error);
+    }
+  else
+    {
+      vec_reset_length (xd->default_mac_address);
+      vec_add (xd->default_mac_address, address, sizeof (address));
+      return NULL;
+    }
+}
+
+clib_error_t *
+dpdk_set_mc_filter (vnet_hw_interface_t * hi,
+		    struct ether_addr mc_addr_vec[], int naddr)
+{
+  int error;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
+
+  error = rte_eth_dev_set_mc_addr_list (xd->device_index, mc_addr_vec, naddr);
+
+  if (error)
+    {
+      return clib_error_return (0, "mc addr list failed: %d", error);
+    }
+  else
+    {
+      return NULL;
+    }
+}
+
+struct rte_mbuf *
+dpdk_replicate_packet_mb (vlib_buffer_t * b)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  struct rte_mbuf **mbufs = 0, *s, *d;
+  u8 nb_segs;
+  unsigned socket_id = rte_socket_id ();
+  int i;
+
+  ASSERT (dm->pktmbuf_pools[socket_id]);
+  s = rte_mbuf_from_vlib_buffer (b);
+  nb_segs = s->nb_segs;
+  vec_validate (mbufs, nb_segs - 1);
+
+  if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
+    {
+      vec_free (mbufs);
+      return 0;
+    }
+
+  d = mbufs[0];
+  d->nb_segs = s->nb_segs;
+  d->data_len = s->data_len;
+  d->pkt_len = s->pkt_len;
+  d->data_off = s->data_off;
+  clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
+
+  for (i = 1; i < nb_segs; i++)
+    {
+      d->next = mbufs[i];
+      d = mbufs[i];
+      s = s->next;
+      d->data_len = s->data_len;
+      clib_memcpy (d->buf_addr, s->buf_addr,
+		   RTE_PKTMBUF_HEADROOM + s->data_len);
+    }
+
+  d = mbufs[0];
+  vec_free (mbufs);
+  return d;
+}
+
+static void
+dpdk_tx_trace_buffer (dpdk_main_t * dm,
+		      vlib_node_runtime_t * node,
+		      dpdk_device_t * xd,
+		      u16 queue_id, u32 buffer_index, vlib_buffer_t * buffer)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  dpdk_tx_dma_trace_t *t0;
+  struct rte_mbuf *mb;
+
+  mb = rte_mbuf_from_vlib_buffer (buffer);
+
+  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
+  t0->queue_index = queue_id;
+  t0->device_index = xd->device_index;
+  t0->buffer_index = buffer_index;
+  clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
+  clib_memcpy (&t0->buffer, buffer,
+	       sizeof (buffer[0]) - sizeof (buffer->pre_data));
+  clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data,
+	       sizeof (t0->buffer.pre_data));
+}
+
+static_always_inline void
+dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b,
+			int maybe_multiseg)
+{
+  struct rte_mbuf *mb, *first_mb, *last_mb;
+
+  /* buffer is coming from non-dpdk source so we need to init
+     rte_mbuf header */
+  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
+    {
+      vlib_buffer_t *b2 = b;
+      last_mb = mb = rte_mbuf_from_vlib_buffer (b2);
+      rte_pktmbuf_reset (mb);
+      while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT))
+	{
+	  b2 = vlib_get_buffer (vm, b2->next_buffer);
+	  mb = rte_mbuf_from_vlib_buffer (b2);
+	  rte_pktmbuf_reset (mb);
+	}
+    }
+
+  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
+  first_mb->nb_segs = 1;
+  mb->data_len = b->current_length;
+  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
+    b->current_length;
+  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
+
+  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
+    {
+      b = vlib_get_buffer (vm, b->next_buffer);
+      mb = rte_mbuf_from_vlib_buffer (b);
+      last_mb->next = mb;
+      last_mb = mb;
+      mb->data_len = b->current_length;
+      mb->pkt_len = b->current_length;
+      mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
+      first_mb->nb_segs++;
+      if (PREDICT_FALSE (b->n_add_refs))
+	{
+	  rte_mbuf_refcnt_update (mb, b->n_add_refs);
+	  b->n_add_refs = 0;
+	}
+    }
+}
+
+/*
+ * This function calls the dpdk's tx_burst function to transmit the packets
+ * on the tx_vector. It manages a lock per-device if the device does not
+ * support multiple queues. It returns the number of packets untransmitted
+ * on the tx_vector. If all packets are transmitted (the normal case), the
+ * function returns 0.
+ *
+ * The function assumes there is at least one packet on the tx_vector.
+ */
+static_always_inline
+  u32 tx_burst_vector_internal (vlib_main_t * vm,
+				dpdk_device_t * xd,
+				struct rte_mbuf **tx_vector)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  u32 n_packets;
+  u32 tx_head;
+  u32 tx_tail;
+  u32 n_retry;
+  int rv;
+  int queue_id;
+  tx_ring_hdr_t *ring;
+
+  ring = vec_header (tx_vector, sizeof (*ring));
+
+  n_packets = ring->tx_head - ring->tx_tail;
+
+  tx_head = ring->tx_head % xd->nb_tx_desc;
+
+  /*
+   * Ensure rte_eth_tx_burst is not called with 0 packets, which can lead to
+   * unpredictable results.
+   */
+  ASSERT (n_packets > 0);
+
+  /*
+   * Check for tx_vector overflow. If this fails it is a system configuration
+   * error. The ring should be sized big enough to handle the largest un-flowed
+   * off burst from a traffic manager. A larger size also helps performance
+   * a bit because it decreases the probability of having to issue two tx_burst
+   * calls due to a ring wrap.
+   */
+  ASSERT (n_packets < xd->nb_tx_desc);
+  ASSERT (ring->tx_tail == 0);
+
+  n_retry = 16;
+  queue_id = vm->cpu_index;
+
+  do
+    {
+      /* start the burst at the tail */
+      tx_tail = ring->tx_tail % xd->nb_tx_desc;
+
+      /*
+       * This device only supports one TX queue,
+       * and we're running multi-threaded...
+       */
+      if (PREDICT_FALSE (xd->lockp != 0))
+	{
+	  queue_id = queue_id % xd->tx_q_used;
+	  while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
+	    /* zzzz */
+	    queue_id = (queue_id + 1) % xd->tx_q_used;
+	}
+
+      if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS))	/* HQoS ON */
+	{
+	  /* no wrap, transmit in one burst */
+	  dpdk_device_hqos_per_worker_thread_t *hqos =
+	    &xd->hqos_wt[vm->cpu_index];
+
+	  ASSERT (hqos->swq != NULL);
+
+	  dpdk_hqos_metadata_set (hqos,
+				  &tx_vector[tx_tail], tx_head - tx_tail);
+	  rv = rte_ring_sp_enqueue_burst (hqos->swq,
+					  (void **) &tx_vector[tx_tail],
+					  (uint16_t) (tx_head - tx_tail));
+	}
+      else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
+	{
+	  /* no wrap, transmit in one burst */
+	  rv = rte_eth_tx_burst (xd->device_index,
+				 (uint16_t) queue_id,
+				 &tx_vector[tx_tail],
+				 (uint16_t) (tx_head - tx_tail));
+	}
+      else
+	{
+	  ASSERT (0);
+	  rv = 0;
+	}
+
+      if (PREDICT_FALSE (xd->lockp != 0))
+	*xd->lockp[queue_id] = 0;
+
+      if (PREDICT_FALSE (rv < 0))
+	{
+	  // emit non-fatal message, bump counter
+	  vnet_main_t *vnm = dm->vnet_main;
+	  vnet_interface_main_t *im = &vnm->interface_main;
+	  u32 node_index;
+
+	  node_index = vec_elt_at_index (im->hw_interfaces,
+					 xd->vlib_hw_if_index)->tx_node_index;
+
+	  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
+	  clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index,
+			rv);
+	  return n_packets;	// untransmitted packets
+	}
+      ring->tx_tail += (u16) rv;
+      n_packets -= (uint16_t) rv;
+    }
+  while (rv && n_packets && (n_retry > 0));
+
+  return n_packets;
+}
+
+static_always_inline void
+dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
+{
+  vlib_buffer_t *b;
+  struct rte_mbuf *mb;
+  b = vlib_get_buffer (vm, bi);
+  mb = rte_mbuf_from_vlib_buffer (b);
+  CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, LOAD);
+  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+}
+
+static_always_inline void
+dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node,
+		     vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  u32 my_cpu = vm->cpu_index;
+  struct rte_mbuf *mb_new;
+
+  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
+    return;
+
+  mb_new = dpdk_replicate_packet_mb (b);
+  if (PREDICT_FALSE (mb_new == 0))
+    {
+      vlib_error_count (vm, node->node_index,
+			DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
+      b->flags |= VLIB_BUFFER_REPL_FAIL;
+    }
+  else
+    *mbp = mb_new;
+
+  vec_add1 (dm->recycle[my_cpu], bi);
+}
+
+/*
+ * Transmits the packets on the frame to the interface associated with the
+ * node. It first copies packets on the frame to a tx_vector containing the
+ * rte_mbuf pointers. It then passes this vector to tx_burst_vector_internal
+ * which calls the dpdk tx_burst function.
+ */
+static uword
+dpdk_interface_tx (vlib_main_t * vm,
+		   vlib_node_runtime_t * node, vlib_frame_t * f)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, rd->dev_instance);
+  u32 n_packets = f->n_vectors;
+  u32 n_left;
+  u32 *from;
+  struct rte_mbuf **tx_vector;
+  u16 i;
+  u16 nb_tx_desc = xd->nb_tx_desc;
+  int queue_id;
+  u32 my_cpu;
+  u32 tx_pkts = 0;
+  tx_ring_hdr_t *ring;
+  u32 n_on_ring;
+
+  my_cpu = vm->cpu_index;
+
+  queue_id = my_cpu;
+
+  tx_vector = xd->tx_vectors[queue_id];
+  ring = vec_header (tx_vector, sizeof (*ring));
+
+  n_on_ring = ring->tx_head - ring->tx_tail;
+  from = vlib_frame_vector_args (f);
+
+  ASSERT (n_packets <= VLIB_FRAME_SIZE);
+
+  if (PREDICT_FALSE (n_on_ring + n_packets > nb_tx_desc))
+    {
+      /*
+       * Overflowing the ring should never happen.
+       * If it does then drop the whole frame.
+       */
+      vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_RING_FULL,
+			n_packets);
+
+      while (n_packets--)
+	{
+	  u32 bi0 = from[n_packets];
+	  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
+	  struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer (b0);
+	  rte_pktmbuf_free (mb0);
+	}
+      return n_on_ring;
+    }
+
+  if (PREDICT_FALSE (dm->tx_pcap_enable))
+    {
+      n_left = n_packets;
+      while (n_left > 0)
+	{
+	  u32 bi0 = from[0];
+	  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
+	  if (dm->pcap_sw_if_index == 0 ||
+	      dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX])
+	    pcap_add_buffer (&dm->pcap_main, vm, bi0, 512);
+	  from++;
+	  n_left--;
+	}
+    }
+
+  from = vlib_frame_vector_args (f);
+  n_left = n_packets;
+  i = ring->tx_head % nb_tx_desc;
+
+  while (n_left >= 8)
+    {
+      u32 bi0, bi1, bi2, bi3;
+      struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
+      vlib_buffer_t *b0, *b1, *b2, *b3;
+      u32 or_flags;
+
+      dpdk_prefetch_buffer_by_index (vm, from[4]);
+      dpdk_prefetch_buffer_by_index (vm, from[5]);
+      dpdk_prefetch_buffer_by_index (vm, from[6]);
+      dpdk_prefetch_buffer_by_index (vm, from[7]);
+
+      bi0 = from[0];
+      bi1 = from[1];
+      bi2 = from[2];
+      bi3 = from[3];
+      from += 4;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      b1 = vlib_get_buffer (vm, bi1);
+      b2 = vlib_get_buffer (vm, bi2);
+      b3 = vlib_get_buffer (vm, bi3);
+
+      or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
+
+      if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
+	{
+	  dpdk_validate_rte_mbuf (vm, b0, 1);
+	  dpdk_validate_rte_mbuf (vm, b1, 1);
+	  dpdk_validate_rte_mbuf (vm, b2, 1);
+	  dpdk_validate_rte_mbuf (vm, b3, 1);
+	}
+      else
+	{
+	  dpdk_validate_rte_mbuf (vm, b0, 0);
+	  dpdk_validate_rte_mbuf (vm, b1, 0);
+	  dpdk_validate_rte_mbuf (vm, b2, 0);
+	  dpdk_validate_rte_mbuf (vm, b3, 0);
+	}
+
+      mb0 = rte_mbuf_from_vlib_buffer (b0);
+      mb1 = rte_mbuf_from_vlib_buffer (b1);
+      mb2 = rte_mbuf_from_vlib_buffer (b2);
+      mb3 = rte_mbuf_from_vlib_buffer (b3);
+
+      if (PREDICT_FALSE (or_flags & VLIB_BUFFER_RECYCLE))
+	{
+	  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
+	  dpdk_buffer_recycle (vm, node, b1, bi1, &mb1);
+	  dpdk_buffer_recycle (vm, node, b2, bi2, &mb2);
+	  dpdk_buffer_recycle (vm, node, b3, bi3, &mb3);
+
+	  /* dont enqueue packets if replication failed as they must
+	     be sent back to recycle */
+	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
+	    tx_vector[i++ % nb_tx_desc] = mb0;
+	  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0))
+	    tx_vector[i++ % nb_tx_desc] = mb1;
+	  if (PREDICT_TRUE ((b2->flags & VLIB_BUFFER_REPL_FAIL) == 0))
+	    tx_vector[i++ % nb_tx_desc] = mb2;
+	  if (PREDICT_TRUE ((b3->flags & VLIB_BUFFER_REPL_FAIL) == 0))
+	    tx_vector[i++ % nb_tx_desc] = mb3;
+	}
+      else
+	{
+	  if (PREDICT_FALSE (i + 3 >= nb_tx_desc))
+	    {
+	      tx_vector[i++ % nb_tx_desc] = mb0;
+	      tx_vector[i++ % nb_tx_desc] = mb1;
+	      tx_vector[i++ % nb_tx_desc] = mb2;
+	      tx_vector[i++ % nb_tx_desc] = mb3;
+	      i %= nb_tx_desc;
+	    }
+	  else
+	    {
+	      tx_vector[i++] = mb0;
+	      tx_vector[i++] = mb1;
+	      tx_vector[i++] = mb2;
+	      tx_vector[i++] = mb3;
+	    }
+	}
+
+
+      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
+	{
+	  if (b0->flags & VLIB_BUFFER_IS_TRACED)
+	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
+	  if (b1->flags & VLIB_BUFFER_IS_TRACED)
+	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1);
+	  if (b2->flags & VLIB_BUFFER_IS_TRACED)
+	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi2, b2);
+	  if (b3->flags & VLIB_BUFFER_IS_TRACED)
+	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi3, b3);
+	}
+
+      n_left -= 4;
+    }
+  while (n_left > 0)
+    {
+      u32 bi0;
+      struct rte_mbuf *mb0;
+      vlib_buffer_t *b0;
+
+      bi0 = from[0];
+      from++;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      dpdk_validate_rte_mbuf (vm, b0, 1);
+
+      mb0 = rte_mbuf_from_vlib_buffer (b0);
+      dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
+
+      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
+	if (b0->flags & VLIB_BUFFER_IS_TRACED)
+	  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
+
+      if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
+	{
+	  tx_vector[i % nb_tx_desc] = mb0;
+	  i++;
+	}
+      n_left--;
+    }
+
+  /* account for additional packets in the ring */
+  ring->tx_head += n_packets;
+  n_on_ring = ring->tx_head - ring->tx_tail;
+
+  /* transmit as many packets as possible */
+  n_packets = tx_burst_vector_internal (vm, xd, tx_vector);
+
+  /*
+   * tx_pkts is the number of packets successfully transmitted
+   * This is the number originally on ring minus the number remaining on ring
+   */
+  tx_pkts = n_on_ring - n_packets;
+
+  {
+    /* If there is no callback then drop any non-transmitted packets */
+    if (PREDICT_FALSE (n_packets))
+      {
+	vlib_simple_counter_main_t *cm;
+	vnet_main_t *vnm = vnet_get_main ();
+
+	cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
+			       VNET_INTERFACE_COUNTER_TX_ERROR);
+
+	vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
+				       n_packets);
+
+	vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
+			  n_packets);
+
+	while (n_packets--)
+	  rte_pktmbuf_free (tx_vector[ring->tx_tail + n_packets]);
+      }
+
+    /* Reset head/tail to avoid unnecessary wrap */
+    ring->tx_head = 0;
+    ring->tx_tail = 0;
+  }
+
+  /* Recycle replicated buffers */
+  if (PREDICT_FALSE (vec_len (dm->recycle[my_cpu])))
+    {
+      vlib_buffer_free (vm, dm->recycle[my_cpu],
+			vec_len (dm->recycle[my_cpu]));
+      _vec_len (dm->recycle[my_cpu]) = 0;
+    }
+
+  ASSERT (ring->tx_head >= ring->tx_tail);
+
+  return tx_pkts;
+}
+
+static void
+dpdk_clear_hw_interface_counters (u32 instance)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
+
+  /*
+   * Set the "last_cleared_stats" to the current stats, so that
+   * things appear to clear from a display perspective.
+   */
+  dpdk_update_counters (xd, vlib_time_now (dm->vlib_main));
+
+  clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
+  clib_memcpy (xd->last_cleared_xstats, xd->xstats,
+	       vec_len (xd->last_cleared_xstats) *
+	       sizeof (xd->last_cleared_xstats[0]));
+
+}
+
+static clib_error_t *
+dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
+{
+  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
+  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hif->dev_instance);
+  int rv = 0;
+
+  if (is_up)
+    {
+      f64 now = vlib_time_now (dm->vlib_main);
+
+      if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
+	{
+	  rv = rte_eth_dev_start (xd->device_index);
+	  if (!rv && xd->default_mac_address)
+	    rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
+						   (struct ether_addr *)
+						   xd->default_mac_address);
+	}
+
+      if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
+	rte_eth_promiscuous_enable (xd->device_index);
+      else
+	rte_eth_promiscuous_disable (xd->device_index);
+
+      rte_eth_allmulticast_enable (xd->device_index);
+      xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
+      dpdk_update_counters (xd, now);
+      dpdk_update_link_state (xd, now);
+    }
+  else
+    {
+      xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
+
+      rte_eth_allmulticast_disable (xd->device_index);
+      vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, 0);
+      rte_eth_dev_stop (xd->device_index);
+
+      /* For bonded interface, stop slave links */
+      if (xd->pmd == VNET_DPDK_PMD_BOND)
+	{
+	  u8 slink[16];
+	  int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16);
+	  while (nlink >= 1)
+	    {
+	      u8 dpdk_port = slink[--nlink];
+	      rte_eth_dev_stop (dpdk_port);
+	    }
+	}
+    }
+
+  if (rv < 0)
+    clib_warning ("rte_eth_dev_%s error: %d", is_up ? "start" : "stop", rv);
+
+  return /* no error */ 0;
+}
+
+/*
+ * Dynamically redirect all pkts from a specific interface
+ * to the specified node
+ */
+static void
+dpdk_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
+			      u32 node_index)
+{
+  dpdk_main_t *xm = &dpdk_main;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
+
+  /* Shut off redirection */
+  if (node_index == ~0)
+    {
+      xd->per_interface_next_index = node_index;
+      return;
+    }
+
+  xd->per_interface_next_index =
+    vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
+}
+
+
+static clib_error_t *
+dpdk_subif_add_del_function (vnet_main_t * vnm,
+			     u32 hw_if_index,
+			     struct vnet_sw_interface_t *st, int is_add)
+{
+  dpdk_main_t *xm = &dpdk_main;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
+  vnet_sw_interface_t *t = (vnet_sw_interface_t *) st;
+  int r, vlan_offload;
+  u32 prev_subifs = xd->num_subifs;
+  clib_error_t *err = 0;
+
+  if (is_add)
+    xd->num_subifs++;
+  else if (xd->num_subifs)
+    xd->num_subifs--;
+
+  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
+    goto done;
+
+  /* currently we program VLANS only for IXGBE VF and I40E VF */
+  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
+    goto done;
+
+  if (t->sub.eth.flags.no_tags == 1)
+    goto done;
+
+  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
+    {
+      xd->num_subifs = prev_subifs;
+      err = clib_error_return (0, "unsupported VLAN setup");
+      goto done;
+    }
+
+  vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index);
+  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
+
+  if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload)))
+    {
+      xd->num_subifs = prev_subifs;
+      err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
+			       xd->device_index, r);
+      goto done;
+    }
+
+
+  if ((r =
+       rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id,
+				is_add)))
+    {
+      xd->num_subifs = prev_subifs;
+      err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
+			       xd->device_index, r);
+      goto done;
+    }
+
+done:
+  if (xd->num_subifs)
+    xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
+  else
+    xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
+
+  return err;
+}
+
+/* *INDENT-OFF* */
+VNET_DEVICE_CLASS (dpdk_device_class) = {
+  .name = "dpdk",
+  .tx_function = dpdk_interface_tx,
+  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
+  .tx_function_error_strings = dpdk_tx_func_error_strings,
+  .format_device_name = format_dpdk_device_name,
+  .format_device = format_dpdk_device,
+  .format_tx_trace = format_dpdk_tx_dma_trace,
+  .clear_counters = dpdk_clear_hw_interface_counters,
+  .admin_up_down_function = dpdk_interface_admin_up_down,
+  .subif_add_del_function = dpdk_subif_add_del_function,
+  .rx_redirect_to_node = dpdk_set_interface_next_node,
+  .mac_addr_change_function = dpdk_set_mac_address,
+};
+
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class, dpdk_interface_tx)
+/* *INDENT-ON* */
+
+#define UP_DOWN_FLAG_EVENT 1
+
+uword
+admin_up_down_process (vlib_main_t * vm,
+		       vlib_node_runtime_t * rt, vlib_frame_t * f)
+{
+  clib_error_t *error = 0;
+  uword event_type;
+  uword *event_data = 0;
+  u32 sw_if_index;
+  u32 flags;
+
+  while (1)
+    {
+      vlib_process_wait_for_event (vm);
+
+      event_type = vlib_process_get_events (vm, &event_data);
+
+      dpdk_main.admin_up_down_in_progress = 1;
+
+      switch (event_type)
+	{
+	case UP_DOWN_FLAG_EVENT:
+	  {
+	    if (vec_len (event_data) == 2)
+	      {
+		sw_if_index = event_data[0];
+		flags = event_data[1];
+		error =
+		  vnet_sw_interface_set_flags (vnet_get_main (), sw_if_index,
+					       flags);
+		clib_error_report (error);
+	      }
+	  }
+	  break;
+	}
+
+      vec_reset_length (event_data);
+
+      dpdk_main.admin_up_down_in_progress = 0;
+
+    }
+  return 0;			/* or not */
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (admin_up_down_process_node,static) = {
+    .function = admin_up_down_process,
+    .type = VLIB_NODE_TYPE_PROCESS,
+    .name = "admin-up-down-process",
+    .process_log2_n_stack_bytes = 17,  // 256KB
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
new file mode 100644
index 00000000..2a1a6205
--- /dev/null
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -0,0 +1,490 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_dpdk_h__
+#define __included_dpdk_h__
+
+/* $$$$ We should rename always_inline -> clib_always_inline */
+#undef always_inline
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#if CLIB_DEBUG > 0
+#define always_inline static inline
+#else
+#define always_inline static inline __attribute__ ((__always_inline__))
+#endif
+
+#include 
+
+#define NB_MBUF   (16<<10)
+
+extern vnet_device_class_t dpdk_device_class;
+extern vlib_node_registration_t dpdk_input_node;
+extern vlib_node_registration_t handoff_dispatch_node;
+
+#define foreach_dpdk_pmd          \
+  _ ("net_thunderx", THUNDERX)    \
+  _ ("net_e1000_em", E1000EM)     \
+  _ ("net_e1000_igb", IGB)        \
+  _ ("net_e1000_igb_vf", IGBVF)   \
+  _ ("net_ixgbe", IXGBE)          \
+  _ ("net_ixgbe_vf", IXGBEVF)     \
+  _ ("net_i40e", I40E)            \
+  _ ("net_i40e_vf", I40EVF)       \
+  _ ("net_virtio", VIRTIO)        \
+  _ ("net_enic", ENIC)            \
+  _ ("net_vmxnet3", VMXNET3)      \
+  _ ("AF_PACKET PMD", AF_PACKET)  \
+  _ ("rte_bond_pmd", BOND)        \
+  _ ("net_fm10k", FM10K)          \
+  _ ("net_cxgbe", CXGBE)          \
+  _ ("net_mlx5", MLX5)            \
+  _ ("net_dpaa2", DPAA2)
+
+typedef enum
+{
+  VNET_DPDK_PMD_NONE,
+#define _(s,f) VNET_DPDK_PMD_##f,
+  foreach_dpdk_pmd
+#undef _
+    VNET_DPDK_PMD_UNKNOWN,	/* must be last */
+} dpdk_pmd_t;
+
+typedef enum
+{
+  VNET_DPDK_PORT_TYPE_ETH_1G,
+  VNET_DPDK_PORT_TYPE_ETH_10G,
+  VNET_DPDK_PORT_TYPE_ETH_40G,
+  VNET_DPDK_PORT_TYPE_ETH_100G,
+  VNET_DPDK_PORT_TYPE_ETH_BOND,
+  VNET_DPDK_PORT_TYPE_ETH_SWITCH,
+  VNET_DPDK_PORT_TYPE_AF_PACKET,
+  VNET_DPDK_PORT_TYPE_UNKNOWN,
+} dpdk_port_type_t;
+
+/*
+ * The header for the tx_vector in dpdk_device_t.
+ * Head and tail are indexes into the tx_vector and are of type
+ * u64 so they never overflow.
+ */
+typedef struct
+{
+  u64 tx_head;
+  u64 tx_tail;
+} tx_ring_hdr_t;
+
+typedef struct
+{
+  struct rte_ring *swq;
+
+  u64 hqos_field0_slabmask;
+  u32 hqos_field0_slabpos;
+  u32 hqos_field0_slabshr;
+  u64 hqos_field1_slabmask;
+  u32 hqos_field1_slabpos;
+  u32 hqos_field1_slabshr;
+  u64 hqos_field2_slabmask;
+  u32 hqos_field2_slabpos;
+  u32 hqos_field2_slabshr;
+  u32 hqos_tc_table[64];
+} dpdk_device_hqos_per_worker_thread_t;
+
+typedef struct
+{
+  struct rte_ring **swq;
+  struct rte_mbuf **pkts_enq;
+  struct rte_mbuf **pkts_deq;
+  struct rte_sched_port *hqos;
+  u32 hqos_burst_enq;
+  u32 hqos_burst_deq;
+  u32 pkts_enq_len;
+  u32 swq_pos;
+  u32 flush_count;
+} dpdk_device_hqos_per_hqos_thread_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  volatile u32 **lockp;
+
+  /* Instance ID */
+  u32 device_index;
+
+  u32 vlib_hw_if_index;
+  u32 vlib_sw_if_index;
+
+  /* next node index if we decide to steal the rx graph arc */
+  u32 per_interface_next_index;
+
+  /* dpdk rte_mbuf rx and tx vectors, VLIB_FRAME_SIZE */
+  struct rte_mbuf ***tx_vectors;	/* one per worker thread */
+  struct rte_mbuf ***rx_vectors;
+
+  /* vector of traced contexts, per device */
+  u32 **d_trace_buffers;
+
+  dpdk_pmd_t pmd:8;
+  i8 cpu_socket;
+
+  u16 flags;
+#define DPDK_DEVICE_FLAG_ADMIN_UP           (1 << 0)
+#define DPDK_DEVICE_FLAG_PROMISC            (1 << 1)
+#define DPDK_DEVICE_FLAG_PMD                (1 << 2)
+#define DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE (1 << 3)
+#define DPDK_DEVICE_FLAG_MAYBE_MULTISEG     (1 << 4)
+#define DPDK_DEVICE_FLAG_HAVE_SUBIF         (1 << 5)
+#define DPDK_DEVICE_FLAG_HQOS               (1 << 6)
+
+  u16 nb_tx_desc;
+    CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
+
+  u8 *interface_name_suffix;
+
+  /* number of sub-interfaces */
+  u16 num_subifs;
+
+  /* PMD related */
+  u16 tx_q_used;
+  u16 rx_q_used;
+  u16 nb_rx_desc;
+  u16 *cpu_socket_id_by_queue;
+  struct rte_eth_conf port_conf;
+  struct rte_eth_txconf tx_conf;
+
+  /* HQoS related */
+  dpdk_device_hqos_per_worker_thread_t *hqos_wt;
+  dpdk_device_hqos_per_hqos_thread_t *hqos_ht;
+
+  /* af_packet */
+  u8 af_packet_port_id;
+
+  struct rte_eth_link link;
+  f64 time_last_link_update;
+
+  struct rte_eth_stats stats;
+  struct rte_eth_stats last_stats;
+  struct rte_eth_stats last_cleared_stats;
+  struct rte_eth_xstat *xstats;
+  struct rte_eth_xstat *last_cleared_xstats;
+  f64 time_last_stats_update;
+  dpdk_port_type_t port_type;
+
+  /* mac address */
+  u8 *default_mac_address;
+} dpdk_device_t;
+
+#define DPDK_STATS_POLL_INTERVAL      (10.0)
+#define DPDK_MIN_STATS_POLL_INTERVAL  (0.001)	/* 1msec */
+
+#define DPDK_LINK_POLL_INTERVAL       (3.0)
+#define DPDK_MIN_LINK_POLL_INTERVAL   (0.001)	/* 1msec */
+
+typedef struct
+{
+  u32 device;
+  u16 queue_id;
+} dpdk_device_and_queue_t;
+
+#ifndef DPDK_HQOS_DBG_BYPASS
+#define DPDK_HQOS_DBG_BYPASS 0
+#endif
+
+#ifndef HQOS_FLUSH_COUNT_THRESHOLD
+#define HQOS_FLUSH_COUNT_THRESHOLD              100000
+#endif
+
+typedef struct dpdk_device_config_hqos_t
+{
+  u32 hqos_thread;
+  u32 hqos_thread_valid;
+
+  u32 swq_size;
+  u32 burst_enq;
+  u32 burst_deq;
+
+  u32 pktfield0_slabpos;
+  u32 pktfield1_slabpos;
+  u32 pktfield2_slabpos;
+  u64 pktfield0_slabmask;
+  u64 pktfield1_slabmask;
+  u64 pktfield2_slabmask;
+  u32 tc_table[64];
+
+  struct rte_sched_port_params port;
+  struct rte_sched_subport_params *subport;
+  struct rte_sched_pipe_params *pipe;
+  uint32_t *pipe_map;
+} dpdk_device_config_hqos_t;
+
+int dpdk_hqos_validate_mask (u64 mask, u32 n);
+void dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
+						   hqos, u32 pipe_profile_id);
+void dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos);
+clib_error_t *dpdk_port_setup_hqos (dpdk_device_t * xd,
+				    dpdk_device_config_hqos_t * hqos);
+void dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
+			     struct rte_mbuf **pkts, u32 n_pkts);
+
+#define foreach_dpdk_device_config_item \
+  _ (num_rx_queues) \
+  _ (num_tx_queues) \
+  _ (num_rx_desc) \
+  _ (num_tx_desc) \
+  _ (rss_fn)
+
+typedef struct
+{
+  vlib_pci_addr_t pci_addr;
+  u8 is_blacklisted;
+  u8 vlan_strip_offload;
+#define DPDK_DEVICE_VLAN_STRIP_DEFAULT 0
+#define DPDK_DEVICE_VLAN_STRIP_OFF 1
+#define DPDK_DEVICE_VLAN_STRIP_ON  2
+
+#define _(x) uword x;
+    foreach_dpdk_device_config_item
+#undef _
+    clib_bitmap_t * workers;
+  u32 hqos_enabled;
+  dpdk_device_config_hqos_t hqos;
+} dpdk_device_config_t;
+
+typedef struct
+{
+
+  /* Config stuff */
+  u8 **eal_init_args;
+  u8 *eal_init_args_str;
+  u8 *uio_driver_name;
+  u8 no_multi_seg;
+  u8 enable_tcp_udp_checksum;
+  u8 cryptodev;
+
+  /* Required config parameters */
+  u8 coremask_set_manually;
+  u8 nchannels_set_manually;
+  u32 coremask;
+  u32 nchannels;
+  u32 num_mbufs;
+  u8 num_kni;			/* while kni_init allows u32, port_id in callback fn is only u8 */
+
+  /*
+   * format interface names ala xxxEthernet%d/%d/%d instead of
+   * xxxEthernet%x/%x/%x.
+   */
+  u8 interface_name_format_decimal;
+
+  /* per-device config */
+  dpdk_device_config_t default_devconf;
+  dpdk_device_config_t *dev_confs;
+  uword *device_config_index_by_pci_addr;
+
+} dpdk_config_main_t;
+
+dpdk_config_main_t dpdk_config_main;
+
+typedef struct
+{
+
+  /* Devices */
+  dpdk_device_t *devices;
+  dpdk_device_and_queue_t **devices_by_cpu;
+  dpdk_device_and_queue_t **devices_by_hqos_cpu;
+
+  /* per-thread recycle lists */
+  u32 **recycle;
+
+  /* buffer flags template, configurable to enable/disable tcp / udp cksum */
+  u32 buffer_flags_template;
+
+  /* vlib buffer free list, must be same size as an rte_mbuf */
+  u32 vlib_buffer_free_list_index;
+
+  /* Ethernet input node index */
+  u32 ethernet_input_node_index;
+
+  /* pcap tracing [only works if (CLIB_DEBUG > 0)] */
+  int tx_pcap_enable;
+  pcap_main_t pcap_main;
+  u8 *pcap_filename;
+  u32 pcap_sw_if_index;
+  u32 pcap_pkts_to_capture;
+
+  /* hashes */
+  uword *dpdk_device_by_kni_port_id;
+  uword *vu_sw_if_index_by_listener_fd;
+  uword *vu_sw_if_index_by_sock_fd;
+  u32 *vu_inactive_interfaces_device_index;
+
+  /*
+   * flag indicating that a posted admin up/down
+   * (via post_sw_interface_set_flags) is in progress
+   */
+  u8 admin_up_down_in_progress;
+
+  u8 use_rss;
+
+  /* which cpus are running dpdk-input */
+  int input_cpu_first_index;
+  int input_cpu_count;
+
+  /* which cpus are running I/O TX */
+  int hqos_cpu_first_index;
+  int hqos_cpu_count;
+
+  /* control interval of dpdk link state and stat polling */
+  f64 link_state_poll_interval;
+  f64 stat_poll_interval;
+
+  /* Sleep for this many MS after each device poll */
+  u32 poll_sleep;
+
+  /* convenience */
+  vlib_main_t *vlib_main;
+  vnet_main_t *vnet_main;
+  dpdk_config_main_t *conf;
+
+  /* mempool */
+  struct rte_mempool **pktmbuf_pools;
+
+  /* API message ID base */
+  u16 msg_id_base;
+} dpdk_main_t;
+
+dpdk_main_t dpdk_main;
+
+typedef struct
+{
+  u32 buffer_index;
+  u16 device_index;
+  u8 queue_index;
+  struct rte_mbuf mb;
+  /* Copy of VLIB buffer; packet data stored in pre_data. */
+  vlib_buffer_t buffer;
+} dpdk_tx_dma_trace_t;
+
+typedef struct
+{
+  u32 buffer_index;
+  u16 device_index;
+  u16 queue_index;
+  struct rte_mbuf mb;
+  vlib_buffer_t buffer;		/* Copy of VLIB buffer; pkt data stored in pre_data. */
+  u8 data[256];			/* First 256 data bytes, used for hexdump */
+} dpdk_rx_dma_trace_t;
+
+void vnet_buffer_needs_dpdk_mb (vlib_buffer_t * b);
+
+clib_error_t *dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address);
+
+clib_error_t *dpdk_set_mc_filter (vnet_hw_interface_t * hi,
+				  struct ether_addr mc_addr_vec[], int naddr);
+
+void dpdk_thread_input (dpdk_main_t * dm, dpdk_device_t * xd);
+
+clib_error_t *dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd);
+
+u32 dpdk_interface_tx_vector (vlib_main_t * vm, u32 dev_instance);
+
+struct rte_mbuf *dpdk_replicate_packet_mb (vlib_buffer_t * b);
+struct rte_mbuf *dpdk_zerocopy_replicate_packet_mb (vlib_buffer_t * b);
+
+#define foreach_dpdk_error						\
+  _(NONE, "no error")							\
+  _(RX_PACKET_ERROR, "Rx packet errors")				\
+  _(RX_BAD_FCS, "Rx bad fcs")						\
+  _(IP_CHECKSUM_ERROR, "Rx ip checksum errors")				\
+  _(RX_ALLOC_FAIL, "rx buf alloc from free list failed")		\
+  _(RX_ALLOC_NO_PHYSMEM, "rx buf alloc failed no physmem")		\
+  _(RX_ALLOC_DROP_PKTS, "rx packets dropped due to alloc error")
+
+typedef enum
+{
+#define _(f,s) DPDK_ERROR_##f,
+  foreach_dpdk_error
+#undef _
+    DPDK_N_ERROR,
+} dpdk_error_t;
+
+int dpdk_set_stat_poll_interval (f64 interval);
+int dpdk_set_link_state_poll_interval (f64 interval);
+void dpdk_update_link_state (dpdk_device_t * xd, f64 now);
+void dpdk_device_lock_init (dpdk_device_t * xd);
+void dpdk_device_lock_free (dpdk_device_t * xd);
+
+void dpdk_rx_trace (dpdk_main_t * dm,
+		    vlib_node_runtime_t * node,
+		    dpdk_device_t * xd,
+		    u16 queue_id, u32 * buffers, uword n_buffers);
+
+#define EFD_OPERATION_LESS_THAN          0
+#define EFD_OPERATION_GREATER_OR_EQUAL   1
+
+format_function_t format_dpdk_device_name;
+format_function_t format_dpdk_device;
+format_function_t format_dpdk_tx_dma_trace;
+format_function_t format_dpdk_rx_dma_trace;
+format_function_t format_dpdk_rte_mbuf;
+format_function_t format_dpdk_rx_rte_mbuf;
+unformat_function_t unformat_socket_mem;
+clib_error_t *unformat_rss_fn (unformat_input_t * input, uword * rss_fn);
+clib_error_t *unformat_hqos (unformat_input_t * input,
+			     dpdk_device_config_hqos_t * hqos);
+
+uword
+admin_up_down_process (vlib_main_t * vm,
+		       vlib_node_runtime_t * rt, vlib_frame_t * f);
+
+#endif /* __included_dpdk_h__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h
new file mode 100644
index 00000000..dd40ff48
--- /dev/null
+++ b/src/plugins/dpdk/device/dpdk_priv.h
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1)
+#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1))
+
+#define DPDK_NB_RX_DESC_DEFAULT   1024
+#define DPDK_NB_TX_DESC_DEFAULT   1024
+#define DPDK_NB_RX_DESC_VIRTIO    256
+#define DPDK_NB_TX_DESC_VIRTIO    256
+
+#define I40E_DEV_ID_SFP_XL710           0x1572
+#define I40E_DEV_ID_QSFP_A              0x1583
+#define I40E_DEV_ID_QSFP_B              0x1584
+#define I40E_DEV_ID_QSFP_C              0x1585
+#define I40E_DEV_ID_10G_BASE_T          0x1586
+#define I40E_DEV_ID_VF                  0x154C
+
+/* These args appear by themselves */
+#define foreach_eal_double_hyphen_predicate_arg \
+_(no-shconf)                                    \
+_(no-hpet)                                      \
+_(no-huge)                                      \
+_(vmware-tsc-map)
+
+#define foreach_eal_single_hyphen_mandatory_arg \
+_(coremask, c)                                  \
+_(nchannels, n)                                 \
+
+#define foreach_eal_single_hyphen_arg           \
+_(blacklist, b)                                 \
+_(mem-alloc-request, m)                         \
+_(force-ranks, r)
+
+/* These args are preceeded by "--" and followed by a single string */
+#define foreach_eal_double_hyphen_arg           \
+_(huge-dir)                                     \
+_(proc-type)                                    \
+_(file-prefix)                                  \
+_(vdev)
+
+static inline void
+dpdk_get_xstats (dpdk_device_t * xd)
+{
+  int len;
+  if ((len = rte_eth_xstats_get (xd->device_index, NULL, 0)) > 0)
+    {
+      vec_validate (xd->xstats, len - 1);
+      vec_validate (xd->last_cleared_xstats, len - 1);
+
+      len =
+	rte_eth_xstats_get (xd->device_index, xd->xstats,
+			    vec_len (xd->xstats));
+
+      ASSERT (vec_len (xd->xstats) == len);
+      ASSERT (vec_len (xd->last_cleared_xstats) == len);
+
+      _vec_len (xd->xstats) = len;
+      _vec_len (xd->last_cleared_xstats) = len;
+
+    }
+}
+
+
+static inline void
+dpdk_update_counters (dpdk_device_t * xd, f64 now)
+{
+  vlib_simple_counter_main_t *cm;
+  vnet_main_t *vnm = vnet_get_main ();
+  u32 my_cpu = os_get_cpu_number ();
+  u64 rxerrors, last_rxerrors;
+
+  /* only update counters for PMD interfaces */
+  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
+    return;
+
+  xd->time_last_stats_update = now ? now : xd->time_last_stats_update;
+  clib_memcpy (&xd->last_stats, &xd->stats, sizeof (xd->last_stats));
+  rte_eth_stats_get (xd->device_index, &xd->stats);
+
+  /* maybe bump interface rx no buffer counter */
+  if (PREDICT_FALSE (xd->stats.rx_nombuf != xd->last_stats.rx_nombuf))
+    {
+      cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
+			     VNET_INTERFACE_COUNTER_RX_NO_BUF);
+
+      vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
+				     xd->stats.rx_nombuf -
+				     xd->last_stats.rx_nombuf);
+    }
+
+  /* missed pkt counter */
+  if (PREDICT_FALSE (xd->stats.imissed != xd->last_stats.imissed))
+    {
+      cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
+			     VNET_INTERFACE_COUNTER_RX_MISS);
+
+      vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
+				     xd->stats.imissed -
+				     xd->last_stats.imissed);
+    }
+  rxerrors = xd->stats.ierrors;
+  last_rxerrors = xd->last_stats.ierrors;
+
+  if (PREDICT_FALSE (rxerrors != last_rxerrors))
+    {
+      cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
+			     VNET_INTERFACE_COUNTER_RX_ERROR);
+
+      vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
+				     rxerrors - last_rxerrors);
+    }
+
+  dpdk_get_xstats (xd);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/format.c b/src/plugins/dpdk/device/format.c
new file mode 100644
index 00000000..25a8c5cb
--- /dev/null
+++ b/src/plugins/dpdk/device/format.c
@@ -0,0 +1,754 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#define foreach_dpdk_counter                    \
+  _ (tx_frames_ok, opackets)                    \
+  _ (tx_bytes_ok, obytes)                       \
+  _ (tx_errors, oerrors)                        \
+  _ (rx_frames_ok, ipackets)                    \
+  _ (rx_bytes_ok, ibytes)                       \
+  _ (rx_errors, ierrors)                        \
+  _ (rx_missed, imissed)                        \
+  _ (rx_no_bufs, rx_nombuf)
+
+#define foreach_dpdk_q_counter                  \
+  _ (rx_frames_ok, q_ipackets)                  \
+  _ (tx_frames_ok, q_opackets)                  \
+  _ (rx_bytes_ok, q_ibytes)                     \
+  _ (tx_bytes_ok, q_obytes)                     \
+  _ (rx_errors, q_errors)
+
+#define foreach_dpdk_rss_hf                    \
+  _(ETH_RSS_FRAG_IPV4,          "ipv4-frag")   \
+  _(ETH_RSS_NONFRAG_IPV4_TCP,   "ipv4-tcp")    \
+  _(ETH_RSS_NONFRAG_IPV4_UDP,   "ipv4-udp")    \
+  _(ETH_RSS_NONFRAG_IPV4_SCTP,  "ipv4-sctp")   \
+  _(ETH_RSS_NONFRAG_IPV4_OTHER, "ipv4-other")  \
+  _(ETH_RSS_IPV4,               "ipv4")        \
+  _(ETH_RSS_IPV6_TCP_EX,        "ipv6-tcp-ex") \
+  _(ETH_RSS_IPV6_UDP_EX,        "ipv6-udp-ex") \
+  _(ETH_RSS_FRAG_IPV6,          "ipv6-frag")   \
+  _(ETH_RSS_NONFRAG_IPV6_TCP,   "ipv6-tcp")    \
+  _(ETH_RSS_NONFRAG_IPV6_UDP,   "ipv6-udp")    \
+  _(ETH_RSS_NONFRAG_IPV6_SCTP,  "ipv6-sctp")   \
+  _(ETH_RSS_NONFRAG_IPV6_OTHER, "ipv6-other")  \
+  _(ETH_RSS_L2_PAYLOAD,         "l2-payload")  \
+  _(ETH_RSS_IPV6_EX,            "ipv6-ex")     \
+  _(ETH_RSS_IPV6,               "ipv6")
+
+
+#define foreach_dpdk_rx_offload_caps            \
+  _(DEV_RX_OFFLOAD_VLAN_STRIP, "vlan-strip")    \
+  _(DEV_RX_OFFLOAD_IPV4_CKSUM, "ipv4-cksum")    \
+  _(DEV_RX_OFFLOAD_UDP_CKSUM , "udp-cksum")     \
+  _(DEV_RX_OFFLOAD_TCP_CKSUM , "tcp-cksum")     \
+  _(DEV_RX_OFFLOAD_TCP_LRO   , "rcp-lro")       \
+  _(DEV_RX_OFFLOAD_QINQ_STRIP, "qinq-strip")
+
+#define foreach_dpdk_tx_offload_caps           \
+  _(DEV_TX_OFFLOAD_VLAN_INSERT, "vlan-insert") \
+  _(DEV_TX_OFFLOAD_IPV4_CKSUM,  "ipv4-cksum")  \
+  _(DEV_TX_OFFLOAD_UDP_CKSUM  , "udp-cksum")   \
+  _(DEV_TX_OFFLOAD_TCP_CKSUM  , "tcp-cksum")   \
+  _(DEV_TX_OFFLOAD_SCTP_CKSUM , "sctp-cksum")  \
+  _(DEV_TX_OFFLOAD_TCP_TSO    , "tcp-tso")     \
+  _(DEV_TX_OFFLOAD_UDP_TSO    , "udp-tso")     \
+  _(DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM, "outer-ipv4-cksum") \
+  _(DEV_TX_OFFLOAD_QINQ_INSERT, "qinq-insert")
+
+#define foreach_dpdk_pkt_rx_offload_flag                                \
+  _ (PKT_RX_VLAN_PKT, "RX packet is a 802.1q VLAN packet")              \
+  _ (PKT_RX_RSS_HASH, "RX packet with RSS hash result")                 \
+  _ (PKT_RX_FDIR, "RX packet with FDIR infos")                          \
+  _ (PKT_RX_L4_CKSUM_BAD, "L4 cksum of RX pkt. is not OK")              \
+  _ (PKT_RX_IP_CKSUM_BAD, "IP cksum of RX pkt. is not OK")              \
+  _ (PKT_RX_VLAN_STRIPPED, "RX packet VLAN tag stripped")               \
+  _ (PKT_RX_IP_CKSUM_GOOD, "IP cksum of RX pkt. is valid")              \
+  _ (PKT_RX_L4_CKSUM_GOOD, "L4 cksum of RX pkt. is valid")              \
+  _ (PKT_RX_IEEE1588_PTP, "RX IEEE1588 L2 Ethernet PT Packet")          \
+  _ (PKT_RX_IEEE1588_TMST, "RX IEEE1588 L2/L4 timestamped packet")      \
+  _ (PKT_RX_QINQ_STRIPPED, "RX packet QinQ tags stripped")
+
+#define foreach_dpdk_pkt_type                                           \
+  _ (L2, ETHER, "Ethernet packet")                                      \
+  _ (L2, ETHER_TIMESYNC, "Ethernet packet for time sync")               \
+  _ (L2, ETHER_ARP, "ARP packet")                                       \
+  _ (L2, ETHER_LLDP, "LLDP (Link Layer Discovery Protocol) packet")     \
+  _ (L2, ETHER_NSH, "NSH (Network Service Header) packet")              \
+  _ (L2, ETHER_VLAN, "VLAN packet")                                     \
+  _ (L2, ETHER_QINQ, "QinQ packet")                                     \
+  _ (L3, IPV4, "IPv4 packet without extension headers")                 \
+  _ (L3, IPV4_EXT, "IPv4 packet with extension headers")                \
+  _ (L3, IPV4_EXT_UNKNOWN, "IPv4 packet with or without extension headers") \
+  _ (L3, IPV6, "IPv6 packet without extension headers")                 \
+  _ (L3, IPV6_EXT, "IPv6 packet with extension headers")                \
+  _ (L3, IPV6_EXT_UNKNOWN, "IPv6 packet with or without extension headers") \
+  _ (L4, TCP, "TCP packet")                                             \
+  _ (L4, UDP, "UDP packet")                                             \
+  _ (L4, FRAG, "Fragmented IP packet")                                  \
+  _ (L4, SCTP, "SCTP (Stream Control Transmission Protocol) packet")    \
+  _ (L4, ICMP, "ICMP packet")                                           \
+  _ (L4, NONFRAG, "Non-fragmented IP packet")                           \
+  _ (TUNNEL, GRE, "GRE tunneling packet")                               \
+  _ (TUNNEL, VXLAN, "VXLAN tunneling packet")                           \
+  _ (TUNNEL, NVGRE, "NVGRE Tunneling packet")                           \
+  _ (TUNNEL, GENEVE, "GENEVE Tunneling packet")                         \
+  _ (TUNNEL, GRENAT, "Teredo, VXLAN or GRE Tunneling packet")           \
+  _ (INNER_L2, ETHER, "Inner Ethernet packet")                          \
+  _ (INNER_L2, ETHER_VLAN, "Inner Ethernet packet with VLAN")           \
+  _ (INNER_L3, IPV4, "Inner IPv4 packet without extension headers")     \
+  _ (INNER_L3, IPV4_EXT, "Inner IPv4 packet with extension headers")    \
+  _ (INNER_L3, IPV4_EXT_UNKNOWN, "Inner IPv4 packet with or without extension headers") \
+  _ (INNER_L3, IPV6, "Inner IPv6 packet without extension headers")     \
+  _ (INNER_L3, IPV6_EXT, "Inner IPv6 packet with extension headers")    \
+  _ (INNER_L3, IPV6_EXT_UNKNOWN, "Inner IPv6 packet with or without extension headers") \
+  _ (INNER_L4, TCP, "Inner TCP packet")                                 \
+  _ (INNER_L4, UDP, "Inner UDP packet")                                 \
+  _ (INNER_L4, FRAG, "Inner fagmented IP packet")                       \
+  _ (INNER_L4, SCTP, "Inner SCTP (Stream Control Transmission Protocol) packet") \
+  _ (INNER_L4, ICMP, "Inner ICMP packet")                               \
+  _ (INNER_L4, NONFRAG, "Inner non-fragmented IP packet")
+
+#define foreach_dpdk_pkt_tx_offload_flag                                \
+  _ (PKT_TX_VLAN_PKT, "TX packet is a 802.1q VLAN packet")              \
+  _ (PKT_TX_IP_CKSUM, "IP cksum of TX pkt. computed by NIC")            \
+  _ (PKT_TX_TCP_CKSUM, "TCP cksum of TX pkt. computed by NIC")          \
+  _ (PKT_TX_SCTP_CKSUM, "SCTP cksum of TX pkt. computed by NIC")        \
+  _ (PKT_TX_IEEE1588_TMST, "TX IEEE1588 packet to timestamp")
+
+#define foreach_dpdk_pkt_offload_flag           \
+  foreach_dpdk_pkt_rx_offload_flag              \
+  foreach_dpdk_pkt_tx_offload_flag
+
+u8 *
+format_dpdk_device_name (u8 * s, va_list * args)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  char *devname_format;
+  char *device_name;
+  u32 i = va_arg (*args, u32);
+  struct rte_eth_dev_info dev_info;
+  u8 *ret;
+
+  if (dm->conf->interface_name_format_decimal)
+    devname_format = "%s%d/%d/%d";
+  else
+    devname_format = "%s%x/%x/%x";
+
+  switch (dm->devices[i].port_type)
+    {
+    case VNET_DPDK_PORT_TYPE_ETH_1G:
+      device_name = "GigabitEthernet";
+      break;
+
+    case VNET_DPDK_PORT_TYPE_ETH_10G:
+      device_name = "TenGigabitEthernet";
+      break;
+
+    case VNET_DPDK_PORT_TYPE_ETH_40G:
+      device_name = "FortyGigabitEthernet";
+      break;
+
+    case VNET_DPDK_PORT_TYPE_ETH_100G:
+      device_name = "HundredGigabitEthernet";
+      break;
+
+    case VNET_DPDK_PORT_TYPE_ETH_BOND:
+      return format (s, "BondEthernet%d", dm->devices[i].device_index);
+
+    case VNET_DPDK_PORT_TYPE_ETH_SWITCH:
+      device_name = "EthernetSwitch";
+      break;
+
+    case VNET_DPDK_PORT_TYPE_AF_PACKET:
+      rte_eth_dev_info_get (i, &dev_info);
+      return format (s, "af_packet%d", dm->devices[i].af_packet_port_id);
+
+    default:
+    case VNET_DPDK_PORT_TYPE_UNKNOWN:
+      device_name = "UnknownEthernet";
+      break;
+    }
+
+  rte_eth_dev_info_get (i, &dev_info);
+
+  if (dev_info.pci_dev)
+    ret = format (s, devname_format, device_name, dev_info.pci_dev->addr.bus,
+		  dev_info.pci_dev->addr.devid,
+		  dev_info.pci_dev->addr.function);
+  else
+    ret = format (s, "%s%d", device_name, dm->devices[i].device_index);
+
+  if (dm->devices[i].interface_name_suffix)
+    return format (ret, "/%s", dm->devices[i].interface_name_suffix);
+  return ret;
+}
+
+static u8 *
+format_dpdk_device_type (u8 * s, va_list * args)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  char *dev_type;
+  u32 i = va_arg (*args, u32);
+
+  switch (dm->devices[i].pmd)
+    {
+    case VNET_DPDK_PMD_E1000EM:
+      dev_type = "Intel 82540EM (e1000)";
+      break;
+
+    case VNET_DPDK_PMD_IGB:
+      dev_type = "Intel e1000";
+      break;
+
+    case VNET_DPDK_PMD_I40E:
+      dev_type = "Intel X710/XL710 Family";
+      break;
+
+    case VNET_DPDK_PMD_I40EVF:
+      dev_type = "Intel X710/XL710 Family VF";
+      break;
+
+    case VNET_DPDK_PMD_FM10K:
+      dev_type = "Intel FM10000 Family Ethernet Switch";
+      break;
+
+    case VNET_DPDK_PMD_IGBVF:
+      dev_type = "Intel e1000 VF";
+      break;
+
+    case VNET_DPDK_PMD_VIRTIO:
+      dev_type = "Red Hat Virtio";
+      break;
+
+    case VNET_DPDK_PMD_IXGBEVF:
+      dev_type = "Intel 82599 VF";
+      break;
+
+    case VNET_DPDK_PMD_IXGBE:
+      dev_type = "Intel 82599";
+      break;
+
+    case VNET_DPDK_PMD_ENIC:
+      dev_type = "Cisco VIC";
+      break;
+
+    case VNET_DPDK_PMD_CXGBE:
+      dev_type = "Chelsio T4/T5";
+      break;
+
+    case VNET_DPDK_PMD_MLX5:
+      dev_type = "Mellanox ConnectX-4 Family";
+      break;
+
+    case VNET_DPDK_PMD_VMXNET3:
+      dev_type = "VMware VMXNET3";
+      break;
+
+    case VNET_DPDK_PMD_AF_PACKET:
+      dev_type = "af_packet";
+      break;
+
+    case VNET_DPDK_PMD_BOND:
+      dev_type = "Ethernet Bonding";
+      break;
+
+    case VNET_DPDK_PMD_DPAA2:
+      dev_type = "NXP DPAA2 Mac";
+      break;
+
+    default:
+    case VNET_DPDK_PMD_UNKNOWN:
+      dev_type = "### UNKNOWN ###";
+      break;
+    }
+
+  return format (s, dev_type);
+}
+
+static u8 *
+format_dpdk_link_status (u8 * s, va_list * args)
+{
+  dpdk_device_t *xd = va_arg (*args, dpdk_device_t *);
+  struct rte_eth_link *l = &xd->link;
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->vlib_hw_if_index);
+
+  s = format (s, "%s ", l->link_status ? "up" : "down");
+  if (l->link_status)
+    {
+      u32 promisc = rte_eth_promiscuous_get (xd->device_index);
+
+      s = format (s, "%s duplex ", (l->link_duplex == ETH_LINK_FULL_DUPLEX) ?
+		  "full" : "half");
+      s = format (s, "speed %u mtu %d %s\n", l->link_speed,
+		  hi->max_packet_bytes, promisc ? " promisc" : "");
+    }
+  else
+    s = format (s, "\n");
+
+  return s;
+}
+
+#define _line_len 72
+#define _(v, str)                                            \
+if (bitmap & v) {                                            \
+  if (format_get_indent (s) > next_split ) {                 \
+    next_split += _line_len;                                 \
+    s = format(s,"\n%U", format_white_space, indent);        \
+  }                                                          \
+  s = format(s, "%s ", str);                                 \
+}
+
+static u8 *
+format_dpdk_rss_hf_name (u8 * s, va_list * args)
+{
+  u64 bitmap = va_arg (*args, u64);
+  int next_split = _line_len;
+  int indent = format_get_indent (s);
+
+  if (!bitmap)
+    return format (s, "none");
+
+  foreach_dpdk_rss_hf return s;
+}
+
+static u8 *
+format_dpdk_rx_offload_caps (u8 * s, va_list * args)
+{
+  u32 bitmap = va_arg (*args, u32);
+  int next_split = _line_len;
+  int indent = format_get_indent (s);
+
+  if (!bitmap)
+    return format (s, "none");
+
+  foreach_dpdk_rx_offload_caps return s;
+}
+
+static u8 *
+format_dpdk_tx_offload_caps (u8 * s, va_list * args)
+{
+  u32 bitmap = va_arg (*args, u32);
+  int next_split = _line_len;
+  int indent = format_get_indent (s);
+  if (!bitmap)
+    return format (s, "none");
+
+  foreach_dpdk_tx_offload_caps return s;
+}
+
+#undef _line_len
+#undef _
+
+u8 *
+format_dpdk_device (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  int verbose = va_arg (*args, int);
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance);
+  uword indent = format_get_indent (s);
+  f64 now = vlib_time_now (dm->vlib_main);
+  struct rte_eth_dev_info di;
+
+  dpdk_update_counters (xd, now);
+  dpdk_update_link_state (xd, now);
+
+  s = format (s, "%U\n%Ucarrier %U",
+	      format_dpdk_device_type, xd->device_index,
+	      format_white_space, indent + 2, format_dpdk_link_status, xd);
+
+  rte_eth_dev_info_get (xd->device_index, &di);
+
+  if (verbose > 1 && xd->flags & DPDK_DEVICE_FLAG_PMD)
+    {
+      struct rte_pci_device *pci;
+      struct rte_eth_rss_conf rss_conf;
+      int vlan_off;
+      int retval;
+
+      rss_conf.rss_key = 0;
+      retval = rte_eth_dev_rss_hash_conf_get (xd->device_index, &rss_conf);
+      if (retval < 0)
+	clib_warning ("rte_eth_dev_rss_hash_conf_get returned %d", retval);
+      pci = di.pci_dev;
+
+      if (pci)
+	s =
+	  format (s,
+		  "%Upci id:            device %04x:%04x subsystem %04x:%04x\n"
+		  "%Upci address:       %04x:%02x:%02x.%02x\n",
+		  format_white_space, indent + 2, pci->id.vendor_id,
+		  pci->id.device_id, pci->id.subsystem_vendor_id,
+		  pci->id.subsystem_device_id, format_white_space, indent + 2,
+		  pci->addr.domain, pci->addr.bus, pci->addr.devid,
+		  pci->addr.function);
+      s =
+	format (s, "%Umax rx packet len: %d\n", format_white_space,
+		indent + 2, di.max_rx_pktlen);
+      s =
+	format (s, "%Umax num of queues: rx %d tx %d\n", format_white_space,
+		indent + 2, di.max_rx_queues, di.max_tx_queues);
+      s =
+	format (s, "%Upromiscuous:       unicast %s all-multicast %s\n",
+		format_white_space, indent + 2,
+		rte_eth_promiscuous_get (xd->device_index) ? "on" : "off",
+		rte_eth_promiscuous_get (xd->device_index) ? "on" : "off");
+      vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
+      s = format (s, "%Uvlan offload:      strip %s filter %s qinq %s\n",
+		  format_white_space, indent + 2,
+		  vlan_off & ETH_VLAN_STRIP_OFFLOAD ? "on" : "off",
+		  vlan_off & ETH_VLAN_FILTER_OFFLOAD ? "on" : "off",
+		  vlan_off & ETH_VLAN_EXTEND_OFFLOAD ? "on" : "off");
+      s = format (s, "%Urx offload caps:   %U\n",
+		  format_white_space, indent + 2,
+		  format_dpdk_rx_offload_caps, di.rx_offload_capa);
+      s = format (s, "%Utx offload caps:   %U\n",
+		  format_white_space, indent + 2,
+		  format_dpdk_tx_offload_caps, di.tx_offload_capa);
+      s = format (s, "%Urss active:        %U\n"
+		  "%Urss supported:     %U\n",
+		  format_white_space, indent + 2,
+		  format_dpdk_rss_hf_name, rss_conf.rss_hf,
+		  format_white_space, indent + 2,
+		  format_dpdk_rss_hf_name, di.flow_type_rss_offloads);
+    }
+
+  s = format (s, "%Urx queues %d, rx desc %d, tx queues %d, tx desc %d\n",
+	      format_white_space, indent + 2,
+	      xd->rx_q_used, xd->nb_rx_desc, xd->tx_q_used, xd->nb_tx_desc);
+
+  if (xd->cpu_socket > -1)
+    s = format (s, "%Ucpu socket %d\n",
+		format_white_space, indent + 2, xd->cpu_socket);
+
+  /* $$$ MIB counters  */
+  {
+#define _(N, V)							\
+    if ((xd->stats.V - xd->last_cleared_stats.V) != 0) {       \
+      s = format (s, "\n%U%-40U%16Ld",                         \
+                  format_white_space, indent + 2,              \
+                  format_c_identifier, #N,                     \
+                  xd->stats.V - xd->last_cleared_stats.V);     \
+    }                                                          \
+
+    foreach_dpdk_counter
+#undef _
+  }
+
+  u8 *xs = 0;
+  u32 i = 0;
+  struct rte_eth_xstat *xstat, *last_xstat;
+  struct rte_eth_xstat_name *xstat_names = 0;
+  int len = rte_eth_xstats_get_names (xd->device_index, NULL, 0);
+  vec_validate (xstat_names, len - 1);
+  rte_eth_xstats_get_names (xd->device_index, xstat_names, len);
+
+  ASSERT (vec_len (xd->xstats) == vec_len (xd->last_cleared_xstats));
+
+  /* *INDENT-OFF* */
+  vec_foreach_index(i, xd->xstats)
+    {
+      u64 delta = 0;
+      xstat = vec_elt_at_index(xd->xstats, i);
+      last_xstat = vec_elt_at_index(xd->last_cleared_xstats, i);
+
+      delta = xstat->value - last_xstat->value;
+      if (verbose == 2 || (verbose && delta))
+        {
+          /* format_c_identifier doesn't like c strings inside vector */
+          u8 * name = format(0,"%s", xstat_names[i].name);
+          xs = format(xs, "\n%U%-38U%16Ld",
+                      format_white_space, indent + 4,
+                      format_c_identifier, name, delta);
+          vec_free(name);
+        }
+    }
+  /* *INDENT-ON* */
+
+  vec_free (xstat_names);
+
+  if (xs)
+    {
+      s = format (s, "\n%Uextended stats:%v",
+		  format_white_space, indent + 2, xs);
+      vec_free (xs);
+    }
+
+  return s;
+}
+
+u8 *
+format_dpdk_tx_dma_trace (u8 * s, va_list * va)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
+  CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
+  dpdk_tx_dma_trace_t *t = va_arg (*va, dpdk_tx_dma_trace_t *);
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index);
+  uword indent = format_get_indent (s);
+  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
+
+  s = format (s, "%U tx queue %d",
+	      format_vnet_sw_interface_name, vnm, sw, t->queue_index);
+
+  s = format (s, "\n%Ubuffer 0x%x: %U",
+	      format_white_space, indent,
+	      t->buffer_index, format_vlib_buffer, &t->buffer);
+
+  s = format (s, "\n%U%U", format_white_space, indent,
+	      format_ethernet_header_with_length, t->buffer.pre_data,
+	      sizeof (t->buffer.pre_data));
+
+  return s;
+}
+
+u8 *
+format_dpdk_rx_dma_trace (u8 * s, va_list * va)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
+  CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
+  dpdk_rx_dma_trace_t *t = va_arg (*va, dpdk_rx_dma_trace_t *);
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index);
+  format_function_t *f;
+  uword indent = format_get_indent (s);
+  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
+
+  s = format (s, "%U rx queue %d",
+	      format_vnet_sw_interface_name, vnm, sw, t->queue_index);
+
+  s = format (s, "\n%Ubuffer 0x%x: %U",
+	      format_white_space, indent,
+	      t->buffer_index, format_vlib_buffer, &t->buffer);
+
+  s = format (s, "\n%U%U",
+	      format_white_space, indent,
+	      format_dpdk_rte_mbuf, &t->mb, &t->data);
+
+  if (vm->trace_main.verbose)
+    {
+      s = format (s, "\n%UPacket Dump%s", format_white_space, indent + 2,
+		  t->mb.data_len > sizeof (t->data) ? " (truncated)" : "");
+      s = format (s, "\n%U%U", format_white_space, indent + 4,
+		  format_hexdump, &t->data,
+		  t->mb.data_len >
+		  sizeof (t->data) ? sizeof (t->data) : t->mb.data_len);
+    }
+  f = node->format_buffer;
+  if (!f)
+    f = format_hex_bytes;
+  s = format (s, "\n%U%U", format_white_space, indent,
+	      f, t->buffer.pre_data, sizeof (t->buffer.pre_data));
+
+  return s;
+}
+
+
+static inline u8 *
+format_dpdk_pkt_types (u8 * s, va_list * va)
+{
+  u32 *pkt_types = va_arg (*va, u32 *);
+  uword indent __attribute__ ((unused)) = format_get_indent (s) + 2;
+
+  if (!*pkt_types)
+    return s;
+
+  s = format (s, "Packet Types");
+
+#define _(L, F, S)             \
+  if ((*pkt_types & RTE_PTYPE_##L##_MASK) == RTE_PTYPE_##L##_##F)           \
+    {                                                                       \
+      s = format (s, "\n%U%s (0x%04x) %s", format_white_space, indent,      \
+                     "RTE_PTYPE_" #L "_" #F, RTE_PTYPE_##L##_##F, S);       \
+    }
+
+  foreach_dpdk_pkt_type
+#undef _
+    return s;
+}
+
+static inline u8 *
+format_dpdk_pkt_offload_flags (u8 * s, va_list * va)
+{
+  u64 *ol_flags = va_arg (*va, u64 *);
+  uword indent = format_get_indent (s) + 2;
+
+  if (!*ol_flags)
+    return s;
+
+  s = format (s, "Packet Offload Flags");
+
+#define _(F, S)             \
+  if (*ol_flags & F)            \
+    {               \
+      s = format (s, "\n%U%s (0x%04x) %s",      \
+      format_white_space, indent, #F, F, S);  \
+    }
+
+  foreach_dpdk_pkt_offload_flag
+#undef _
+    return s;
+}
+
+u8 *
+format_dpdk_rte_mbuf_vlan (u8 * s, va_list * va)
+{
+  ethernet_vlan_header_tv_t *vlan_hdr =
+    va_arg (*va, ethernet_vlan_header_tv_t *);
+
+  if (clib_net_to_host_u16 (vlan_hdr->type) == ETHERNET_TYPE_DOT1AD)
+    {
+      s = format (s, "%U 802.1q vlan ",
+		  format_ethernet_vlan_tci,
+		  clib_net_to_host_u16 (vlan_hdr->priority_cfi_and_id));
+      vlan_hdr++;
+    }
+
+  s = format (s, "%U",
+	      format_ethernet_vlan_tci,
+	      clib_net_to_host_u16 (vlan_hdr->priority_cfi_and_id));
+
+  return s;
+}
+
+u8 *
+format_dpdk_rte_mbuf (u8 * s, va_list * va)
+{
+  struct rte_mbuf *mb = va_arg (*va, struct rte_mbuf *);
+  ethernet_header_t *eth_hdr = va_arg (*va, ethernet_header_t *);
+  uword indent = format_get_indent (s) + 2;
+
+  s = format (s, "PKT MBUF: port %d, nb_segs %d, pkt_len %d"
+	      "\n%Ubuf_len %d, data_len %d, ol_flags 0x%x, data_off %d, phys_addr 0x%x"
+	      "\n%Upacket_type 0x%x",
+	      mb->port, mb->nb_segs, mb->pkt_len,
+	      format_white_space, indent,
+	      mb->buf_len, mb->data_len, mb->ol_flags, mb->data_off,
+	      mb->buf_physaddr, format_white_space, indent, mb->packet_type);
+
+  if (mb->ol_flags)
+    s = format (s, "\n%U%U", format_white_space, indent,
+		format_dpdk_pkt_offload_flags, &mb->ol_flags);
+
+  if ((mb->ol_flags & PKT_RX_VLAN_PKT) &&
+      ((mb->ol_flags & (PKT_RX_VLAN_STRIPPED | PKT_RX_QINQ_STRIPPED)) == 0))
+    {
+      ethernet_vlan_header_tv_t *vlan_hdr =
+	((ethernet_vlan_header_tv_t *) & (eth_hdr->type));
+      s = format (s, " %U", format_dpdk_rte_mbuf_vlan, vlan_hdr);
+    }
+
+  if (mb->packet_type)
+    s = format (s, "\n%U%U", format_white_space, indent,
+		format_dpdk_pkt_types, &mb->packet_type);
+
+  return s;
+}
+
+/* FIXME is this function used? */
+#if 0
+uword
+unformat_socket_mem (unformat_input_t * input, va_list * va)
+{
+  uword **r = va_arg (*va, uword **);
+  int i = 0;
+  u32 mem;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, ","))
+	hash_set (*r, i, 1024);
+      else if (unformat (input, "%u,", &mem))
+	hash_set (*r, i, mem);
+      else if (unformat (input, "%u", &mem))
+	hash_set (*r, i, mem);
+      else
+	{
+	  unformat_put_input (input);
+	  goto done;
+	}
+      i++;
+    }
+
+done:
+  return 1;
+}
+#endif
+
+clib_error_t *
+unformat_rss_fn (unformat_input_t * input, uword * rss_fn)
+{
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (0)
+	;
+#undef _
+#define _(f, s)                                 \
+      else if (unformat (input, s))             \
+        *rss_fn |= f;
+
+      foreach_dpdk_rss_hf
+#undef _
+	else
+	{
+	  return clib_error_return (0, "unknown input `%U'",
+				    format_unformat_error, input);
+	}
+    }
+  return 0;
+}
+
+clib_error_t *
+unformat_hqos (unformat_input_t * input, dpdk_device_config_hqos_t * hqos)
+{
+  clib_error_t *error = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "hqos-thread %u", &hqos->hqos_thread))
+	hqos->hqos_thread_valid = 1;
+      else
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, input);
+	  break;
+	}
+    }
+
+  return error;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
new file mode 100644
index 00000000..8824d789
--- /dev/null
+++ b/src/plugins/dpdk/device/node.c
@@ -0,0 +1,674 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+static char *dpdk_error_strings[] = {
+#define _(n,s) s,
+  foreach_dpdk_error
+#undef _
+};
+
+always_inline int
+vlib_buffer_is_ip4 (vlib_buffer_t * b)
+{
+  ethernet_header_t *h = (ethernet_header_t *) b->data;
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
+}
+
+always_inline int
+vlib_buffer_is_ip6 (vlib_buffer_t * b)
+{
+  ethernet_header_t *h = (ethernet_header_t *) b->data;
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
+}
+
+always_inline int
+vlib_buffer_is_mpls (vlib_buffer_t * b)
+{
+  ethernet_header_t *h = (ethernet_header_t *) b->data;
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST));
+}
+
+always_inline u32
+dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0)
+{
+  if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
+    if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0))
+      return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
+    else
+      return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
+  else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
+    return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
+  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+    return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
+  else
+    return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+}
+
+always_inline int
+dpdk_mbuf_is_vlan (struct rte_mbuf *mb)
+{
+  return (mb->packet_type & RTE_PTYPE_L2_ETHER_VLAN) ==
+    RTE_PTYPE_L2_ETHER_VLAN;
+}
+
+always_inline int
+dpdk_mbuf_is_ip4 (struct rte_mbuf *mb)
+{
+  return RTE_ETH_IS_IPV4_HDR (mb->packet_type) != 0;
+}
+
+always_inline int
+dpdk_mbuf_is_ip6 (struct rte_mbuf *mb)
+{
+  return RTE_ETH_IS_IPV6_HDR (mb->packet_type) != 0;
+}
+
+always_inline u32
+dpdk_rx_next_from_mb (struct rte_mbuf * mb, vlib_buffer_t * b0)
+{
+  if (PREDICT_FALSE (dpdk_mbuf_is_vlan (mb)))
+    return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+  else if (PREDICT_TRUE (dpdk_mbuf_is_ip4 (mb)))
+    return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
+  else if (PREDICT_TRUE (dpdk_mbuf_is_ip6 (mb)))
+    return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
+  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
+    return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
+  else
+    return dpdk_rx_next_from_etype (mb, b0);
+}
+
+always_inline void
+dpdk_rx_error_from_mb (struct rte_mbuf *mb, u32 * next, u8 * error)
+{
+  if (mb->ol_flags & PKT_RX_IP_CKSUM_BAD)
+    {
+      *error = DPDK_ERROR_IP_CHECKSUM_ERROR;
+      *next = VNET_DEVICE_INPUT_NEXT_DROP;
+    }
+  else
+    *error = DPDK_ERROR_NONE;
+}
+
+void
+dpdk_rx_trace (dpdk_main_t * dm,
+	       vlib_node_runtime_t * node,
+	       dpdk_device_t * xd,
+	       u16 queue_id, u32 * buffers, uword n_buffers)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  u32 *b, n_left;
+  u32 next0;
+
+  n_left = n_buffers;
+  b = buffers;
+
+  while (n_left >= 1)
+    {
+      u32 bi0;
+      vlib_buffer_t *b0;
+      dpdk_rx_dma_trace_t *t0;
+      struct rte_mbuf *mb;
+      u8 error0;
+
+      bi0 = b[0];
+      n_left -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      mb = rte_mbuf_from_vlib_buffer (b0);
+
+      if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
+	next0 = xd->per_interface_next_index;
+      else if (PREDICT_TRUE
+	       ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0))
+	next0 = dpdk_rx_next_from_mb (mb, b0);
+      else
+	next0 = dpdk_rx_next_from_etype (mb, b0);
+
+      dpdk_rx_error_from_mb (mb, &next0, &error0);
+
+      vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
+      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+      t0->queue_index = queue_id;
+      t0->device_index = xd->device_index;
+      t0->buffer_index = bi0;
+
+      clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
+      clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
+      clib_memcpy (t0->buffer.pre_data, b0->data,
+		   sizeof (t0->buffer.pre_data));
+      clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data));
+
+      b += 1;
+    }
+}
+
+static inline u32
+dpdk_rx_burst (dpdk_main_t * dm, dpdk_device_t * xd, u16 queue_id)
+{
+  u32 n_buffers;
+  u32 n_left;
+  u32 n_this_chunk;
+
+  n_left = VLIB_FRAME_SIZE;
+  n_buffers = 0;
+
+  if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
+    {
+      while (n_left)
+	{
+	  n_this_chunk = rte_eth_rx_burst (xd->device_index, queue_id,
+					   xd->rx_vectors[queue_id] +
+					   n_buffers, n_left);
+	  n_buffers += n_this_chunk;
+	  n_left -= n_this_chunk;
+
+	  /* Empirically, DPDK r1.8 produces vectors w/ 32 or fewer elts */
+	  if (n_this_chunk < 32)
+	    break;
+	}
+    }
+  else
+    {
+      ASSERT (0);
+    }
+
+  return n_buffers;
+}
+
+
+static_always_inline void
+dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b,
+			  struct rte_mbuf *mb, vlib_buffer_free_list_t * fl)
+{
+  u8 nb_seg = 1;
+  struct rte_mbuf *mb_seg = 0;
+  vlib_buffer_t *b_seg, *b_chain = 0;
+  mb_seg = mb->next;
+  b_chain = b;
+
+  while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
+    {
+      ASSERT (mb_seg != 0);
+
+      b_seg = vlib_buffer_from_rte_mbuf (mb_seg);
+      vlib_buffer_init_for_free_list (b_seg, fl);
+
+      ASSERT ((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
+      ASSERT (b_seg->current_data == 0);
+
+      /*
+       * The driver (e.g. virtio) may not put the packet data at the start
+       * of the segment, so don't assume b_seg->current_data == 0 is correct.
+       */
+      b_seg->current_data =
+	(mb_seg->buf_addr + mb_seg->data_off) - (void *) b_seg->data;
+
+      b_seg->current_length = mb_seg->data_len;
+      b->total_length_not_including_first_buffer += mb_seg->data_len;
+
+      b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
+      b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
+
+      b_chain = b_seg;
+      mb_seg = mb_seg->next;
+      nb_seg++;
+    }
+}
+
+static_always_inline void
+dpdk_prefetch_buffer (struct rte_mbuf *mb)
+{
+  vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
+  CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, LOAD);
+  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, STORE);
+}
+
+/*
+ * This function is used when there are no worker threads.
+ * The main thread performs IO and forwards the packets.
+ */
+static_always_inline u32
+dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
+		   vlib_node_runtime_t * node, u32 cpu_index, u16 queue_id)
+{
+  u32 n_buffers;
+  u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+  u32 n_left_to_next, *to_next;
+  u32 mb_index;
+  vlib_main_t *vm = vlib_get_main ();
+  uword n_rx_bytes = 0;
+  u32 n_trace, trace_cnt __attribute__ ((unused));
+  vlib_buffer_free_list_t *fl;
+  u32 buffer_flags_template;
+
+  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
+    return 0;
+
+  n_buffers = dpdk_rx_burst (dm, xd, queue_id);
+
+  if (n_buffers == 0)
+    {
+      return 0;
+    }
+
+  buffer_flags_template = dm->buffer_flags_template;
+
+  vec_reset_length (xd->d_trace_buffers[cpu_index]);
+  trace_cnt = n_trace = vlib_get_trace_count (vm, node);
+
+  if (n_trace > 0)
+    {
+      u32 n = clib_min (n_trace, n_buffers);
+      mb_index = 0;
+
+      while (n--)
+	{
+	  struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index++];
+	  vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
+	  vec_add1 (xd->d_trace_buffers[cpu_index],
+		    vlib_get_buffer_index (vm, b));
+	}
+    }
+
+  fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+
+  mb_index = 0;
+
+  while (n_buffers > 0)
+    {
+      vlib_buffer_t *b0, *b1, *b2, *b3;
+      u32 bi0, next0, l3_offset0;
+      u32 bi1, next1, l3_offset1;
+      u32 bi2, next2, l3_offset2;
+      u32 bi3, next3, l3_offset3;
+      u8 error0, error1, error2, error3;
+      u64 or_ol_flags;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_buffers > 8 && n_left_to_next > 4)
+	{
+	  struct rte_mbuf *mb0 = xd->rx_vectors[queue_id][mb_index];
+	  struct rte_mbuf *mb1 = xd->rx_vectors[queue_id][mb_index + 1];
+	  struct rte_mbuf *mb2 = xd->rx_vectors[queue_id][mb_index + 2];
+	  struct rte_mbuf *mb3 = xd->rx_vectors[queue_id][mb_index + 3];
+
+	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 4]);
+	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 5]);
+	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 6]);
+	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 7]);
+
+	  if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG)
+	    {
+	      if (PREDICT_FALSE (mb0->nb_segs > 1))
+		dpdk_prefetch_buffer (mb0->next);
+	      if (PREDICT_FALSE (mb1->nb_segs > 1))
+		dpdk_prefetch_buffer (mb1->next);
+	      if (PREDICT_FALSE (mb2->nb_segs > 1))
+		dpdk_prefetch_buffer (mb2->next);
+	      if (PREDICT_FALSE (mb3->nb_segs > 1))
+		dpdk_prefetch_buffer (mb3->next);
+	    }
+
+	  ASSERT (mb0);
+	  ASSERT (mb1);
+	  ASSERT (mb2);
+	  ASSERT (mb3);
+
+	  or_ol_flags = (mb0->ol_flags | mb1->ol_flags |
+			 mb2->ol_flags | mb3->ol_flags);
+	  b0 = vlib_buffer_from_rte_mbuf (mb0);
+	  b1 = vlib_buffer_from_rte_mbuf (mb1);
+	  b2 = vlib_buffer_from_rte_mbuf (mb2);
+	  b3 = vlib_buffer_from_rte_mbuf (mb3);
+
+	  vlib_buffer_init_for_free_list (b0, fl);
+	  vlib_buffer_init_for_free_list (b1, fl);
+	  vlib_buffer_init_for_free_list (b2, fl);
+	  vlib_buffer_init_for_free_list (b3, fl);
+
+	  bi0 = vlib_get_buffer_index (vm, b0);
+	  bi1 = vlib_get_buffer_index (vm, b1);
+	  bi2 = vlib_get_buffer_index (vm, b2);
+	  bi3 = vlib_get_buffer_index (vm, b3);
+
+	  to_next[0] = bi0;
+	  to_next[1] = bi1;
+	  to_next[2] = bi2;
+	  to_next[3] = bi3;
+	  to_next += 4;
+	  n_left_to_next -= 4;
+
+	  if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
+	    {
+	      next0 = next1 = next2 = next3 = xd->per_interface_next_index;
+	    }
+	  else if (PREDICT_TRUE
+		   ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0))
+	    {
+	      next0 = dpdk_rx_next_from_mb (mb0, b0);
+	      next1 = dpdk_rx_next_from_mb (mb1, b1);
+	      next2 = dpdk_rx_next_from_mb (mb2, b2);
+	      next3 = dpdk_rx_next_from_mb (mb3, b3);
+	    }
+	  else
+	    {
+	      next0 = dpdk_rx_next_from_etype (mb0, b0);
+	      next1 = dpdk_rx_next_from_etype (mb1, b1);
+	      next2 = dpdk_rx_next_from_etype (mb2, b2);
+	      next3 = dpdk_rx_next_from_etype (mb3, b3);
+	    }
+
+	  if (PREDICT_FALSE (or_ol_flags & PKT_RX_IP_CKSUM_BAD))
+	    {
+	      dpdk_rx_error_from_mb (mb0, &next0, &error0);
+	      dpdk_rx_error_from_mb (mb1, &next1, &error1);
+	      dpdk_rx_error_from_mb (mb2, &next2, &error2);
+	      dpdk_rx_error_from_mb (mb3, &next3, &error3);
+	      b0->error = node->errors[error0];
+	      b1->error = node->errors[error1];
+	      b2->error = node->errors[error2];
+	      b3->error = node->errors[error3];
+	    }
+	  else
+	    {
+	      b0->error = b1->error = node->errors[DPDK_ERROR_NONE];
+	      b2->error = b3->error = node->errors[DPDK_ERROR_NONE];
+	    }
+
+	  l3_offset0 = device_input_next_node_advance[next0];
+	  l3_offset1 = device_input_next_node_advance[next1];
+	  l3_offset2 = device_input_next_node_advance[next2];
+	  l3_offset3 = device_input_next_node_advance[next3];
+
+	  b0->current_data = l3_offset0 + mb0->data_off;
+	  b1->current_data = l3_offset1 + mb1->data_off;
+	  b2->current_data = l3_offset2 + mb2->data_off;
+	  b3->current_data = l3_offset3 + mb3->data_off;
+
+	  b0->current_data -= RTE_PKTMBUF_HEADROOM;
+	  b1->current_data -= RTE_PKTMBUF_HEADROOM;
+	  b2->current_data -= RTE_PKTMBUF_HEADROOM;
+	  b3->current_data -= RTE_PKTMBUF_HEADROOM;
+
+	  b0->current_length = mb0->data_len - l3_offset0;
+	  b1->current_length = mb1->data_len - l3_offset1;
+	  b2->current_length = mb2->data_len - l3_offset2;
+	  b3->current_length = mb3->data_len - l3_offset3;
+
+	  b0->flags = buffer_flags_template;
+	  b1->flags = buffer_flags_template;
+	  b2->flags = buffer_flags_template;
+	  b3->flags = buffer_flags_template;
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b1)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b2)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b3)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  vnet_buffer (b2)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  vnet_buffer (b3)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+	  n_rx_bytes += mb0->pkt_len;
+	  n_rx_bytes += mb1->pkt_len;
+	  n_rx_bytes += mb2->pkt_len;
+	  n_rx_bytes += mb3->pkt_len;
+
+	  /* Process subsequent segments of multi-segment packets */
+	  if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG)
+	    {
+	      dpdk_process_subseq_segs (vm, b0, mb0, fl);
+	      dpdk_process_subseq_segs (vm, b1, mb1, fl);
+	      dpdk_process_subseq_segs (vm, b2, mb2, fl);
+	      dpdk_process_subseq_segs (vm, b3, mb3, fl);
+	    }
+
+	  /*
+	   * Turn this on if you run into
+	   * "bad monkey" contexts, and you want to know exactly
+	   * which nodes they've visited... See main.c...
+	   */
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
+
+	  /* Do we have any driver RX features configured on the interface? */
+	  vnet_feature_start_device_input_x4 (xd->vlib_sw_if_index,
+					      &next0, &next1, &next2, &next3,
+					      b0, b1, b2, b3,
+					      l3_offset0, l3_offset1,
+					      l3_offset2, l3_offset3);
+
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	  n_buffers -= 4;
+	  mb_index += 4;
+	}
+      while (n_buffers > 0 && n_left_to_next > 0)
+	{
+	  struct rte_mbuf *mb0 = xd->rx_vectors[queue_id][mb_index];
+
+	  ASSERT (mb0);
+
+	  b0 = vlib_buffer_from_rte_mbuf (mb0);
+
+	  /* Prefetch one next segment if it exists. */
+	  if (PREDICT_FALSE (mb0->nb_segs > 1))
+	    dpdk_prefetch_buffer (mb0->next);
+
+	  vlib_buffer_init_for_free_list (b0, fl);
+
+	  bi0 = vlib_get_buffer_index (vm, b0);
+
+	  to_next[0] = bi0;
+	  to_next++;
+	  n_left_to_next--;
+
+	  if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
+	    next0 = xd->per_interface_next_index;
+	  else if (PREDICT_TRUE
+		   ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0))
+	    next0 = dpdk_rx_next_from_mb (mb0, b0);
+	  else
+	    next0 = dpdk_rx_next_from_etype (mb0, b0);
+
+	  dpdk_rx_error_from_mb (mb0, &next0, &error0);
+	  b0->error = node->errors[error0];
+
+	  l3_offset0 = device_input_next_node_advance[next0];
+
+	  b0->current_data = l3_offset0;
+	  b0->current_data += mb0->data_off - RTE_PKTMBUF_HEADROOM;
+	  b0->current_length = mb0->data_len - l3_offset0;
+
+	  b0->flags = buffer_flags_template;
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  n_rx_bytes += mb0->pkt_len;
+
+	  /* Process subsequent segments of multi-segment packets */
+	  dpdk_process_subseq_segs (vm, b0, mb0, fl);
+
+	  /*
+	   * Turn this on if you run into
+	   * "bad monkey" contexts, and you want to know exactly
+	   * which nodes they've visited... See main.c...
+	   */
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+
+	  /* Do we have any driver RX features configured on the interface? */
+	  vnet_feature_start_device_input_x1 (xd->vlib_sw_if_index, &next0,
+					      b0, l3_offset0);
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	  n_buffers--;
+	  mb_index++;
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[cpu_index]) > 0))
+    {
+      dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers[cpu_index],
+		     vec_len (xd->d_trace_buffers[cpu_index]));
+      vlib_set_trace_count (vm, node, n_trace -
+			    vec_len (xd->d_trace_buffers[cpu_index]));
+    }
+
+  vlib_increment_combined_counter
+    (vnet_get_main ()->interface_main.combined_sw_if_counters
+     + VNET_INTERFACE_COUNTER_RX,
+     cpu_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes);
+
+  vnet_device_increment_rx_packets (cpu_index, mb_index);
+
+  return mb_index;
+}
+
+static inline void
+poll_rate_limit (dpdk_main_t * dm)
+{
+  /* Limit the poll rate by sleeping for N msec between polls */
+  if (PREDICT_FALSE (dm->poll_sleep != 0))
+    {
+      struct timespec ts, tsrem;
+
+      ts.tv_sec = 0;
+      ts.tv_nsec = 1000 * 1000 * dm->poll_sleep;	/* 1ms */
+
+      while (nanosleep (&ts, &tsrem) < 0)
+	{
+	  ts = tsrem;
+	}
+    }
+}
+
+/** \brief Main DPDK input node
+    @node dpdk-input
+
+    This is the main DPDK input node: across each assigned interface,
+    call rte_eth_rx_burst(...) or similar to obtain a vector of
+    packets to process. Handle early packet discard. Derive @c
+    vlib_buffer_t metadata from struct rte_mbuf metadata,
+    Depending on the resulting metadata: adjust b->current_data,
+    b->current_length  and dispatch directly to
+    ip4-input-no-checksum, or ip6-input. Trace the packet if required.
+
+    @param vm   vlib_main_t corresponding to the current thread
+    @param node vlib_node_runtime_t
+    @param f    vlib_frame_t input-node, not used.
+
+    @par Graph mechanics: buffer metadata, next index usage
+
+    @em Uses:
+    - struct rte_mbuf mb->ol_flags
+        - PKT_RX_IP_CKSUM_BAD
+    -  RTE_ETH_IS_xxx_HDR(mb->packet_type) 
+        - packet classification result
+
+    @em Sets:
+    - b->error if the packet is to be dropped immediately
+    - b->current_data, b->current_length
+        - adjusted as needed to skip the L2 header in  direct-dispatch cases
+    - vnet_buffer(b)->sw_if_index[VLIB_RX]
+        - rx interface sw_if_index
+    - vnet_buffer(b)->sw_if_index[VLIB_TX] = ~0
+        - required by ipX-lookup
+    - b->flags
+        - to indicate multi-segment pkts (VLIB_BUFFER_NEXT_PRESENT), etc.
+
+    Next Nodes:
+    - Static arcs to: error-drop, ethernet-input,
+      ip4-input-no-checksum, ip6-input, mpls-input
+    - per-interface redirection, controlled by
+      xd->per_interface_next_index
+*/
+
+static uword
+dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd;
+  uword n_rx_packets = 0;
+  dpdk_device_and_queue_t *dq;
+  u32 cpu_index = os_get_cpu_number ();
+
+  /*
+   * Poll all devices on this cpu for input/interrupts.
+   */
+  /* *INDENT-OFF* */
+  vec_foreach (dq, dm->devices_by_cpu[cpu_index])
+    {
+      xd = vec_elt_at_index(dm->devices, dq->device);
+      n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id);
+    }
+  /* *INDENT-ON* */
+
+  poll_rate_limit (dm);
+
+  return n_rx_packets;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (dpdk_input_node) = {
+  .function = dpdk_input,
+  .type = VLIB_NODE_TYPE_INPUT,
+  .name = "dpdk-input",
+  .sibling_of = "device-input",
+
+  /* Will be enabled if/when hardware is detected. */
+  .state = VLIB_NODE_STATE_DISABLED,
+
+  .format_buffer = format_ethernet_header_with_length,
+  .format_trace = format_dpdk_rx_dma_trace,
+
+  .n_errors = DPDK_N_ERROR,
+  .error_strings = dpdk_error_strings,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (dpdk_input_node, dpdk_input);
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/dir.dox b/src/plugins/dpdk/dir.dox
new file mode 100644
index 00000000..43e36753
--- /dev/null
+++ b/src/plugins/dpdk/dir.dox
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Doxygen directory documentation */
+
+/**
+@dir
+@brief DPDK Abstraction Layer.
+
+This directory contains the source code for the DPDK abstraction layer.
+
+*/
+/*? %%clicmd:group_label DPDK and pcap tx %% ?*/
+/*? %%syscfg:group_label DPDK and pcap tx %% ?*/
diff --git a/src/plugins/dpdk/hqos/hqos.c b/src/plugins/dpdk/hqos/hqos.c
new file mode 100644
index 00000000..a288fca7
--- /dev/null
+++ b/src/plugins/dpdk/hqos/hqos.c
@@ -0,0 +1,775 @@
+/*
+ * Copyright(c) 2016 Intel Corporation. All rights reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 	/* enumerate all vlib messages */
+
+#define vl_typedefs		/* define message structures */
+#include 
+#undef vl_typedefs
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+#include 
+
+dpdk_main_t dpdk_main;
+
+/***
+ *
+ * HQoS default configuration values
+ *
+ ***/
+
+static dpdk_device_config_hqos_t hqos_params_default = {
+  .hqos_thread_valid = 0,
+
+  .swq_size = 4096,
+  .burst_enq = 256,
+  .burst_deq = 220,
+
+  /*
+   * Packet field to identify the subport.
+   *
+   * Default value: Since only one subport is defined by default (see below:
+   *     n_subports_per_port = 1), the subport ID is hardcoded to 0.
+   */
+  .pktfield0_slabpos = 0,
+  .pktfield0_slabmask = 0,
+
+  /*
+   * Packet field to identify the pipe.
+   *
+   * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
+   */
+  .pktfield1_slabpos = 40,
+  .pktfield1_slabmask = 0x0000000FFF000000LLU,
+
+  /* Packet field used as index into TC translation table to identify the traffic
+   *     class and queue.
+   *
+   * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
+   */
+  .pktfield2_slabpos = 8,
+  .pktfield2_slabmask = 0x00000000000000FCLLU,
+  .tc_table = {
+	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	       },
+
+  /* port */
+  .port = {
+	   .name = NULL,	/* Set at init */
+	   .socket = 0,		/* Set at init */
+	   .rate = 1250000000,	/* Assuming 10GbE port */
+	   .mtu = 14 + 1500,	/* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
+	   .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
+	   .n_subports_per_port = 1,
+	   .n_pipes_per_subport = 4096,
+	   .qsize = {64, 64, 64, 64},
+	   .pipe_profiles = NULL,	/* Set at config */
+	   .n_pipe_profiles = 1,
+
+#ifdef RTE_SCHED_RED
+	   .red_params = {
+			  /* Traffic Class 0 Colors Green / Yellow / Red */
+			  [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+
+			  /* Traffic Class 1 - Colors Green / Yellow / Red */
+			  [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+
+			  /* Traffic Class 2 - Colors Green / Yellow / Red */
+			  [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+
+			  /* Traffic Class 3 - Colors Green / Yellow / Red */
+			  [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9},
+			  [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
+				    10,.wq_log2 = 9}
+			  },
+#endif /* RTE_SCHED_RED */
+	   },
+};
+
+static struct rte_sched_subport_params hqos_subport_params_default = {
+  .tb_rate = 1250000000,	/* 10GbE line rate (measured in bytes/second) */
+  .tb_size = 1000000,
+  .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
+  .tc_period = 10,
+};
+
+static struct rte_sched_pipe_params hqos_pipe_params_default = {
+  .tb_rate = 305175,		/* 10GbE line rate divided by 4K pipes */
+  .tb_size = 1000000,
+  .tc_rate = {305175, 305175, 305175, 305175},
+  .tc_period = 40,
+#ifdef RTE_SCHED_SUBPORT_TC_OV
+  .tc_ov_weight = 1,
+#endif
+  .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+};
+
+/***
+ *
+ * HQoS configuration
+ *
+ ***/
+
+int
+dpdk_hqos_validate_mask (u64 mask, u32 n)
+{
+  int count = __builtin_popcountll (mask);
+  int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
+  int pos_trail = __builtin_ctzll (mask);
+  int count_expected = __builtin_popcount (n - 1);
+
+  /* Handle the exceptions */
+  if (n == 0)
+    return -1;			/* Error */
+
+  if ((mask == 0) && (n == 1))
+    return 0;			/* OK */
+
+  if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
+    return -2;			/* Error */
+
+  /* Check that mask is contiguous */
+  if ((pos_lead - pos_trail) != count)
+    return -3;			/* Error */
+
+  /* Check that mask contains the expected number of bits set */
+  if (count != count_expected)
+    return -4;			/* Error */
+
+  return 0;			/* OK */
+}
+
+void
+dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
+					      hqos, u32 pipe_profile_id)
+{
+  memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
+	  sizeof (hqos_pipe_params_default));
+}
+
+void
+dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos)
+{
+  struct rte_sched_subport_params *subport_params;
+  struct rte_sched_pipe_params *pipe_params;
+  u32 *pipe_map;
+  u32 i;
+
+  memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
+
+  /* pipe */
+  vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
+
+  for (i = 0; i < vec_len (hqos->pipe); i++)
+    memcpy (&pipe_params[i],
+	    &hqos_pipe_params_default, sizeof (hqos_pipe_params_default));
+
+  hqos->port.pipe_profiles = hqos->pipe;
+
+  /* subport */
+  vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
+
+  for (i = 0; i < vec_len (hqos->subport); i++)
+    memcpy (&subport_params[i],
+	    &hqos_subport_params_default,
+	    sizeof (hqos_subport_params_default));
+
+  /* pipe profile */
+  vec_add2 (hqos->pipe_map,
+	    pipe_map,
+	    hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
+
+  for (i = 0; i < vec_len (hqos->pipe_map); i++)
+    pipe_map[i] = 0;
+}
+
+/***
+ *
+ * HQoS init
+ *
+ ***/
+
+clib_error_t *
+dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos)
+{
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  char name[32];
+  u32 subport_id, i;
+  int rv;
+
+  /* Detect the set of worker threads */
+  int worker_thread_first = 0;
+  int worker_thread_count = 0;
+
+  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  vlib_thread_registration_t *tr =
+    p ? (vlib_thread_registration_t *) p[0] : 0;
+
+  if (tr && tr->count > 0)
+    {
+      worker_thread_first = tr->first_index;
+      worker_thread_count = tr->count;
+    }
+
+  /* Allocate the per-thread device data array */
+  vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1,
+			CLIB_CACHE_LINE_BYTES);
+  memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
+
+  vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES);
+  memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
+
+  /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
+  vec_validate (xd->hqos_ht->swq, worker_thread_count);
+
+  /* SWQ */
+  for (i = 0; i < worker_thread_count + 1; i++)
+    {
+      u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
+
+      snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
+		xd->device_index);
+      xd->hqos_ht->swq[i] =
+	rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
+      if (xd->hqos_ht->swq[i] == NULL)
+	return clib_error_return (0,
+				  "SWQ-worker%u-to-device%u: rte_ring_create err",
+				  i, xd->device_index);
+    }
+
+  /*
+   * HQoS
+   */
+
+  /* HQoS port */
+  snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
+  hqos->port.name = strdup (name);
+  if (hqos->port.name == NULL)
+    return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
+
+  hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
+  if (hqos->port.socket == SOCKET_ID_ANY)
+    hqos->port.socket = 0;
+
+  xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
+  if (xd->hqos_ht->hqos == NULL)
+    return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
+			      xd->device_index);
+
+  /* HQoS subport */
+  for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
+       subport_id++)
+    {
+      u32 pipe_id;
+
+      rv =
+	rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
+				  &hqos->subport[subport_id]);
+      if (rv)
+	return clib_error_return (0,
+				  "HQoS%u subport %u: rte_sched_subport_config err (%d)",
+				  xd->device_index, subport_id, rv);
+
+      /* HQoS pipe */
+      for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
+	{
+	  u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
+	  u32 profile_id = hqos->pipe_map[pos];
+
+	  rv =
+	    rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
+				   profile_id);
+	  if (rv)
+	    return clib_error_return (0,
+				      "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
+				      xd->device_index, subport_id, pipe_id,
+				      rv);
+	}
+    }
+
+  /* Set up per-thread device data for the I/O TX thread */
+  xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
+  xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
+  vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
+  vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
+  xd->hqos_ht->pkts_enq_len = 0;
+  xd->hqos_ht->swq_pos = 0;
+  xd->hqos_ht->flush_count = 0;
+
+  /* Set up per-thread device data for each worker thread */
+  for (i = 0; i < worker_thread_count + 1; i++)
+    {
+      u32 tid;
+      if (i)
+	tid = worker_thread_first + (i - 1);
+      else
+	tid = i;
+
+      xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
+      xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
+      xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
+      xd->hqos_wt[tid].hqos_field0_slabshr =
+	__builtin_ctzll (hqos->pktfield0_slabmask);
+      xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
+      xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
+      xd->hqos_wt[tid].hqos_field1_slabshr =
+	__builtin_ctzll (hqos->pktfield1_slabmask);
+      xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
+      xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
+      xd->hqos_wt[tid].hqos_field2_slabshr =
+	__builtin_ctzll (hqos->pktfield2_slabmask);
+      memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
+	      sizeof (hqos->tc_table));
+    }
+
+  return 0;
+}
+
+/***
+ *
+ * HQoS run-time
+ *
+ ***/
+/*
+ * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
+ *
+ * w
+ *     Information for the current thread
+ */
+static_always_inline void
+dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  u32 cpu_index = vm->cpu_index;
+  u32 dev_pos;
+
+  dev_pos = 0;
+  while (1)
+    {
+      vlib_worker_thread_barrier_check ();
+
+      u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
+      if (dev_pos >= n_devs)
+	dev_pos = 0;
+
+      dpdk_device_and_queue_t *dq =
+	vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
+      dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
+
+      dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
+      u32 device_index = xd->device_index;
+      u16 queue_id = dq->queue_id;
+
+      struct rte_mbuf **pkts_enq = hqos->pkts_enq;
+      u32 pkts_enq_len = hqos->pkts_enq_len;
+      u32 swq_pos = hqos->swq_pos;
+      u32 n_swq = vec_len (hqos->swq), i;
+      u32 flush_count = hqos->flush_count;
+
+      for (i = 0; i < n_swq; i++)
+	{
+	  /* Get current SWQ for this device */
+	  struct rte_ring *swq = hqos->swq[swq_pos];
+
+	  /* Read SWQ burst to packet buffer of this device */
+	  pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
+						     (void **)
+						     &pkts_enq[pkts_enq_len],
+						     hqos->hqos_burst_enq);
+
+	  /* Get next SWQ for this device */
+	  swq_pos++;
+	  if (swq_pos >= n_swq)
+	    swq_pos = 0;
+	  hqos->swq_pos = swq_pos;
+
+	  /* HWQ TX enqueue when burst available */
+	  if (pkts_enq_len >= hqos->hqos_burst_enq)
+	    {
+	      u32 n_pkts = rte_eth_tx_burst (device_index,
+					     (uint16_t) queue_id,
+					     pkts_enq,
+					     (uint16_t) pkts_enq_len);
+
+	      for (; n_pkts < pkts_enq_len; n_pkts++)
+		rte_pktmbuf_free (pkts_enq[n_pkts]);
+
+	      pkts_enq_len = 0;
+	      flush_count = 0;
+	      break;
+	    }
+	}
+      if (pkts_enq_len)
+	{
+	  flush_count++;
+	  if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
+	    {
+	      rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
+
+	      pkts_enq_len = 0;
+	      flush_count = 0;
+	    }
+	}
+      hqos->pkts_enq_len = pkts_enq_len;
+      hqos->flush_count = flush_count;
+
+      /* Advance to next device */
+      dev_pos++;
+    }
+}
+
+static_always_inline void
+dpdk_hqos_thread_internal (vlib_main_t * vm)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  u32 cpu_index = vm->cpu_index;
+  u32 dev_pos;
+
+  dev_pos = 0;
+  while (1)
+    {
+      vlib_worker_thread_barrier_check ();
+
+      u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
+      if (PREDICT_FALSE (n_devs == 0))
+	{
+	  dev_pos = 0;
+	  continue;
+	}
+      if (dev_pos >= n_devs)
+	dev_pos = 0;
+
+      dpdk_device_and_queue_t *dq =
+	vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
+      dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
+
+      dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
+      u32 device_index = xd->device_index;
+      u16 queue_id = dq->queue_id;
+
+      struct rte_mbuf **pkts_enq = hqos->pkts_enq;
+      struct rte_mbuf **pkts_deq = hqos->pkts_deq;
+      u32 pkts_enq_len = hqos->pkts_enq_len;
+      u32 swq_pos = hqos->swq_pos;
+      u32 n_swq = vec_len (hqos->swq), i;
+      u32 flush_count = hqos->flush_count;
+
+      /*
+       * SWQ dequeue and HQoS enqueue for current device
+       */
+      for (i = 0; i < n_swq; i++)
+	{
+	  /* Get current SWQ for this device */
+	  struct rte_ring *swq = hqos->swq[swq_pos];
+
+	  /* Read SWQ burst to packet buffer of this device */
+	  pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
+						     (void **)
+						     &pkts_enq[pkts_enq_len],
+						     hqos->hqos_burst_enq);
+
+	  /* Get next SWQ for this device */
+	  swq_pos++;
+	  if (swq_pos >= n_swq)
+	    swq_pos = 0;
+	  hqos->swq_pos = swq_pos;
+
+	  /* HQoS enqueue when burst available */
+	  if (pkts_enq_len >= hqos->hqos_burst_enq)
+	    {
+	      rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
+
+	      pkts_enq_len = 0;
+	      flush_count = 0;
+	      break;
+	    }
+	}
+      if (pkts_enq_len)
+	{
+	  flush_count++;
+	  if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
+	    {
+	      rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
+
+	      pkts_enq_len = 0;
+	      flush_count = 0;
+	    }
+	}
+      hqos->pkts_enq_len = pkts_enq_len;
+      hqos->flush_count = flush_count;
+
+      /*
+       * HQoS dequeue and HWQ TX enqueue for current device
+       */
+      {
+	u32 pkts_deq_len, n_pkts;
+
+	pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
+					       pkts_deq,
+					       hqos->hqos_burst_deq);
+
+	for (n_pkts = 0; n_pkts < pkts_deq_len;)
+	  n_pkts += rte_eth_tx_burst (device_index,
+				      (uint16_t) queue_id,
+				      &pkts_deq[n_pkts],
+				      (uint16_t) (pkts_deq_len - n_pkts));
+      }
+
+      /* Advance to next device */
+      dev_pos++;
+    }
+}
+
+void
+dpdk_hqos_thread (vlib_worker_thread_t * w)
+{
+  vlib_main_t *vm;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_main_t *dm = &dpdk_main;
+
+  vm = vlib_get_main ();
+
+  ASSERT (vm->cpu_index == os_get_cpu_number ());
+
+  clib_time_init (&vm->clib_time);
+  clib_mem_set_heap (w->thread_mheap);
+
+  /* Wait until the dpdk init sequence is complete */
+  while (tm->worker_thread_release == 0)
+    vlib_worker_thread_barrier_check ();
+
+  if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0)
+    return
+      clib_error
+      ("current I/O TX thread does not have any devices assigned to it");
+
+  if (DPDK_HQOS_DBG_BYPASS)
+    dpdk_hqos_thread_internal_hqos_dbg_bypass (vm);
+  else
+    dpdk_hqos_thread_internal (vm);
+}
+
+void
+dpdk_hqos_thread_fn (void *arg)
+{
+  vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
+  vlib_worker_thread_init (w);
+  dpdk_hqos_thread (w);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
+{
+  .name = "hqos-threads",
+  .short_name = "hqos-threads",
+  .function = dpdk_hqos_thread_fn,
+};
+/* *INDENT-ON* */
+
+/*
+ * HQoS run-time code to be called by the worker threads
+ */
+#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
+({                                                              \
+  u64 slab = *((u64 *) &byte_array[slab_pos]);                  \
+  u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;   \
+  val;                                                          \
+})
+
+#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
+  ((((u64) (queue)) & 0x3) |                               \
+  ((((u64) (traffic_class)) & 0x3) << 2) |                 \
+  ((((u64) (color)) & 0x3) << 4) |                         \
+  ((((u64) (subport)) & 0xFFFF) << 16) |                   \
+  ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
+
+void
+dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
+			struct rte_mbuf **pkts, u32 n_pkts)
+{
+  u32 i;
+
+  for (i = 0; i < (n_pkts & (~0x3)); i += 4)
+    {
+      struct rte_mbuf *pkt0 = pkts[i];
+      struct rte_mbuf *pkt1 = pkts[i + 1];
+      struct rte_mbuf *pkt2 = pkts[i + 2];
+      struct rte_mbuf *pkt3 = pkts[i + 3];
+
+      u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
+      u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
+      u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
+      u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
+
+      u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
+				   hqos->hqos_field0_slabmask,
+				   hqos->hqos_field0_slabshr);
+      u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
+				hqos->hqos_field1_slabmask,
+				hqos->hqos_field1_slabshr);
+      u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
+				hqos->hqos_field2_slabmask,
+				hqos->hqos_field2_slabshr);
+      u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
+      u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
+
+      u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
+				   hqos->hqos_field0_slabmask,
+				   hqos->hqos_field0_slabshr);
+      u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
+				hqos->hqos_field1_slabmask,
+				hqos->hqos_field1_slabshr);
+      u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
+				hqos->hqos_field2_slabmask,
+				hqos->hqos_field2_slabshr);
+      u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
+      u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
+
+      u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
+				   hqos->hqos_field0_slabmask,
+				   hqos->hqos_field0_slabshr);
+      u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
+				hqos->hqos_field1_slabmask,
+				hqos->hqos_field1_slabshr);
+      u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
+				hqos->hqos_field2_slabmask,
+				hqos->hqos_field2_slabshr);
+      u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
+      u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
+
+      u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
+				   hqos->hqos_field0_slabmask,
+				   hqos->hqos_field0_slabshr);
+      u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
+				hqos->hqos_field1_slabmask,
+				hqos->hqos_field1_slabshr);
+      u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
+				hqos->hqos_field2_slabmask,
+				hqos->hqos_field2_slabshr);
+      u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
+      u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
+
+      u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
+						 pkt0_pipe,
+						 pkt0_tc,
+						 pkt0_tc_q,
+						 0);
+      u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
+						 pkt1_pipe,
+						 pkt1_tc,
+						 pkt1_tc_q,
+						 0);
+      u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
+						 pkt2_pipe,
+						 pkt2_tc,
+						 pkt2_tc_q,
+						 0);
+      u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
+						 pkt3_pipe,
+						 pkt3_tc,
+						 pkt3_tc_q,
+						 0);
+
+      pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
+      pkt0->hash.sched.hi = pkt0_sched >> 32;
+      pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
+      pkt1->hash.sched.hi = pkt1_sched >> 32;
+      pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
+      pkt2->hash.sched.hi = pkt2_sched >> 32;
+      pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
+      pkt3->hash.sched.hi = pkt3_sched >> 32;
+    }
+
+  for (; i < n_pkts; i++)
+    {
+      struct rte_mbuf *pkt = pkts[i];
+
+      u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
+
+      u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
+				  hqos->hqos_field0_slabmask,
+				  hqos->hqos_field0_slabshr);
+      u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
+			       hqos->hqos_field1_slabmask,
+			       hqos->hqos_field1_slabshr);
+      u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
+			       hqos->hqos_field2_slabmask,
+			       hqos->hqos_field2_slabshr);
+      u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
+      u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
+
+      u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
+						pkt_pipe,
+						pkt_tc,
+						pkt_tc_q,
+						0);
+
+      pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
+      pkt->hash.sched.hi = pkt_sched >> 32;
+    }
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/hqos/qos_doc.md b/src/plugins/dpdk/hqos/qos_doc.md
new file mode 100644
index 00000000..7c064246
--- /dev/null
+++ b/src/plugins/dpdk/hqos/qos_doc.md
@@ -0,0 +1,411 @@
+# QoS Hierarchical Scheduler    {#qos_doc}
+
+The Quality-of-Service (QoS) scheduler performs egress-traffic management by
+prioritizing the transmission of the packets of different type services and
+subcribers based on the Service Level Agreements (SLAs). The QoS scheduler can
+be enabled on one or more NIC output interfaces depending upon the
+requirement.
+
+
+## Overview
+
+The QoS schdeuler supports a number of scheduling and shaping levels which
+construct hierarchical-tree. The first level in the hierarchy is port (i.e.
+the physical interface) that constitutes the root node of the tree. The
+subsequent level is subport which represents the group of the
+users/subscribers. The individual user/subscriber is represented by the pipe
+at the next level. Each user can have different traffic type based on the
+criteria of specific loss rate, jitter, and latency. These traffic types are
+represented at the traffic-class level in the form of different traffic-
+classes. The last level contains number of queues which are grouped together
+to host the packets of the specific class type traffic.
+
+The QoS scheduler implementation requires flow classification, enqueue  and
+dequeue operations. The flow classification is mandatory stage for HQoS where
+incoming packets are classified by mapping the packet fields information to
+5-tuple (HQoS subport, pipe, traffic class, queue within traffic class, and
+color) and storing that information in mbuf sched field. The enqueue operation
+uses this information to determine the queue for storing the packet, and at
+this stage, if the specific queue is full, QoS drops the packet. The dequeue
+operation consists of scheduling the packet based on its length and available
+credits, and handing over the scheduled packet to the output interface.
+
+For more information on QoS Scheduler, please refer DPDK Programmer's Guide-
+http://dpdk.org/doc/guides/prog_guide/qos_framework.html
+
+
+### QoS Schdeuler Parameters
+
+Following illustrates the default HQoS configuration for each 10GbE output
+port:
+
+Single subport (subport 0):
+  - Subport rate set to 100% of port rate
+  - Each of the 4 traffic classes has rate set to 100% of port rate
+
+4K pipes per subport 0 (pipes 0 .. 4095) with identical configuration:
+  - Pipe rate set to 1/4K of port rate
+  - Each of the 4 traffic classes has rate set to 100% of pipe rate
+  - Within each traffic class, the byte-level WRR weights for the 4 queues are set to 1:1:1:1
+
+
+#### Port configuration
+
+```
+port {
+  rate 1250000000           /* Assuming 10GbE port */
+  frame_overhead 24         /* Overhead fields per Ethernet frame:
+                             * 7B (Preamble) +
+                             * 1B (Start of Frame Delimiter (SFD)) + 
+                             * 4B (Frame Check Sequence (FCS)) +
+                             * 12B (Inter Frame Gap (IFG))
+                             */
+  mtu 1522                  /* Assuming Ethernet/IPv4 pkt (FCS not included) */
+  n_subports_per_port 1     /* Number of subports per output interface */
+  n_pipes_per_subport 4096  /* Number of pipes (users/subscribers) */
+  queue_sizes 64 64 64 64   /* Packet queue size for each traffic class.
+                             * All queues within the same pipe traffic class
+                             * have the same size. Queues from different
+                             * pipes serving the same traffic class have
+                             * the same size. */
+}
+```
+
+
+#### Subport configuration
+
+```
+subport 0 {
+  tb_rate 1250000000        /* Subport level token bucket rate (bytes per second) */
+  tb_size 1000000           /* Subport level token bucket size (bytes) */
+  tc0_rate 1250000000       /* Subport level token bucket rate for traffic class 0 (bytes per second) */
+  tc1_rate 1250000000       /* Subport level token bucket rate for traffic class 1 (bytes per second) */
+  tc2_rate 1250000000       /* Subport level token bucket rate for traffic class 2 (bytes per second) */
+  tc3_rate 1250000000       /* Subport level token bucket rate for traffic class 3 (bytes per second) */
+  tc_period 10              /* Time interval for refilling the token bucket associated with traffic class (Milliseconds) */
+  pipe 0 4095 profile 0     /* pipes (users/subscribers) configured with pipe profile 0 */
+}
+```
+
+
+#### Pipe configuration
+
+```
+pipe_profile 0 {
+  tb_rate 305175          /* Pipe level token bucket rate (bytes per second) */
+  tb_size 1000000         /* Pipe level token bucket size (bytes) */
+  tc0_rate 305175         /* Pipe level token bucket rate for traffic class 0 (bytes per second) */
+  tc1_rate 305175         /* Pipe level token bucket rate for traffic class 1 (bytes per second) */
+  tc2_rate 305175         /* Pipe level token bucket rate for traffic class 2 (bytes per second) */
+  tc3_rate 305175         /* Pipe level token bucket rate for traffic class 3 (bytes per second) */
+  tc_period 40            /* Time interval for refilling the token bucket associated with traffic class at pipe level (Milliseconds) */
+  tc3_oversubscription_weight 1 /* Weight traffic class 3 oversubscription */
+  tc0_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 0 */
+  tc1_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 1 */
+  tc2_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 2 */
+  tc3_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 3 */
+}
+```
+
+
+#### Random Early Detection (RED) parameters per traffic class and color (Green / Yellow / Red)
+
+```
+red {
+  tc0_wred_min 48 40 32     /* Minimum threshold for traffic class 0 queue (min_th) in number of packets */
+  tc0_wred_max 64 64 64     /* Maximum threshold for traffic class 0 queue (max_th) in number of packets */
+  tc0_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 0 queue (maxp = 1 / maxp_inv) */
+  tc0_wred_weight 9 9 9     /* Traffic Class 0 queue weight */
+  tc1_wred_min 48 40 32     /* Minimum threshold for traffic class 1 queue (min_th) in number of packets */
+  tc1_wred_max 64 64 64     /* Maximum threshold for traffic class 1 queue (max_th) in number of packets */
+  tc1_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 1 queue (maxp = 1 / maxp_inv) */
+  tc1_wred_weight 9 9 9     /* Traffic Class 1 queue weight */
+  tc2_wred_min 48 40 32     /* Minimum threshold for traffic class 2 queue (min_th) in number of packets */
+  tc2_wred_max 64 64 64     /* Maximum threshold for traffic class 2 queue (max_th) in number of packets */
+  tc2_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 2 queue (maxp = 1 / maxp_inv) */
+  tc2_wred_weight 9 9 9     /* Traffic Class 2 queue weight */
+  tc3_wred_min 48 40 32     /* Minimum threshold for traffic class 3 queue (min_th) in number of packets */
+  tc3_wred_max 64 64 64     /* Maximum threshold for traffic class 3 queue (max_th) in number of packets */
+  tc3_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 3 queue (maxp = 1 / maxp_inv) */
+  tc3_wred_weight 9 9 9     /* Traffic Class 3 queue weight */
+}
+```
+
+
+### DPDK QoS Scheduler Integration in VPP
+
+The Hierarchical Quaity-of-Service (HQoS) scheduler object could be seen as
+part of the logical NIC output interface. To enable HQoS on specific output
+interface, vpp startup.conf file has to be configured accordingly. The output
+interface that requires HQoS, should have "hqos" parameter specified in dpdk
+section. Another optional parameter "hqos-thread"  has been defined which can
+be used to associate the output interface with specific hqos thread. In cpu
+section of the config file, "corelist-hqos-threads" is introduced to assign
+logical cpu cores to run the HQoS threads. A HQoS thread can run multiple HQoS
+objects each associated with different output interfaces. All worker threads
+instead of writing packets to NIC TX queue directly, write the packets to a
+software queues. The hqos_threads read the software queues, and enqueue the
+packets to HQoS objects, as well as dequeue packets from HQOS objects and
+write them to NIC output interfaces. The worker threads need to be able to
+send the packets to any output interface, therefore, each HQoS object
+associated with NIC output interface should have software queues equal to
+worker threads count.
+
+Following illustrates the sample startup configuration file with 4x worker
+threads feeding 2x hqos threads that handle each QoS scheduler for 1x output
+interface.
+
+```
+dpdk {
+  socket-mem 16384,16384
+
+  dev 0000:02:00.0 {
+    num-rx-queues 2
+    hqos
+  }
+  dev 0000:06:00.0 {
+    num-rx-queues 2
+    hqos
+  }
+
+  num-mbufs 1000000
+}
+
+cpu {
+  main-core 0
+  corelist-workers  1, 2, 3, 4
+  corelist-hqos-threads  5, 6
+}
+```
+
+
+### QoS scheduler CLI Commands
+
+Each QoS scheduler instance is initialised with default parameters required to
+configure hqos port, subport, pipe and queues. Some of the parameters can be
+re-configured in run-time through CLI commands.
+
+
+#### Configuration
+
+Following commands can be used to configure QoS scheduler parameters.
+
+The command below can be used to set the subport level parameters such as
+token bucket rate (bytes per seconds), token bucket size (bytes), traffic
+class rates (bytes per seconds) and token update period (Milliseconds).
+
+```
+set dpdk interface hqos subport  subport  [rate ]
+    [bktsize ] [tc0 ] [tc1 ] [tc2 ] [tc3 ] [period ]
+```
+
+For setting the pipe profile, following command can be used.    
+
+```
+set dpdk interface hqos pipe  subport  pipe 
+    profile 
+```
+
+To assign QoS scheduler instance to the specific thread, following command can
+be used.
+
+```
+set dpdk interface hqos placement  thread 
+```
+
+The command below is used to set the packet fields required for classifiying
+the incoming packet. As a result of classification process,     packet field
+information will be mapped to 5 tuples (subport, pipe, traffic class, pipe,
+color) and stored in packet mbuf.
+
+```
+set dpdk interface hqos pktfield  id subport|pipe|tc offset 
+    mask 
+```
+
+The DSCP table entries used for idenfiying the traffic class and queue can be set using the command below;   
+
+```  
+set dpdk interface hqos tctbl  entry  tc  queue 
+```
+
+
+#### Show Command
+
+The QoS Scheduler configuration can displayed using the command below.
+
+```
+   vpp# show dpdk interface hqos TenGigabitEthernet2/0/0
+   Thread:
+     Input SWQ size = 4096 packets
+     Enqueue burst size = 256 packets
+     Dequeue burst size = 220 packets
+     Packet field 0: slab position =    0, slab bitmask = 0x0000000000000000   (subport)
+     Packet field 1: slab position =   40, slab bitmask = 0x0000000fff000000   (pipe)
+     Packet field 2: slab position =    8, slab bitmask = 0x00000000000000fc   (tc)
+     Packet field 2  tc translation table: ([Mapped Value Range]: tc/queue tc/queue ...)
+     [ 0 .. 15]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+     [16 .. 31]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+     [32 .. 47]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+     [48 .. 63]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
+   Port:
+     Rate = 1250000000 bytes/second
+     MTU = 1514 bytes
+     Frame overhead = 24 bytes
+     Number of subports = 1
+     Number of pipes per subport = 4096
+     Packet queue size: TC0 = 64, TC1 = 64, TC2 = 64, TC3 = 64 packets
+     Number of pipe profiles = 1
+   Subport 0:
+     Rate = 120000000 bytes/second
+     Token bucket size = 1000000 bytes
+     Traffic class rate: TC0 = 120000000, TC1 = 120000000, TC2 = 120000000, TC3 = 120000000 bytes/second
+     TC period = 10 milliseconds
+   Pipe profile 0:
+     Rate = 305175 bytes/second
+     Token bucket size = 1000000 bytes
+     Traffic class rate: TC0 = 305175, TC1 = 305175, TC2 = 305175, TC3 = 305175 bytes/second
+     TC period = 40 milliseconds
+     TC0 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+     TC1 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+     TC2 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+     TC3 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
+```
+
+The QoS Scheduler placement over the logical cpu cores can be displayed using
+below command.
+
+```
+    vpp# show dpdk interface hqos placement
+    Thread 5 (vpp_hqos-threads_0 at lcore 5):
+      TenGigabitEthernet2/0/0 queue 0
+    Thread 6 (vpp_hqos-threads_1 at lcore 6):
+      TenGigabitEthernet4/0/1 queue 0
+```
+
+
+### QoS Scheduler Binary APIs
+
+This section explans the available binary APIs for configuring QoS scheduler
+parameters in run-time.
+
+The following API can be used to set the pipe profile of a pipe that belongs
+to a given subport:
+
+```
+sw_interface_set_dpdk_hqos_pipe rx  | sw_if_index 
+    subport  pipe  profile 
+```
+
+The data structures used for set the pipe profile parameter are as follows;
+
+```
+  /** \\brief DPDK interface HQoS pipe profile set request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - the interface
+    @param subport - subport ID
+    @param pipe - pipe ID within its subport
+    @param profile - pipe profile ID
+  */
+  define sw_interface_set_dpdk_hqos_pipe {
+    u32 client_index;
+    u32 context;
+    u32 sw_if_index;
+    u32 subport;
+    u32 pipe;
+    u32 profile;
+  };
+
+  /** \\brief DPDK interface HQoS pipe profile set reply
+    @param context - sender context, to match reply w/ request
+    @param retval - request return code
+  */
+  define sw_interface_set_dpdk_hqos_pipe_reply {
+    u32 context;
+    i32 retval;
+  };
+```
+
+The following API can be used to set the subport level parameters, for
+example- token bucket rate (bytes per seconds), token bucket size (bytes),
+traffic class rate (bytes per seconds) and tokens update period.
+
+```
+sw_interface_set_dpdk_hqos_subport rx  | sw_if_index 
+    subport  [rate ] [bktsize ]
+    [tc0 ] [tc1 ] [tc2 ] [tc3 ] [period ]
+```
+
+The data structures used for set the subport level parameter are as follows;
+
+```
+  /** \\brief DPDK interface HQoS subport parameters set request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - the interface
+    @param subport - subport ID
+    @param tb_rate - subport token bucket rate (measured in bytes/second)
+    @param tb_size - subport token bucket size (measured in credits)
+    @param tc_rate - subport traffic class 0 .. 3 rates (measured in bytes/second)
+    @param tc_period - enforcement period for rates (measured in milliseconds)
+  */
+  define sw_interface_set_dpdk_hqos_subport {
+    u32 client_index;
+    u32 context;
+    u32 sw_if_index;
+    u32 subport;
+    u32 tb_rate;
+    u32 tb_size;
+    u32 tc_rate[4];
+    u32 tc_period;
+  };
+
+  /** \\brief DPDK interface HQoS subport parameters set reply
+    @param context - sender context, to match reply w/ request
+    @param retval - request return code
+  */
+  define sw_interface_set_dpdk_hqos_subport_reply {
+    u32 context;
+    i32 retval;
+  };
+```
+
+The following API can be used set the DSCP table entry. The DSCP table have
+64 entries to map the packet DSCP field onto traffic class and hqos input
+queue.
+
+```
+sw_interface_set_dpdk_hqos_tctbl rx  | sw_if_index  
+    entry  tc  queue 
+```
+
+The data structures used for setting DSCP table entries are given below.
+
+```
+  /** \\brief DPDK interface HQoS tctbl entry set request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - the interface
+    @param entry - entry index ID
+    @param tc - traffic class (0 .. 3)
+    @param queue - traffic class queue (0 .. 3)
+  */
+  define sw_interface_set_dpdk_hqos_tctbl {
+    u32 client_index;
+    u32 context;
+    u32 sw_if_index;
+    u32 entry;
+    u32 tc;
+    u32 queue;
+  };
+
+  /** \\brief DPDK interface HQoS tctbl entry set reply
+    @param context - sender context, to match reply w/ request
+    @param retval - request return code
+  */
+  define sw_interface_set_dpdk_hqos_tctbl_reply {
+    u32 context;
+    i32 retval;
+  };
+```
diff --git a/src/plugins/dpdk/init.c b/src/plugins/dpdk/init.c
new file mode 100755
index 00000000..e009ef3e
--- /dev/null
+++ b/src/plugins/dpdk/init.c
@@ -0,0 +1,2074 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+dpdk_main_t dpdk_main;
+
+#include 
+#include 
+
+/* define message IDs */
+#include 
+
+#define vl_typedefs		/* define message structures */
+#include 
+#undef vl_typedefs
+
+#define vl_endianfun		/* define message structures */
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+/* Macro to finish up custom dump fns */
+#define FINISH                                  \
+    vec_add1 (s, 0);                            \
+    vl_print (handle, (char *)s);               \
+    vec_free (s);                               \
+    return handle;
+
+#include 
+
+static void
+  vl_api_sw_interface_set_dpdk_hqos_pipe_t_handler
+  (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp)
+{
+  vl_api_sw_interface_set_dpdk_hqos_pipe_reply_t *rmp;
+  int rv = 0;
+
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd;
+
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  u32 subport = ntohl (mp->subport);
+  u32 pipe = ntohl (mp->pipe);
+  u32 profile = ntohl (mp->profile);
+  vnet_hw_interface_t *hw;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  /* hw_if & dpdk device */
+  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
+
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rv = rte_sched_pipe_config (xd->hqos_ht->hqos, subport, pipe, profile);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY);
+}
+
+static void *vl_api_sw_interface_set_dpdk_hqos_pipe_t_print
+  (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_pipe ");
+
+  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
+
+  s = format (s, "subport %u  pipe %u  profile %u ",
+	      ntohl (mp->subport), ntohl (mp->pipe), ntohl (mp->profile));
+
+  FINISH;
+}
+
+static void
+  vl_api_sw_interface_set_dpdk_hqos_subport_t_handler
+  (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp)
+{
+  vl_api_sw_interface_set_dpdk_hqos_subport_reply_t *rmp;
+  int rv = 0;
+
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd;
+  struct rte_sched_subport_params p;
+
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  u32 subport = ntohl (mp->subport);
+  p.tb_rate = ntohl (mp->tb_rate);
+  p.tb_size = ntohl (mp->tb_size);
+  p.tc_rate[0] = ntohl (mp->tc_rate[0]);
+  p.tc_rate[1] = ntohl (mp->tc_rate[1]);
+  p.tc_rate[2] = ntohl (mp->tc_rate[2]);
+  p.tc_rate[3] = ntohl (mp->tc_rate[3]);
+  p.tc_period = ntohl (mp->tc_period);
+
+  vnet_hw_interface_t *hw;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  /* hw_if & dpdk device */
+  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
+
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport, &p);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY);
+}
+
+static void *vl_api_sw_interface_set_dpdk_hqos_subport_t_print
+  (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_subport ");
+
+  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
+
+  s =
+    format (s,
+	    "subport %u  rate %u  bkt_size %u  tc0 %u tc1 %u tc2 %u tc3 %u period %u",
+	    ntohl (mp->subport), ntohl (mp->tb_rate), ntohl (mp->tb_size),
+	    ntohl (mp->tc_rate[0]), ntohl (mp->tc_rate[1]),
+	    ntohl (mp->tc_rate[2]), ntohl (mp->tc_rate[3]),
+	    ntohl (mp->tc_period));
+
+  FINISH;
+}
+
+static void
+  vl_api_sw_interface_set_dpdk_hqos_tctbl_t_handler
+  (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp)
+{
+  vl_api_sw_interface_set_dpdk_hqos_tctbl_reply_t *rmp;
+  int rv = 0;
+
+  dpdk_main_t *dm = &dpdk_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_device_t *xd;
+
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  u32 entry = ntohl (mp->entry);
+  u32 tc = ntohl (mp->tc);
+  u32 queue = ntohl (mp->queue);
+  u32 val, i;
+
+  vnet_hw_interface_t *hw;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  /* hw_if & dpdk device */
+  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
+
+  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
+
+  if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
+    {
+      clib_warning ("invalid traffic class !!");
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+  if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS)
+    {
+      clib_warning ("invalid queue !!");
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  /* Detect the set of worker threads */
+  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+
+  if (p == 0)
+    {
+      clib_warning ("worker thread registration AWOL !!");
+      rv = VNET_API_ERROR_INVALID_VALUE_2;
+      goto done;
+    }
+
+  vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
+  int worker_thread_first = tr->first_index;
+  int worker_thread_count = tr->count;
+
+  val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue;
+  for (i = 0; i < worker_thread_count; i++)
+    xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val;
+
+  BAD_SW_IF_INDEX_LABEL;
+done:
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY);
+}
+
+static void *vl_api_sw_interface_set_dpdk_hqos_tctbl_t_print
+  (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_tctbl ");
+
+  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
+
+  s = format (s, "entry %u  tc %u  queue %u",
+	      ntohl (mp->entry), ntohl (mp->tc), ntohl (mp->queue));
+
+  FINISH;
+}
+
+#define foreach_dpdk_plugin_api_msg                                       \
+_(SW_INTERFACE_SET_DPDK_HQOS_PIPE, sw_interface_set_dpdk_hqos_pipe)       \
+_(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, sw_interface_set_dpdk_hqos_subport) \
+_(SW_INTERFACE_SET_DPDK_HQOS_TCTBL, sw_interface_set_dpdk_hqos_tctbl)
+
+/* Set up the API message handling tables */
+static clib_error_t *
+dpdk_plugin_api_hookup (vlib_main_t * vm)
+{
+  dpdk_main_t *dm __attribute__ ((unused)) = &dpdk_main;
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + dm->msg_id_base),     \
+                           #n,          \
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_dpdk_plugin_api_msg;
+#undef _
+  return 0;
+}
+
+#define vl_msg_name_crc_list
+#include 
+#undef vl_msg_name_crc_list
+
+static void
+setup_message_id_table (dpdk_main_t * dm, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + dm->msg_id_base);
+  foreach_vl_msg_name_crc_dpdk;
+#undef _
+}
+
+//  TODO
+/*
+static void plugin_custom_dump_configure (dpdk_main_t * dm)
+{
+#define _(n,f) dm->api_main->msg_print_handlers \
+  [VL_API_##n + dm->msg_id_base]                \
+    = (void *) vl_api_##f##_t_print;
+  foreach_dpdk_plugin_api_msg;
+#undef _
+}
+*/
+/* force linker to link functions used by vlib and declared weak */
+void *vlib_weakly_linked_functions[] = {
+  &rte_pktmbuf_init,
+  &rte_pktmbuf_pool_init,
+};
+
+#define LINK_STATE_ELOGS	0
+
+#define DEFAULT_HUGE_DIR "/run/vpp/hugepages"
+#define VPP_RUN_DIR "/run/vpp"
+
+/* Port configuration, mildly modified Intel app values */
+
+static struct rte_eth_conf port_conf_template = {
+  .rxmode = {
+	     .split_hdr_size = 0,
+	     .header_split = 0,		/**< Header Split disabled */
+	     .hw_ip_checksum = 0,	/**< IP checksum offload disabled */
+	     .hw_vlan_filter = 0,	/**< VLAN filtering disabled */
+	     .hw_strip_crc = 0,		/**< CRC stripped by hardware */
+	     },
+  .txmode = {
+	     .mq_mode = ETH_MQ_TX_NONE,
+	     },
+};
+
+clib_error_t *
+dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd)
+{
+  int rv;
+  int j;
+
+  ASSERT (os_get_cpu_number () == 0);
+
+  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
+    {
+      vnet_hw_interface_set_flags (dm->vnet_main, xd->vlib_hw_if_index, 0);
+      rte_eth_dev_stop (xd->device_index);
+    }
+
+  rv = rte_eth_dev_configure (xd->device_index, xd->rx_q_used,
+			      xd->tx_q_used, &xd->port_conf);
+
+  if (rv < 0)
+    return clib_error_return (0, "rte_eth_dev_configure[%d]: err %d",
+			      xd->device_index, rv);
+
+  /* Set up one TX-queue per worker thread */
+  for (j = 0; j < xd->tx_q_used; j++)
+    {
+      rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
+				   xd->cpu_socket, &xd->tx_conf);
+
+      /* retry with any other CPU socket */
+      if (rv < 0)
+	rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
+				     SOCKET_ID_ANY, &xd->tx_conf);
+      if (rv < 0)
+	break;
+    }
+
+  if (rv < 0)
+    return clib_error_return (0, "rte_eth_tx_queue_setup[%d]: err %d",
+			      xd->device_index, rv);
+
+  for (j = 0; j < xd->rx_q_used; j++)
+    {
+
+      rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
+				   xd->cpu_socket, 0,
+				   dm->
+				   pktmbuf_pools[xd->cpu_socket_id_by_queue
+						 [j]]);
+
+      /* retry with any other CPU socket */
+      if (rv < 0)
+	rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
+				     SOCKET_ID_ANY, 0,
+				     dm->
+				     pktmbuf_pools[xd->cpu_socket_id_by_queue
+						   [j]]);
+      if (rv < 0)
+	return clib_error_return (0, "rte_eth_rx_queue_setup[%d]: err %d",
+				  xd->device_index, rv);
+    }
+
+  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
+    {
+      int rv;
+      rv = rte_eth_dev_start (xd->device_index);
+      if (!rv && xd->default_mac_address)
+	rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
+					       (struct ether_addr *)
+					       xd->default_mac_address);
+      if (rv < 0)
+	clib_warning ("rte_eth_dev_start %d returned %d",
+		      xd->device_index, rv);
+    }
+  return 0;
+}
+
+static u32
+dpdk_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
+  u32 old = 0;
+
+  if (ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC (flags))
+    {
+      old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
+
+      if (flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL)
+	xd->flags |= DPDK_DEVICE_FLAG_PROMISC;
+      else
+	xd->flags &= ~DPDK_DEVICE_FLAG_PROMISC;
+
+      if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
+	{
+	  if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
+	    rte_eth_promiscuous_enable (xd->device_index);
+	  else
+	    rte_eth_promiscuous_disable (xd->device_index);
+	}
+    }
+  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
+    {
+      /*
+       * DAW-FIXME: The Cisco VIC firmware does not provide an api for a
+       *            driver to dynamically change the mtu.  If/when the
+       *            VIC firmware gets fixed, then this should be removed.
+       */
+      if (xd->pmd == VNET_DPDK_PMD_ENIC)
+	{
+	  struct rte_eth_dev_info dev_info;
+
+	  /*
+	   * Restore mtu to what has been set by CIMC in the firmware cfg.
+	   */
+	  rte_eth_dev_info_get (xd->device_index, &dev_info);
+	  hi->max_packet_bytes = dev_info.max_rx_pktlen;
+
+	  vlib_cli_output (vlib_get_main (),
+			   "Cisco VIC mtu can only be changed "
+			   "using CIMC then rebooting the server!");
+	}
+      else
+	{
+	  int rv;
+
+	  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
+
+	  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
+	    rte_eth_dev_stop (xd->device_index);
+
+	  rv = rte_eth_dev_configure
+	    (xd->device_index, xd->rx_q_used, xd->tx_q_used, &xd->port_conf);
+
+	  if (rv < 0)
+	    vlib_cli_output (vlib_get_main (),
+			     "rte_eth_dev_configure[%d]: err %d",
+			     xd->device_index, rv);
+
+	  rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
+
+	  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
+	    {
+	      int rv = rte_eth_dev_start (xd->device_index);
+	      if (!rv && xd->default_mac_address)
+		rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
+						       (struct ether_addr *)
+						       xd->default_mac_address);
+	      if (rv < 0)
+		clib_warning ("rte_eth_dev_start %d returned %d",
+			      xd->device_index, rv);
+	    }
+	}
+    }
+  return old;
+}
+
+void
+dpdk_device_lock_init (dpdk_device_t * xd)
+{
+  int q;
+  vec_validate (xd->lockp, xd->tx_q_used - 1);
+  for (q = 0; q < xd->tx_q_used; q++)
+    {
+      xd->lockp[q] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+					     CLIB_CACHE_LINE_BYTES);
+      memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
+    }
+}
+
+void
+dpdk_device_lock_free (dpdk_device_t * xd)
+{
+  int q;
+
+  for (q = 0; q < vec_len (xd->lockp); q++)
+    clib_mem_free ((void *) xd->lockp[q]);
+  vec_free (xd->lockp);
+  xd->lockp = 0;
+}
+
+static clib_error_t *
+dpdk_lib_init (dpdk_main_t * dm)
+{
+  u32 nports;
+  u32 nb_desc = 0;
+  int i;
+  clib_error_t *error;
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  vnet_sw_interface_t *sw;
+  vnet_hw_interface_t *hi;
+  dpdk_device_t *xd;
+  vlib_pci_addr_t last_pci_addr;
+  u32 last_pci_addr_port = 0;
+  vlib_thread_registration_t *tr, *tr_hqos;
+  uword *p, *p_hqos;
+
+  u32 next_cpu = 0, next_hqos_cpu = 0;
+  u8 af_packet_port_id = 0;
+  last_pci_addr.as_u32 = ~0;
+
+  dm->input_cpu_first_index = 0;
+  dm->input_cpu_count = 1;
+
+  /* find out which cpus will be used for input */
+  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
+
+  if (tr && tr->count > 0)
+    {
+      dm->input_cpu_first_index = tr->first_index;
+      dm->input_cpu_count = tr->count;
+    }
+
+  vec_validate_aligned (dm->devices_by_cpu, tm->n_vlib_mains - 1,
+			CLIB_CACHE_LINE_BYTES);
+
+  dm->hqos_cpu_first_index = 0;
+  dm->hqos_cpu_count = 0;
+
+  /* find out which cpus will be used for I/O TX */
+  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
+  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
+
+  if (tr_hqos && tr_hqos->count > 0)
+    {
+      dm->hqos_cpu_first_index = tr_hqos->first_index;
+      dm->hqos_cpu_count = tr_hqos->count;
+    }
+
+  vec_validate_aligned (dm->devices_by_hqos_cpu, tm->n_vlib_mains - 1,
+			CLIB_CACHE_LINE_BYTES);
+
+  nports = rte_eth_dev_count ();
+  if (nports < 1)
+    {
+      clib_warning ("DPDK drivers found no ports...");
+    }
+
+  if (CLIB_DEBUG > 0)
+    clib_warning ("DPDK drivers found %d ports...", nports);
+
+  /*
+   * All buffers are all allocated from the same rte_mempool.
+   * Thus they all have the same number of data bytes.
+   */
+  dm->vlib_buffer_free_list_index =
+    vlib_buffer_get_or_create_free_list (vm,
+					 VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
+					 "dpdk rx");
+
+  if (dm->conf->enable_tcp_udp_checksum)
+    dm->buffer_flags_template &= ~(IP_BUFFER_L4_CHECKSUM_CORRECT
+				   | IP_BUFFER_L4_CHECKSUM_COMPUTED);
+
+  for (i = 0; i < nports; i++)
+    {
+      u8 addr[6];
+      u8 vlan_strip = 0;
+      int j;
+      struct rte_eth_dev_info dev_info;
+      clib_error_t *rv;
+      struct rte_eth_link l;
+      dpdk_device_config_t *devconf = 0;
+      vlib_pci_addr_t pci_addr;
+      uword *p = 0;
+
+      rte_eth_dev_info_get (i, &dev_info);
+      if (dev_info.pci_dev)	/* bonded interface has no pci info */
+	{
+	  pci_addr.domain = dev_info.pci_dev->addr.domain;
+	  pci_addr.bus = dev_info.pci_dev->addr.bus;
+	  pci_addr.slot = dev_info.pci_dev->addr.devid;
+	  pci_addr.function = dev_info.pci_dev->addr.function;
+	  p =
+	    hash_get (dm->conf->device_config_index_by_pci_addr,
+		      pci_addr.as_u32);
+	}
+
+      if (p)
+	devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
+      else
+	devconf = &dm->conf->default_devconf;
+
+      /* Create vnet interface */
+      vec_add2_aligned (dm->devices, xd, 1, CLIB_CACHE_LINE_BYTES);
+      xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT;
+      xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT;
+      xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
+
+      /* Handle interface naming for devices with multiple ports sharing same PCI ID */
+      if (dev_info.pci_dev)
+	{
+	  struct rte_eth_dev_info di = { 0 };
+	  rte_eth_dev_info_get (i + 1, &di);
+	  if (di.pci_dev && pci_addr.as_u32 != last_pci_addr.as_u32 &&
+	      memcmp (&dev_info.pci_dev->addr, &di.pci_dev->addr,
+		      sizeof (struct rte_pci_addr)) == 0)
+	    {
+	      xd->interface_name_suffix = format (0, "0");
+	      last_pci_addr.as_u32 = pci_addr.as_u32;
+	      last_pci_addr_port = i;
+	    }
+	  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
+	    {
+	      xd->interface_name_suffix =
+		format (0, "%u", i - last_pci_addr_port);
+	    }
+	  else
+	    {
+	      last_pci_addr.as_u32 = ~0;
+	    }
+	}
+      else
+	last_pci_addr.as_u32 = ~0;
+
+      clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
+		   sizeof (struct rte_eth_txconf));
+      if (dm->conf->no_multi_seg)
+	{
+	  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
+	  port_conf_template.rxmode.jumbo_frame = 0;
+	}
+      else
+	{
+	  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
+	  port_conf_template.rxmode.jumbo_frame = 1;
+	  xd->flags |= DPDK_DEVICE_FLAG_MAYBE_MULTISEG;
+	}
+
+      clib_memcpy (&xd->port_conf, &port_conf_template,
+		   sizeof (struct rte_eth_conf));
+
+      xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
+
+      if (devconf->num_tx_queues > 0
+	  && devconf->num_tx_queues < xd->tx_q_used)
+	xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
+
+      if (devconf->num_rx_queues > 1 && dm->use_rss == 0)
+	{
+	  dm->use_rss = 1;
+	}
+
+      if (devconf->num_rx_queues > 1
+	  && dev_info.max_rx_queues >= devconf->num_rx_queues)
+	{
+	  xd->rx_q_used = devconf->num_rx_queues;
+	  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
+	  if (devconf->rss_fn == 0)
+	    xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
+	      ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
+	  else
+	    xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
+	}
+      else
+	xd->rx_q_used = 1;
+
+      xd->flags |= DPDK_DEVICE_FLAG_PMD;
+
+      /* workaround for drivers not setting driver_name */
+      if ((!dev_info.driver_name) && (dev_info.pci_dev))
+	dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
+
+      ASSERT (dev_info.driver_name);
+
+      if (!xd->pmd)
+	{
+
+
+#define _(s,f) else if (dev_info.driver_name &&                 \
+                        !strcmp(dev_info.driver_name, s))       \
+                 xd->pmd = VNET_DPDK_PMD_##f;
+	  if (0)
+	    ;
+	  foreach_dpdk_pmd
+#undef _
+	    else
+	    xd->pmd = VNET_DPDK_PMD_UNKNOWN;
+
+	  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
+	  xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT;
+	  xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT;
+
+	  switch (xd->pmd)
+	    {
+	      /* 1G adapters */
+	    case VNET_DPDK_PMD_E1000EM:
+	    case VNET_DPDK_PMD_IGB:
+	    case VNET_DPDK_PMD_IGBVF:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G;
+	      break;
+
+	      /* 10G adapters */
+	    case VNET_DPDK_PMD_IXGBE:
+	    case VNET_DPDK_PMD_IXGBEVF:
+	    case VNET_DPDK_PMD_THUNDERX:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+	      break;
+	    case VNET_DPDK_PMD_DPAA2:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+	      break;
+
+	      /* Cisco VIC */
+	    case VNET_DPDK_PMD_ENIC:
+	      rte_eth_link_get_nowait (i, &l);
+	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
+	      if (l.link_speed == 40000)
+		xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
+	      else
+		xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+	      break;
+
+	      /* Intel Fortville */
+	    case VNET_DPDK_PMD_I40E:
+	    case VNET_DPDK_PMD_I40EVF:
+	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
+
+	      switch (dev_info.pci_dev->id.device_id)
+		{
+		case I40E_DEV_ID_10G_BASE_T:
+		case I40E_DEV_ID_SFP_XL710:
+		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+		  break;
+		case I40E_DEV_ID_QSFP_A:
+		case I40E_DEV_ID_QSFP_B:
+		case I40E_DEV_ID_QSFP_C:
+		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
+		  break;
+		case I40E_DEV_ID_VF:
+		  rte_eth_link_get_nowait (i, &l);
+		  xd->port_type = l.link_speed == 10000 ?
+		    VNET_DPDK_PORT_TYPE_ETH_10G : VNET_DPDK_PORT_TYPE_ETH_40G;
+		  break;
+		default:
+		  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
+		}
+	      break;
+
+	    case VNET_DPDK_PMD_CXGBE:
+	      switch (dev_info.pci_dev->id.device_id)
+		{
+		case 0x540d:	/* T580-CR */
+		case 0x5410:	/* T580-LP-cr */
+		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
+		  break;
+		case 0x5403:	/* T540-CR */
+		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+		  break;
+		default:
+		  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
+		}
+	      break;
+
+	    case VNET_DPDK_PMD_MLX5:
+	      {
+		char *pn_100g[] = { "MCX415A-CCAT", "MCX416A-CCAT", 0 };
+		char *pn_40g[] = { "MCX413A-BCAT", "MCX414A-BCAT",
+		  "MCX415A-BCAT", "MCX416A-BCAT", "MCX4131A-BCAT", 0
+		};
+		char *pn_10g[] = { "MCX4111A-XCAT", "MCX4121A-XCAT", 0 };
+
+		vlib_pci_device_t *pd = vlib_get_pci_device (&pci_addr);
+		u8 *pn = 0;
+		char **c;
+		int found = 0;
+		pn = format (0, "%U%c",
+			     format_vlib_pci_vpd, pd->vpd_r, "PN", 0);
+
+		if (!pn)
+		  break;
+
+		c = pn_100g;
+		while (!found && c[0])
+		  {
+		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
+		      {
+			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_100G;
+			break;
+		      }
+		    c++;
+		  }
+
+		c = pn_40g;
+		while (!found && c[0])
+		  {
+		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
+		      {
+			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
+			break;
+		      }
+		    c++;
+		  }
+
+		c = pn_10g;
+		while (!found && c[0])
+		  {
+		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
+		      {
+			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+			break;
+		      }
+		    c++;
+		  }
+
+		vec_free (pn);
+	      }
+
+	      break;
+	      /* Intel Red Rock Canyon */
+	    case VNET_DPDK_PMD_FM10K:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_SWITCH;
+	      break;
+
+	      /* virtio */
+	    case VNET_DPDK_PMD_VIRTIO:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G;
+	      xd->nb_rx_desc = DPDK_NB_RX_DESC_VIRTIO;
+	      xd->nb_tx_desc = DPDK_NB_TX_DESC_VIRTIO;
+	      break;
+
+	      /* vmxnet3 */
+	    case VNET_DPDK_PMD_VMXNET3:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G;
+	      xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
+	      break;
+
+	    case VNET_DPDK_PMD_AF_PACKET:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_AF_PACKET;
+	      xd->af_packet_port_id = af_packet_port_id++;
+	      break;
+
+	    case VNET_DPDK_PMD_BOND:
+	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
+	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_BOND;
+	      break;
+
+	    default:
+	      xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
+	    }
+
+	  if (devconf->num_rx_desc)
+	    xd->nb_rx_desc = devconf->num_rx_desc;
+
+	  if (devconf->num_tx_desc)
+	    xd->nb_tx_desc = devconf->num_tx_desc;
+	}
+
+      /*
+       * Ensure default mtu is not > the mtu read from the hardware.
+       * Otherwise rte_eth_dev_configure() will fail and the port will
+       * not be available.
+       */
+      if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
+	{
+	  /*
+	   * This device does not support the platforms's max frame
+	   * size. Use it's advertised mru instead.
+	   */
+	  xd->port_conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen;
+	}
+      else
+	{
+	  xd->port_conf.rxmode.max_rx_pkt_len = ETHERNET_MAX_PACKET_BYTES;
+
+	  /*
+	   * Some platforms do not account for Ethernet FCS (4 bytes) in
+	   * MTU calculations. To interop with them increase mru but only
+	   * if the device's settings can support it.
+	   */
+	  if ((dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)) &&
+	      xd->port_conf.rxmode.hw_strip_crc)
+	    {
+	      /*
+	       * Allow additional 4 bytes (for Ethernet FCS). These bytes are
+	       * stripped by h/w and so will not consume any buffer memory.
+	       */
+	      xd->port_conf.rxmode.max_rx_pkt_len += 4;
+	    }
+	}
+
+      if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
+	{
+	  f64 now = vlib_time_now (vm);
+	  u32 rnd;
+	  rnd = (u32) (now * 1e6);
+	  rnd = random_u32 (&rnd);
+	  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
+	  addr[0] = 2;
+	  addr[1] = 0xfe;
+	}
+      else
+	rte_eth_macaddr_get (i, (struct ether_addr *) addr);
+
+      if (xd->tx_q_used < tm->n_vlib_mains)
+	dpdk_device_lock_init (xd);
+
+      xd->device_index = xd - dm->devices;
+      ASSERT (i == xd->device_index);
+      xd->per_interface_next_index = ~0;
+
+      /* assign interface to input thread */
+      dpdk_device_and_queue_t *dq;
+      int q;
+
+      if (devconf->workers)
+	{
+	  int i;
+	  q = 0;
+	  /* *INDENT-OFF* */
+	  clib_bitmap_foreach (i, devconf->workers, ({
+	    int cpu = dm->input_cpu_first_index + i;
+	    unsigned lcore = vlib_worker_threads[cpu].lcore_id;
+	    vec_validate(xd->cpu_socket_id_by_queue, q);
+	    xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id(lcore);
+	    vec_add2(dm->devices_by_cpu[cpu], dq, 1);
+	    dq->device = xd->device_index;
+	    dq->queue_id = q++;
+	  }));
+	  /* *INDENT-ON* */
+	}
+      else
+	for (q = 0; q < xd->rx_q_used; q++)
+	  {
+	    int cpu = dm->input_cpu_first_index + next_cpu;
+	    unsigned lcore = vlib_worker_threads[cpu].lcore_id;
+
+	    /*
+	     * numa node for worker thread handling this queue
+	     * needed for taking buffers from the right mempool
+	     */
+	    vec_validate (xd->cpu_socket_id_by_queue, q);
+	    xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id (lcore);
+
+	    /*
+	     * construct vector of (device,queue) pairs for each worker thread
+	     */
+	    vec_add2 (dm->devices_by_cpu[cpu], dq, 1);
+	    dq->device = xd->device_index;
+	    dq->queue_id = q;
+
+	    next_cpu++;
+	    if (next_cpu == dm->input_cpu_count)
+	      next_cpu = 0;
+	  }
+
+
+      if (devconf->hqos_enabled)
+	{
+	  xd->flags |= DPDK_DEVICE_FLAG_HQOS;
+
+	  if (devconf->hqos.hqos_thread_valid)
+	    {
+	      int cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
+
+	      if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
+		return clib_error_return (0, "invalid HQoS thread index");
+
+	      vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
+	      dq->device = xd->device_index;
+	      dq->queue_id = 0;
+	    }
+	  else
+	    {
+	      int cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
+
+	      if (dm->hqos_cpu_count == 0)
+		return clib_error_return (0, "no HQoS threads available");
+
+	      vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
+	      dq->device = xd->device_index;
+	      dq->queue_id = 0;
+
+	      next_hqos_cpu++;
+	      if (next_hqos_cpu == dm->hqos_cpu_count)
+		next_hqos_cpu = 0;
+
+	      devconf->hqos.hqos_thread_valid = 1;
+	      devconf->hqos.hqos_thread = cpu;
+	    }
+	}
+
+      vec_validate_aligned (xd->tx_vectors, tm->n_vlib_mains,
+			    CLIB_CACHE_LINE_BYTES);
+      for (j = 0; j < tm->n_vlib_mains; j++)
+	{
+	  vec_validate_ha (xd->tx_vectors[j], xd->nb_tx_desc,
+			   sizeof (tx_ring_hdr_t), CLIB_CACHE_LINE_BYTES);
+	  vec_reset_length (xd->tx_vectors[j]);
+	}
+
+      vec_validate_aligned (xd->rx_vectors, xd->rx_q_used,
+			    CLIB_CACHE_LINE_BYTES);
+      for (j = 0; j < xd->rx_q_used; j++)
+	{
+	  vec_validate_aligned (xd->rx_vectors[j], VLIB_FRAME_SIZE - 1,
+				CLIB_CACHE_LINE_BYTES);
+	  vec_reset_length (xd->rx_vectors[j]);
+	}
+
+      vec_validate_aligned (xd->d_trace_buffers, tm->n_vlib_mains,
+			    CLIB_CACHE_LINE_BYTES);
+
+      rv = dpdk_port_setup (dm, xd);
+
+      if (rv)
+	return rv;
+
+      if (devconf->hqos_enabled)
+	{
+	  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
+	  if (rv)
+	    return rv;
+	}
+
+      /* count the number of descriptors used for this device */
+      nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
+
+      error = ethernet_register_interface
+	(dm->vnet_main, dpdk_device_class.index, xd->device_index,
+	 /* ethernet address */ addr,
+	 &xd->vlib_hw_if_index, dpdk_flag_change);
+      if (error)
+	return error;
+
+      sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->vlib_hw_if_index);
+      xd->vlib_sw_if_index = sw->sw_if_index;
+      hi = vnet_get_hw_interface (dm->vnet_main, xd->vlib_hw_if_index);
+
+      /*
+       * DAW-FIXME: The Cisco VIC firmware does not provide an api for a
+       *            driver to dynamically change the mtu.  If/when the
+       *            VIC firmware gets fixed, then this should be removed.
+       */
+      if (xd->pmd == VNET_DPDK_PMD_ENIC)
+	{
+	  /*
+	   * Initialize mtu to what has been set by CIMC in the firmware cfg.
+	   */
+	  hi->max_packet_bytes = dev_info.max_rx_pktlen;
+	  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
+	    vlan_strip = 1;	/* remove vlan tag from VIC port by default */
+	  else
+	    clib_warning ("VLAN strip disabled for interface\n");
+	}
+      else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
+	vlan_strip = 1;
+
+      if (vlan_strip)
+	{
+	  int vlan_off;
+	  vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
+	  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
+	  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
+	  if (rte_eth_dev_set_vlan_offload (xd->device_index, vlan_off) == 0)
+	    clib_warning ("VLAN strip enabled for interface\n");
+	  else
+	    clib_warning ("VLAN strip cannot be supported by interface\n");
+	}
+
+      hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] =
+	xd->port_conf.rxmode.max_rx_pkt_len - sizeof (ethernet_header_t);
+
+      rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
+    }
+
+  if (nb_desc > dm->conf->num_mbufs)
+    clib_warning ("%d mbufs allocated but total rx/tx ring size is %d\n",
+		  dm->conf->num_mbufs, nb_desc);
+
+  return 0;
+}
+
+static void
+dpdk_bind_devices_to_uio (dpdk_config_main_t * conf)
+{
+  vlib_pci_main_t *pm = &pci_main;
+  clib_error_t *error;
+  vlib_pci_device_t *d;
+  u8 *pci_addr = 0;
+  int num_whitelisted = vec_len (conf->dev_confs);
+
+  /* *INDENT-OFF* */
+  pool_foreach (d, pm->pci_devs, ({
+    dpdk_device_config_t * devconf = 0;
+    vec_reset_length (pci_addr);
+    pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, &d->bus_address, 0);
+
+    if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && d->device_class != PCI_CLASS_PROCESSOR_CO)
+      continue;
+
+    if (num_whitelisted)
+      {
+	uword * p = hash_get (conf->device_config_index_by_pci_addr, d->bus_address.as_u32);
+
+	if (!p)
+	  continue;
+
+	devconf = pool_elt_at_index (conf->dev_confs, p[0]);
+      }
+
+    /* virtio */
+    if (d->vendor_id == 0x1af4 && d->device_id == 0x1000)
+      ;
+    /* vmxnet3 */
+    else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
+      ;
+    /* all Intel devices */
+    else if (d->vendor_id == 0x8086)
+      ;
+    /* Cisco VIC */
+    else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
+      ;
+    /* Chelsio T4/T5 */
+    else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
+      ;
+    else
+      {
+        clib_warning ("Unsupported Ethernet PCI device 0x%04x:0x%04x found "
+		      "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
+		      pci_addr);
+        continue;
+      }
+
+    error = vlib_pci_bind_to_uio (d, (char *) conf->uio_driver_name);
+
+    if (error)
+      {
+	if (devconf == 0)
+	  {
+	    pool_get (conf->dev_confs, devconf);
+	    hash_set (conf->device_config_index_by_pci_addr, d->bus_address.as_u32,
+		      devconf - conf->dev_confs);
+	    devconf->pci_addr.as_u32 = d->bus_address.as_u32;
+	  }
+	devconf->is_blacklisted = 1;
+	clib_error_report (error);
+      }
+  }));
+  /* *INDENT-ON* */
+  vec_free (pci_addr);
+}
+
+static clib_error_t *
+dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
+		    unformat_input_t * input, u8 is_default)
+{
+  clib_error_t *error = 0;
+  uword *p;
+  dpdk_device_config_t *devconf;
+  unformat_input_t sub_input;
+
+  if (is_default)
+    {
+      devconf = &conf->default_devconf;
+    }
+  else
+    {
+      p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
+
+      if (!p)
+	{
+	  pool_get (conf->dev_confs, devconf);
+	  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
+		    devconf - conf->dev_confs);
+	}
+      else
+	return clib_error_return (0,
+				  "duplicate configuration for PCI address %U",
+				  format_vlib_pci_addr, &pci_addr);
+    }
+
+  devconf->pci_addr.as_u32 = pci_addr.as_u32;
+  devconf->hqos_enabled = 0;
+  dpdk_device_config_hqos_default (&devconf->hqos);
+
+  if (!input)
+    return 0;
+
+  unformat_skip_white_space (input);
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
+	;
+      else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
+	;
+      else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
+	;
+      else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
+	;
+      else if (unformat (input, "workers %U", unformat_bitmap_list,
+			 &devconf->workers))
+	;
+      else
+	if (unformat
+	    (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
+	{
+	  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
+	  if (error)
+	    break;
+	}
+      else if (unformat (input, "vlan-strip-offload off"))
+	devconf->vlan_strip_offload = DPDK_DEVICE_VLAN_STRIP_OFF;
+      else if (unformat (input, "vlan-strip-offload on"))
+	devconf->vlan_strip_offload = DPDK_DEVICE_VLAN_STRIP_ON;
+      else
+	if (unformat
+	    (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
+	{
+	  devconf->hqos_enabled = 1;
+	  error = unformat_hqos (&sub_input, &devconf->hqos);
+	  if (error)
+	    break;
+	}
+      else if (unformat (input, "hqos"))
+	{
+	  devconf->hqos_enabled = 1;
+	}
+      else
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, input);
+	  break;
+	}
+    }
+
+  if (error)
+    return error;
+
+  if (devconf->workers && devconf->num_rx_queues == 0)
+    devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
+  else if (devconf->workers &&
+	   clib_bitmap_count_set_bits (devconf->workers) !=
+	   devconf->num_rx_queues)
+    error =
+      clib_error_return (0,
+			 "%U: number of worker threadds must be "
+			 "equal to number of rx queues", format_vlib_pci_addr,
+			 &pci_addr);
+
+  return error;
+}
+
+static clib_error_t *
+dpdk_config (vlib_main_t * vm, unformat_input_t * input)
+{
+  clib_error_t *error = 0;
+  dpdk_main_t *dm = &dpdk_main;
+  dpdk_config_main_t *conf = &dpdk_config_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  dpdk_device_config_t *devconf;
+  vlib_pci_addr_t pci_addr;
+  unformat_input_t sub_input;
+  u8 *s, *tmp = 0;
+  u8 *rte_cmd = 0, *ethname = 0;
+  u32 log_level;
+  int ret, i;
+  int num_whitelisted = 0;
+  u8 no_pci = 0;
+  u8 no_huge = 0;
+  u8 huge_dir = 0;
+  u8 file_prefix = 0;
+  u8 *socket_mem = 0;
+
+  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      /* Prime the pump */
+      if (unformat (input, "no-hugetlb"))
+	{
+	  vec_add1 (conf->eal_init_args, (u8 *) "no-huge");
+	  no_huge = 1;
+	}
+
+      else if (unformat (input, "enable-tcp-udp-checksum"))
+	conf->enable_tcp_udp_checksum = 1;
+
+      else if (unformat (input, "decimal-interface-names"))
+	conf->interface_name_format_decimal = 1;
+
+      else if (unformat (input, "no-multi-seg"))
+	conf->no_multi_seg = 1;
+
+      else if (unformat (input, "enable-cryptodev"))
+	conf->cryptodev = 1;
+
+      else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
+			 &sub_input))
+	{
+	  error =
+	    dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
+				1);
+
+	  if (error)
+	    return error;
+	}
+      else
+	if (unformat
+	    (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
+	     unformat_vlib_cli_sub_input, &sub_input))
+	{
+	  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
+
+	  if (error)
+	    return error;
+
+	  num_whitelisted++;
+	}
+      else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
+	{
+	  error = dpdk_device_config (conf, pci_addr, 0, 0);
+
+	  if (error)
+	    return error;
+
+	  num_whitelisted++;
+	}
+      else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
+	;
+      else if (unformat (input, "kni %d", &conf->num_kni))
+	;
+      else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
+	;
+      else if (unformat (input, "socket-mem %s", &socket_mem))
+	;
+      else if (unformat (input, "no-pci"))
+	{
+	  no_pci = 1;
+	  tmp = format (0, "--no-pci%c", 0);
+	  vec_add1 (conf->eal_init_args, tmp);
+	}
+      else if (unformat (input, "poll-sleep %d", &dm->poll_sleep))
+	;
+
+#define _(a)                                    \
+      else if (unformat(input, #a))             \
+        {                                       \
+          tmp = format (0, "--%s%c", #a, 0);    \
+          vec_add1 (conf->eal_init_args, tmp);    \
+        }
+      foreach_eal_double_hyphen_predicate_arg
+#undef _
+#define _(a)                                          \
+	else if (unformat(input, #a " %s", &s))	      \
+	  {					      \
+            if (!strncmp(#a, "huge-dir", 8))          \
+              huge_dir = 1;                           \
+            else if (!strncmp(#a, "file-prefix", 11)) \
+              file_prefix = 1;                        \
+	    tmp = format (0, "--%s%c", #a, 0);	      \
+	    vec_add1 (conf->eal_init_args, tmp);      \
+	    vec_add1 (s, 0);			      \
+            if (!strncmp(#a, "vdev", 4))              \
+              if (strstr((char*)s, "af_packet"))      \
+                clib_warning ("af_packet obsoleted. Use CLI 'create host-interface'."); \
+	    vec_add1 (conf->eal_init_args, s);	      \
+	  }
+	foreach_eal_double_hyphen_arg
+#undef _
+#define _(a,b)						\
+	  else if (unformat(input, #a " %s", &s))	\
+	    {						\
+	      tmp = format (0, "-%s%c", #b, 0);		\
+	      vec_add1 (conf->eal_init_args, tmp);	\
+	      vec_add1 (s, 0);				\
+	      vec_add1 (conf->eal_init_args, s);	\
+	    }
+	foreach_eal_single_hyphen_arg
+#undef _
+#define _(a,b)						\
+	    else if (unformat(input, #a " %s", &s))	\
+	      {						\
+		tmp = format (0, "-%s%c", #b, 0);	\
+		vec_add1 (conf->eal_init_args, tmp);	\
+		vec_add1 (s, 0);			\
+		vec_add1 (conf->eal_init_args, s);	\
+		conf->a##_set_manually = 1;		\
+	      }
+	foreach_eal_single_hyphen_mandatory_arg
+#undef _
+	else if (unformat (input, "default"))
+	;
+
+      else if (unformat_skip_white_space (input))
+	;
+      else
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, input);
+	  goto done;
+	}
+    }
+
+  if (!conf->uio_driver_name)
+    conf->uio_driver_name = format (0, "uio_pci_generic%c", 0);
+
+  /*
+   * Use 1G huge pages if available.
+   */
+  if (!no_huge && !huge_dir)
+    {
+      u32 x, *mem_by_socket = 0;
+      uword c = 0;
+      u8 use_1g = 1;
+      u8 use_2m = 1;
+      u8 less_than_1g = 1;
+      int rv;
+
+      umount (DEFAULT_HUGE_DIR);
+
+      /* Process "socket-mem" parameter value */
+      if (vec_len (socket_mem))
+	{
+	  unformat_input_t in;
+	  unformat_init_vector (&in, socket_mem);
+	  while (unformat_check_input (&in) != UNFORMAT_END_OF_INPUT)
+	    {
+	      if (unformat (&in, "%u,", &x))
+		;
+	      else if (unformat (&in, "%u", &x))
+		;
+	      else if (unformat (&in, ","))
+		x = 0;
+	      else
+		break;
+
+	      vec_add1 (mem_by_socket, x);
+
+	      if (x > 1023)
+		less_than_1g = 0;
+	    }
+	  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
+	  unformat_free (&in);
+	  socket_mem = 0;
+	}
+      else
+	{
+	  /* *INDENT-OFF* */
+	  clib_bitmap_foreach (c, tm->cpu_socket_bitmap, (
+	    {
+	      vec_validate(mem_by_socket, c);
+	      mem_by_socket[c] = 256; /* default per-socket mem */
+	    }
+	  ));
+	  /* *INDENT-ON* */
+	}
+
+      /* check if available enough 1GB pages for each socket */
+      /* *INDENT-OFF* */
+      clib_bitmap_foreach (c, tm->cpu_socket_bitmap, (
+        {
+	  int pages_avail, page_size, mem;
+
+	  vec_validate(mem_by_socket, c);
+	  mem = mem_by_socket[c];
+
+	  page_size = 1024;
+	  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
+
+	  if (pages_avail < 0 || page_size * pages_avail < mem)
+	    use_1g = 0;
+
+	  page_size = 2;
+	  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
+
+	  if (pages_avail < 0 || page_size * pages_avail < mem)
+	    use_2m = 0;
+      }));
+      /* *INDENT-ON* */
+
+      if (mem_by_socket == 0)
+	{
+	  error = clib_error_return (0, "mem_by_socket NULL");
+	  goto done;
+	}
+      _vec_len (mem_by_socket) = c + 1;
+
+      /* regenerate socket_mem string */
+      vec_foreach_index (x, mem_by_socket)
+	socket_mem = format (socket_mem, "%s%u",
+			     socket_mem ? "," : "", mem_by_socket[x]);
+      socket_mem = format (socket_mem, "%c", 0);
+
+      vec_free (mem_by_socket);
+
+      rv = mkdir (VPP_RUN_DIR, 0755);
+      if (rv && errno != EEXIST)
+	{
+	  error = clib_error_return (0, "mkdir '%s' failed errno %d",
+				     VPP_RUN_DIR, errno);
+	  goto done;
+	}
+
+      rv = mkdir (DEFAULT_HUGE_DIR, 0755);
+      if (rv && errno != EEXIST)
+	{
+	  error = clib_error_return (0, "mkdir '%s' failed errno %d",
+				     DEFAULT_HUGE_DIR, errno);
+	  goto done;
+	}
+
+      if (use_1g && !(less_than_1g && use_2m))
+	{
+	  rv =
+	    mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, "pagesize=1G");
+	}
+      else if (use_2m)
+	{
+	  rv = mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, NULL);
+	}
+      else
+	{
+	  return clib_error_return (0, "not enough free huge pages");
+	}
+
+      if (rv)
+	{
+	  error = clib_error_return (0, "mount failed %d", errno);
+	  goto done;
+	}
+
+      tmp = format (0, "--huge-dir%c", 0);
+      vec_add1 (conf->eal_init_args, tmp);
+      tmp = format (0, "%s%c", DEFAULT_HUGE_DIR, 0);
+      vec_add1 (conf->eal_init_args, tmp);
+      if (!file_prefix)
+	{
+	  tmp = format (0, "--file-prefix%c", 0);
+	  vec_add1 (conf->eal_init_args, tmp);
+	  tmp = format (0, "vpp%c", 0);
+	  vec_add1 (conf->eal_init_args, tmp);
+	}
+    }
+
+  vec_free (rte_cmd);
+  vec_free (ethname);
+
+  if (error)
+    return error;
+
+  /* I'll bet that -c and -n must be the first and second args... */
+  if (!conf->coremask_set_manually)
+    {
+      vlib_thread_registration_t *tr;
+      uword *coremask = 0;
+      int i;
+
+      /* main thread core */
+      coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
+
+      for (i = 0; i < vec_len (tm->registrations); i++)
+	{
+	  tr = tm->registrations[i];
+	  coremask = clib_bitmap_or (coremask, tr->coremask);
+	}
+
+      vec_insert (conf->eal_init_args, 2, 1);
+      conf->eal_init_args[1] = (u8 *) "-c";
+      tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
+      conf->eal_init_args[2] = tmp;
+      clib_bitmap_free (coremask);
+    }
+
+  if (!conf->nchannels_set_manually)
+    {
+      vec_insert (conf->eal_init_args, 2, 3);
+      conf->eal_init_args[3] = (u8 *) "-n";
+      tmp = format (0, "%d", conf->nchannels);
+      conf->eal_init_args[4] = tmp;
+    }
+
+  if (no_pci == 0 && geteuid () == 0)
+    dpdk_bind_devices_to_uio (conf);
+
+#define _(x) \
+    if (devconf->x == 0 && conf->default_devconf.x > 0) \
+      devconf->x = conf->default_devconf.x ;
+
+  /* *INDENT-OFF* */
+  pool_foreach (devconf, conf->dev_confs, ({
+
+    /* default per-device config items */
+    foreach_dpdk_device_config_item
+
+    /* add DPDK EAL whitelist/blacklist entry */
+    if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
+      {
+	tmp = format (0, "-w%c", 0);
+	vec_add1 (conf->eal_init_args, tmp);
+	tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
+	vec_add1 (conf->eal_init_args, tmp);
+      }
+    else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
+      {
+	tmp = format (0, "-b%c", 0);
+	vec_add1 (conf->eal_init_args, tmp);
+	tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
+	vec_add1 (conf->eal_init_args, tmp);
+      }
+  }));
+  /* *INDENT-ON* */
+
+#undef _
+
+  /* set master-lcore */
+  tmp = format (0, "--master-lcore%c", 0);
+  vec_add1 (conf->eal_init_args, tmp);
+  tmp = format (0, "%u%c", tm->main_lcore, 0);
+  vec_add1 (conf->eal_init_args, tmp);
+
+  /* set socket-mem */
+  tmp = format (0, "--socket-mem%c", 0);
+  vec_add1 (conf->eal_init_args, tmp);
+  tmp = format (0, "%s%c", socket_mem, 0);
+  vec_add1 (conf->eal_init_args, tmp);
+
+  /* NULL terminate the "argv" vector, in case of stupidity */
+  vec_add1 (conf->eal_init_args, 0);
+  _vec_len (conf->eal_init_args) -= 1;
+
+  /* Set up DPDK eal and packet mbuf pool early. */
+
+  log_level = (CLIB_DEBUG > 0) ? RTE_LOG_DEBUG : RTE_LOG_NOTICE;
+
+  rte_set_log_level (log_level);
+
+  vm = vlib_get_main ();
+
+  /* make copy of args as rte_eal_init tends to mess up with arg array */
+  for (i = 1; i < vec_len (conf->eal_init_args); i++)
+    conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
+				      conf->eal_init_args[i]);
+
+  ret =
+    rte_eal_init (vec_len (conf->eal_init_args),
+		  (char **) conf->eal_init_args);
+
+  /* lazy umount hugepages */
+  umount2 (DEFAULT_HUGE_DIR, MNT_DETACH);
+
+  if (ret < 0)
+    return clib_error_return (0, "rte_eal_init returned %d", ret);
+
+  /* Dump the physical memory layout prior to creating the mbuf_pool */
+  fprintf (stdout, "DPDK physical memory layout:\n");
+  rte_dump_physmem_layout (stdout);
+
+  /* main thread 1st */
+  error = vlib_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
+  if (error)
+    return error;
+
+  for (i = 0; i < RTE_MAX_LCORE; i++)
+    {
+      error = vlib_buffer_pool_create (vm, conf->num_mbufs,
+				       rte_lcore_to_socket_id (i));
+      if (error)
+	return error;
+    }
+
+done:
+  return error;
+}
+
+VLIB_CONFIG_FUNCTION (dpdk_config, "dpdk");
+
+void
+dpdk_update_link_state (dpdk_device_t * xd, f64 now)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  struct rte_eth_link prev_link = xd->link;
+  u32 hw_flags = 0;
+  u8 hw_flags_chg = 0;
+
+  /* only update link state for PMD interfaces */
+  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
+    return;
+
+  xd->time_last_link_update = now ? now : xd->time_last_link_update;
+  memset (&xd->link, 0, sizeof (xd->link));
+  rte_eth_link_get_nowait (xd->device_index, &xd->link);
+
+  if (LINK_STATE_ELOGS)
+    {
+      vlib_main_t *vm = vlib_get_main ();
+      ELOG_TYPE_DECLARE (e) =
+      {
+      .format =
+	  "update-link-state: sw_if_index %d, admin_up %d,"
+	  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
+
+      struct
+      {
+	u32 sw_if_index;
+	u8 admin_up;
+	u8 old_link_state;
+	u8 new_link_state;
+      } *ed;
+      ed = ELOG_DATA (&vm->elog_main, e);
+      ed->sw_if_index = xd->vlib_sw_if_index;
+      ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
+      ed->old_link_state = (u8)
+	vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index);
+      ed->new_link_state = (u8) xd->link.link_status;
+    }
+
+  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) &&
+      ((xd->link.link_status != 0) ^
+       vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index)))
+    {
+      hw_flags_chg = 1;
+      hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
+    }
+
+  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
+    {
+      hw_flags_chg = 1;
+      switch (xd->link.link_duplex)
+	{
+	case ETH_LINK_HALF_DUPLEX:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_HALF_DUPLEX;
+	  break;
+	case ETH_LINK_FULL_DUPLEX:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_FULL_DUPLEX;
+	  break;
+	default:
+	  break;
+	}
+    }
+  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
+    {
+      hw_flags_chg = 1;
+      switch (xd->link.link_speed)
+	{
+	case ETH_SPEED_NUM_10M:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_10M;
+	  break;
+	case ETH_SPEED_NUM_100M:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_100M;
+	  break;
+	case ETH_SPEED_NUM_1G:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
+	  break;
+	case ETH_SPEED_NUM_10G:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_10G;
+	  break;
+	case ETH_SPEED_NUM_40G:
+	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_40G;
+	  break;
+	case 0:
+	  break;
+	default:
+	  clib_warning ("unknown link speed %d", xd->link.link_speed);
+	  break;
+	}
+    }
+  if (hw_flags_chg)
+    {
+      if (LINK_STATE_ELOGS)
+	{
+	  vlib_main_t *vm = vlib_get_main ();
+
+	  ELOG_TYPE_DECLARE (e) =
+	  {
+	  .format =
+	      "update-link-state: sw_if_index %d, new flags %d",.format_args
+	      = "i4i4",};
+
+	  struct
+	  {
+	    u32 sw_if_index;
+	    u32 flags;
+	  } *ed;
+	  ed = ELOG_DATA (&vm->elog_main, e);
+	  ed->sw_if_index = xd->vlib_sw_if_index;
+	  ed->flags = hw_flags;
+	}
+      vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, hw_flags);
+    }
+}
+
+static uword
+dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
+{
+  clib_error_t *error;
+  vnet_main_t *vnm = vnet_get_main ();
+  dpdk_main_t *dm = &dpdk_main;
+  ethernet_main_t *em = ðernet_main;
+  dpdk_device_t *xd;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  int i;
+
+  error = dpdk_lib_init (dm);
+
+  /*
+   * Turn on the input node if we found some devices to drive
+   * and we're not running worker threads or i/o threads
+   */
+
+  if (error == 0 && vec_len (dm->devices) > 0)
+    {
+      if (tm->n_vlib_mains == 1)
+	vlib_node_set_state (vm, dpdk_input_node.index,
+			     VLIB_NODE_STATE_POLLING);
+      else
+	for (i = 0; i < tm->n_vlib_mains; i++)
+	  if (vec_len (dm->devices_by_cpu[i]) > 0)
+	    vlib_node_set_state (vlib_mains[i], dpdk_input_node.index,
+				 VLIB_NODE_STATE_POLLING);
+    }
+
+  if (error)
+    clib_error_report (error);
+
+  tm->worker_thread_release = 1;
+
+  f64 now = vlib_time_now (vm);
+  vec_foreach (xd, dm->devices)
+  {
+    dpdk_update_link_state (xd, now);
+  }
+
+  {
+    /*
+     * Extra set up for bond interfaces:
+     *  1. Setup MACs for bond interfaces and their slave links which was set
+     *     in dpdk_port_setup() but needs to be done again here to take effect.
+     *  2. Set up info for bond interface related CLI support.
+     */
+    int nports = rte_eth_dev_count ();
+    if (nports > 0)
+      {
+	for (i = 0; i < nports; i++)
+	  {
+	    struct rte_eth_dev_info dev_info;
+	    rte_eth_dev_info_get (i, &dev_info);
+	    if (!dev_info.driver_name)
+	      dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
+
+	    ASSERT (dev_info.driver_name);
+	    if (strncmp (dev_info.driver_name, "rte_bond_pmd", 12) == 0)
+	      {
+		u8 addr[6];
+		u8 slink[16];
+		int nlink = rte_eth_bond_slaves_get (i, slink, 16);
+		if (nlink > 0)
+		  {
+		    vnet_hw_interface_t *bhi;
+		    ethernet_interface_t *bei;
+		    int rv;
+
+		    /* Get MAC of 1st slave link */
+		    rte_eth_macaddr_get (slink[0],
+					 (struct ether_addr *) addr);
+		    /* Set MAC of bounded interface to that of 1st slave link */
+		    rv =
+		      rte_eth_bond_mac_address_set (i,
+						    (struct ether_addr *)
+						    addr);
+		    if (rv < 0)
+		      clib_warning ("Failed to set MAC address");
+
+		    /* Populate MAC of bonded interface in VPP hw tables */
+		    bhi =
+		      vnet_get_hw_interface (vnm,
+					     dm->devices[i].vlib_hw_if_index);
+		    bei =
+		      pool_elt_at_index (em->interfaces, bhi->hw_instance);
+		    clib_memcpy (bhi->hw_address, addr, 6);
+		    clib_memcpy (bei->address, addr, 6);
+		    /* Init l3 packet size allowed on bonded interface */
+		    bhi->max_packet_bytes = ETHERNET_MAX_PACKET_BYTES;
+		    bhi->max_l3_packet_bytes[VLIB_RX] =
+		      bhi->max_l3_packet_bytes[VLIB_TX] =
+		      ETHERNET_MAX_PACKET_BYTES - sizeof (ethernet_header_t);
+		    while (nlink >= 1)
+		      {		/* for all slave links */
+			int slave = slink[--nlink];
+			dpdk_device_t *sdev = &dm->devices[slave];
+			vnet_hw_interface_t *shi;
+			vnet_sw_interface_t *ssi;
+			/* Add MAC to all slave links except the first one */
+			if (nlink)
+			  rte_eth_dev_mac_addr_add (slave,
+						    (struct ether_addr *)
+						    addr, 0);
+			/* Set slaves bitmap for bonded interface */
+			bhi->bond_info =
+			  clib_bitmap_set (bhi->bond_info,
+					   sdev->vlib_hw_if_index, 1);
+			/* Set slave link flags on slave interface */
+			shi =
+			  vnet_get_hw_interface (vnm, sdev->vlib_hw_if_index);
+			ssi =
+			  vnet_get_sw_interface (vnm, sdev->vlib_sw_if_index);
+			shi->bond_info = VNET_HW_INTERFACE_BOND_INFO_SLAVE;
+			ssi->flags |= VNET_SW_INTERFACE_FLAG_BOND_SLAVE;
+
+			/* Set l3 packet size allowed as the lowest of slave */
+			if (bhi->max_l3_packet_bytes[VLIB_RX] >
+			    shi->max_l3_packet_bytes[VLIB_RX])
+			  bhi->max_l3_packet_bytes[VLIB_RX] =
+			    bhi->max_l3_packet_bytes[VLIB_TX] =
+			    shi->max_l3_packet_bytes[VLIB_RX];
+
+			/* Set max packet size allowed as the lowest of slave */
+			if (bhi->max_packet_bytes > shi->max_packet_bytes)
+			  bhi->max_packet_bytes = shi->max_packet_bytes;
+		      }
+		  }
+	      }
+	  }
+      }
+  }
+
+  while (1)
+    {
+      /*
+       * check each time through the loop in case intervals are changed
+       */
+      f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
+	dm->link_state_poll_interval : dm->stat_poll_interval;
+
+      vlib_process_wait_for_event_or_clock (vm, min_wait);
+
+      if (dm->admin_up_down_in_progress)
+	/* skip the poll if an admin up down is in progress (on any interface) */
+	continue;
+
+      vec_foreach (xd, dm->devices)
+      {
+	f64 now = vlib_time_now (vm);
+	if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
+	  dpdk_update_counters (xd, now);
+	if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
+	  dpdk_update_link_state (xd, now);
+
+      }
+    }
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (dpdk_process_node,static) = {
+    .function = dpdk_process,
+    .type = VLIB_NODE_TYPE_PROCESS,
+    .name = "dpdk-process",
+    .process_log2_n_stack_bytes = 17,
+};
+/* *INDENT-ON* */
+
+int
+dpdk_set_stat_poll_interval (f64 interval)
+{
+  if (interval < DPDK_MIN_STATS_POLL_INTERVAL)
+    return (VNET_API_ERROR_INVALID_VALUE);
+
+  dpdk_main.stat_poll_interval = interval;
+
+  return 0;
+}
+
+int
+dpdk_set_link_state_poll_interval (f64 interval)
+{
+  if (interval < DPDK_MIN_LINK_POLL_INTERVAL)
+    return (VNET_API_ERROR_INVALID_VALUE);
+
+  dpdk_main.link_state_poll_interval = interval;
+
+  return 0;
+}
+
+clib_error_t *
+dpdk_init (vlib_main_t * vm)
+{
+  dpdk_main_t *dm = &dpdk_main;
+  vlib_node_t *ei;
+  clib_error_t *error = 0;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+
+  /* verify that structs are cacheline aligned */
+  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
+		 "Cache line marker must be 1st element in dpdk_device_t");
+  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
+		 CLIB_CACHE_LINE_BYTES,
+		 "Data in cache line 0 is bigger than cache line size");
+  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
+		 "Cache line marker must be 1st element in frame_queue_trace_t");
+
+  u8 *name;
+  name = format (0, "dpdk_%08x%c", api_version, 0);
+
+  /* Ask for a correctly-sized block of API message decode slots */
+  dm->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+  vec_free (name);
+
+  dm->vlib_main = vm;
+  dm->vnet_main = vnet_get_main ();
+  dm->conf = &dpdk_config_main;
+
+  error = dpdk_plugin_api_hookup (vm);
+
+  /* Add our API messages to the global name_crc hash table */
+  setup_message_id_table (dm, &api_main);
+
+//  TODO
+//  plugin_custom_dump_configure (dm);
+
+  ei = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
+  if (ei == 0)
+    return clib_error_return (0, "ethernet-input node AWOL");
+
+  dm->ethernet_input_node_index = ei->index;
+
+  dm->conf->nchannels = 4;
+  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
+  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
+
+  dm->dpdk_device_by_kni_port_id = hash_create (0, sizeof (uword));
+  dm->vu_sw_if_index_by_listener_fd = hash_create (0, sizeof (uword));
+  dm->vu_sw_if_index_by_sock_fd = hash_create (0, sizeof (uword));
+
+  /* $$$ use n_thread_stacks since it's known-good at this point */
+  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
+
+  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
+  dm->buffer_flags_template =
+    (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
+     | IP_BUFFER_L4_CHECKSUM_COMPUTED | IP_BUFFER_L4_CHECKSUM_CORRECT);
+
+  dm->stat_poll_interval = DPDK_STATS_POLL_INTERVAL;
+  dm->link_state_poll_interval = DPDK_LINK_POLL_INTERVAL;
+
+  /* init CLI */
+  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
+    return error;
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (dpdk_init);
+
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/ipsec/cli.c b/src/plugins/dpdk/ipsec/cli.c
new file mode 100644
index 00000000..40cee39b
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/cli.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+
+static void
+dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display)
+{
+  dpdk_config_main_t *conf = &dpdk_config_main;
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  u32 i, skip_master;
+
+  if (!conf->cryptodev)
+    {
+      vlib_cli_output (vm, "DPDK Cryptodev support is disabled\n");
+      return;
+    }
+
+  if (detail_display)
+    vlib_cli_output (vm, "worker\t%10s\t%15s\tdir\tdev\tqp\n",
+		     "cipher", "auth");
+  else
+    vlib_cli_output (vm, "worker\tcrypto device id(type)\n");
+
+  skip_master = vlib_num_workers () > 0;
+
+  for (i = 0; i < tm->n_vlib_mains; i++)
+    {
+      uword key, data;
+      u32 cpu_index = vlib_mains[i]->cpu_index;
+      crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
+      u8 *s = 0;
+
+      if (skip_master)
+	{
+	  skip_master = 0;
+	  continue;
+	}
+
+      if (!detail_display)
+	{
+	  i32 last_cdev = -1;
+	  crypto_qp_data_t *qpd;
+
+	  s = format (s, "%u\t", cpu_index);
+
+	  /* *INDENT-OFF* */
+	  vec_foreach (qpd, cwm->qp_data)
+	    {
+	      u32 dev_id = qpd->dev_id;
+
+	      if ((u16) last_cdev != dev_id)
+		{
+		  struct rte_cryptodev_info cdev_info;
+
+		  rte_cryptodev_info_get (dev_id, &cdev_info);
+
+		  s = format(s, "%u(%s)\t", dev_id, cdev_info.feature_flags &
+			     RTE_CRYPTODEV_FF_HW_ACCELERATED ? "HW" : "SW");
+		}
+	      last_cdev = dev_id;
+	    }
+	  /* *INDENT-ON* */
+	  vlib_cli_output (vm, "%s", s);
+	}
+      else
+	{
+	  char cipher_str[15], auth_str[15];
+	  struct rte_cryptodev_capabilities cap;
+	  crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
+	  /* *INDENT-OFF* */
+	  hash_foreach (key, data, cwm->algo_qp_map,
+	  ({
+	    cap.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+	    cap.sym.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	    cap.sym.cipher.algo = p_key->cipher_algo;
+	    check_algo_is_supported (&cap, cipher_str);
+	    cap.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+	    cap.sym.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	    cap.sym.auth.algo = p_key->auth_algo;
+	    check_algo_is_supported (&cap, auth_str);
+	    vlib_cli_output (vm, "%u\t%10s\t%15s\t%3s\t%u\t%u\n",
+			     vlib_mains[i]->cpu_index, cipher_str, auth_str,
+			     p_key->is_outbound ? "out" : "in",
+			     cwm->qp_data[data].dev_id,
+			     cwm->qp_data[data].qp_id);
+	  }));
+	  /* *INDENT-ON* */
+	}
+    }
+}
+
+static clib_error_t *
+lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input,
+			vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u16 detail = 0;
+  clib_error_t *error = NULL;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "verbose"))
+	detail = 1;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  dpdk_ipsec_show_mapping (vm, detail);
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (lcore_cryptodev_map, static) = {
+    .path = "show crypto device mapping",
+    .short_help =
+    "show cryptodev device mapping ",
+    .function = lcore_cryptodev_map_fn,
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/ipsec/crypto_node.c b/src/plugins/dpdk/ipsec/crypto_node.c
new file mode 100644
index 00000000..dc3452b2
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/crypto_node.c
@@ -0,0 +1,215 @@
+/*
+ *------------------------------------------------------------------
+ * crypto_node.c - DPDK Cryptodev input node
+ *
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#define foreach_dpdk_crypto_input_next		\
+  _(DROP, "error-drop")				\
+  _(ENCRYPT_POST, "dpdk-esp-encrypt-post")	\
+  _(DECRYPT_POST, "dpdk-esp-decrypt-post")
+
+typedef enum
+{
+#define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f,
+  foreach_dpdk_crypto_input_next
+#undef _
+    DPDK_CRYPTO_INPUT_N_NEXT,
+} dpdk_crypto_input_next_t;
+
+#define foreach_dpdk_crypto_input_error		\
+  _(DQ_COPS, "Crypto ops dequeued")		\
+  _(COP_FAILED, "Crypto op failed")
+
+typedef enum
+{
+#define _(f,s) DPDK_CRYPTO_INPUT_ERROR_##f,
+  foreach_dpdk_crypto_input_error
+#undef _
+    DPDK_CRYPTO_INPUT_N_ERROR,
+} dpdk_crypto_input_error_t;
+
+static char *dpdk_crypto_input_error_strings[] = {
+#define _(n, s) s,
+  foreach_dpdk_crypto_input_error
+#undef _
+};
+
+vlib_node_registration_t dpdk_crypto_input_node;
+
+typedef struct
+{
+  u32 cdev;
+  u32 qp;
+  u32 status;
+  u32 sa_idx;
+  u32 next_index;
+} dpdk_crypto_input_trace_t;
+
+static u8 *
+format_dpdk_crypto_input_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  dpdk_crypto_input_trace_t *t = va_arg (*args, dpdk_crypto_input_trace_t *);
+
+  s = format (s, "dpdk_crypto: cryptodev-id %u queue-pair %u next-index %d",
+	      t->cdev, t->qp, t->next_index);
+
+  s = format (s, " status %u sa-idx %u\n", t->status, t->sa_idx);
+
+  return s;
+}
+
+static_always_inline u32
+dpdk_crypto_dequeue (vlib_main_t * vm, vlib_node_runtime_t * node,
+		     crypto_qp_data_t * qpd)
+{
+  u32 n_deq, *to_next = 0, next_index, n_cops, def_next_index;
+  struct rte_crypto_op **cops = qpd->cops;
+
+  if (qpd->inflights == 0)
+    return 0;
+
+  if (qpd->is_outbound)
+    def_next_index = DPDK_CRYPTO_INPUT_NEXT_ENCRYPT_POST;
+  else
+    def_next_index = DPDK_CRYPTO_INPUT_NEXT_DECRYPT_POST;
+
+  n_cops = rte_cryptodev_dequeue_burst (qpd->dev_id, qpd->qp_id,
+					cops, VLIB_FRAME_SIZE);
+  n_deq = n_cops;
+  next_index = def_next_index;
+
+  qpd->inflights -= n_cops;
+  ASSERT (qpd->inflights >= 0);
+
+  while (n_cops > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_cops > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0, next0;
+	  vlib_buffer_t *b0 = 0;
+	  struct rte_crypto_op *cop;
+	  struct rte_crypto_sym_op *sym_cop;
+
+	  cop = cops[0];
+	  cops += 1;
+	  n_cops -= 1;
+	  n_left_to_next -= 1;
+
+	  next0 = def_next_index;
+
+	  if (PREDICT_FALSE (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
+	    {
+	      next0 = DPDK_CRYPTO_INPUT_NEXT_DROP;
+	      vlib_node_increment_counter (vm, dpdk_crypto_input_node.index,
+					   DPDK_CRYPTO_INPUT_ERROR_COP_FAILED,
+					   1);
+	    }
+	  cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+
+	  sym_cop = (struct rte_crypto_sym_op *) (cop + 1);
+	  b0 = vlib_buffer_from_rte_mbuf (sym_cop->m_src);
+	  bi0 = vlib_get_buffer_index (vm, b0);
+
+	  to_next[0] = bi0;
+	  to_next += 1;
+
+	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      vlib_trace_next_frame (vm, node, next0);
+	      dpdk_crypto_input_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      tr->cdev = qpd->dev_id;
+	      tr->qp = qpd->qp_id;
+	      tr->status = cop->status;
+	      tr->next_index = next0;
+	      tr->sa_idx = vnet_buffer (b0)->ipsec.sad_index;
+	    }
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  crypto_free_cop (qpd, qpd->cops, n_deq);
+
+  vlib_node_increment_counter (vm, dpdk_crypto_input_node.index,
+			       DPDK_CRYPTO_INPUT_ERROR_DQ_COPS, n_deq);
+  return n_deq;
+}
+
+static uword
+dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		      vlib_frame_t * frame)
+{
+  u32 cpu_index = os_get_cpu_number ();
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
+  crypto_qp_data_t *qpd;
+  u32 n_deq = 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (qpd, cwm->qp_data)
+      n_deq += dpdk_crypto_dequeue(vm, node, qpd);
+  /* *INDENT-ON* */
+
+  return n_deq;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (dpdk_crypto_input_node) =
+{
+  .function = dpdk_crypto_input_fn,
+  .name = "dpdk-crypto-input",
+  .format_trace = format_dpdk_crypto_input_trace,
+  .type = VLIB_NODE_TYPE_INPUT,
+  .state = VLIB_NODE_STATE_DISABLED,
+  .n_errors = DPDK_CRYPTO_INPUT_N_ERROR,
+  .error_strings = dpdk_crypto_input_error_strings,
+  .n_next_nodes = DPDK_CRYPTO_INPUT_N_NEXT,
+  .next_nodes =
+  {
+#define _(s,n) [DPDK_CRYPTO_INPUT_NEXT_##s] = n,
+    foreach_dpdk_crypto_input_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (dpdk_crypto_input_node, dpdk_crypto_input_fn)
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/ipsec/dir.dox b/src/plugins/dpdk/ipsec/dir.dox
new file mode 100644
index 00000000..ffebfc4d
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/dir.dox
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ @dir vnet/vnet/devices/dpdk/ipsec
+ @brief IPSec ESP encrypt/decrypt using DPDK Cryptodev API
+*/
diff --git a/src/plugins/dpdk/ipsec/dpdk_crypto_ipsec_doc.md b/src/plugins/dpdk/ipsec/dpdk_crypto_ipsec_doc.md
new file mode 100644
index 00000000..fed2fe0e
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/dpdk_crypto_ipsec_doc.md
@@ -0,0 +1,86 @@
+# VPP IPSec implementation using DPDK Cryptodev API    {#dpdk_crypto_ipsec_doc}
+
+This document is meant to contain all related information about implementation and usability.
+
+
+## VPP IPsec with DPDK Cryptodev
+
+DPDK Cryptodev is an asynchronous crypto API that supports both Hardware and Software implementations (for more details refer to [DPDK Cryptography Device Library documentation](http://dpdk.org/doc/guides/prog_guide/cryptodev_lib.html)).
+
+When DPDK support is enabled and there are enough Cryptodev resources for all workers, the node graph is reconfigured by adding and changing default next nodes.
+
+The following nodes are added:
+* dpdk-crypto-input : polling input node, basically dequeuing from crypto devices.
+* dpdk-esp-encrypt : internal node.
+* dpdk-esp-decrypt : internal node.
+* dpdk-esp-encrypt-post : internal node.
+* dpdk-esp-decrypt-post : internal node.
+
+Set new default next nodes:
+* for esp encryption: esp-encrypt -> dpdk-esp-encrypt
+* for esp decryption: esp-decrypt -> dpdk-esp-decrypt
+
+
+### How to enable VPP IPSec with DPDK Cryptodev support
+
+DPDK Cryptodev is supported in DPDK enabled VPP.
+By default, only HW Cryptodev is supported but needs to be explicetly enabled with the following config option:
+
+```
+dpdk {
+    enable-cryptodev
+}
+```
+
+To enable SW Cryptodev support (AESNI-MB-PMD and GCM-PMD), we need the following env option:
+
+    vpp_uses_dpdk_cryptodev_sw=yes
+
+A couple of ways to achive this:
+* uncomment/add it in the platforms config (ie. build-data/platforms/vpp.mk)
+* set the option when building vpp (ie. make vpp_uses_dpdk_cryptodev_sw=yes build-release)
+
+When enabling SW Cryptodev support, it means that you need to pre-build the required crypto libraries needed by those SW Cryptodev PMDs.
+
+
+### Crypto Resources allocation
+
+VPP allocates crypto resources based on a best effort approach:
+* first allocate Hardware crypto resources, then Software.
+* if there are not enough crypto resources for all workers, the graph node is not modifed, therefore the default VPP IPsec implementation based in OpenSSL is used. The following message is displayed:
+
+      0: dpdk_ipsec_init: not enough cryptodevs for ipsec
+
+
+### Configuration example
+
+To enable DPDK Cryptodev the user just need to provide the startup.conf option
+as mentioned previously.
+
+Example startup.conf:
+
+```
+dpdk {
+    socket-mem 1024,1024
+    num-mbufs 131072
+    dev 0000:81:00.0
+    dev 0000:81:00.1
+    enable-cryptodev
+    dev 0000:85:01.0
+    dev 0000:85:01.1
+    vdev cryptodev_aesni_mb_pmd,socket_id=1
+    vdev cryptodev_aesni_mb_pmd,socket_id=1
+}
+```
+
+In the above configuration:
+* 0000:85:01.0 and 0000:85:01.1 are crypto BDFs and they require the same driver binding as DPDK Ethernet devices but they do not support any extra configuration options.
+* Two AESNI-MB Software Cryptodev PMDs are created in NUMA node 1.
+
+For further details refer to [DPDK Crypto Device Driver documentation](http://dpdk.org/doc/guides/cryptodevs/index.html)
+
+### Operational data
+
+The following CLI command displays the Cryptodev/Worker mapping:
+
+    show crypto device mapping [verbose]
diff --git a/src/plugins/dpdk/ipsec/esp.h b/src/plugins/dpdk/ipsec/esp.h
new file mode 100644
index 00000000..320295b1
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/esp.h
@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __DPDK_ESP_H__
+#define __DPDK_ESP_H__
+
+#include 
+#include 
+#include 
+
+typedef struct
+{
+  enum rte_crypto_cipher_algorithm algo;
+  u8 key_len;
+  u8 iv_len;
+} dpdk_esp_crypto_alg_t;
+
+typedef struct
+{
+  enum rte_crypto_auth_algorithm algo;
+  u8 trunc_size;
+} dpdk_esp_integ_alg_t;
+
+typedef struct
+{
+  dpdk_esp_crypto_alg_t *esp_crypto_algs;
+  dpdk_esp_integ_alg_t *esp_integ_algs;
+} dpdk_esp_main_t;
+
+dpdk_esp_main_t dpdk_esp_main;
+
+static_always_inline void
+dpdk_esp_init ()
+{
+  dpdk_esp_main_t *em = &dpdk_esp_main;
+  dpdk_esp_integ_alg_t *i;
+  dpdk_esp_crypto_alg_t *c;
+
+  vec_validate (em->esp_crypto_algs, IPSEC_CRYPTO_N_ALG - 1);
+
+  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
+  c->algo = RTE_CRYPTO_CIPHER_AES_CBC;
+  c->key_len = 16;
+  c->iv_len = 16;
+
+  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
+  c->algo = RTE_CRYPTO_CIPHER_AES_CBC;
+  c->key_len = 24;
+  c->iv_len = 16;
+
+  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
+  c->algo = RTE_CRYPTO_CIPHER_AES_CBC;
+  c->key_len = 32;
+  c->iv_len = 16;
+
+  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
+  c->algo = RTE_CRYPTO_CIPHER_AES_GCM;
+  c->key_len = 16;
+  c->iv_len = 8;
+
+  vec_validate (em->esp_integ_algs, IPSEC_INTEG_N_ALG - 1);
+
+  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA1_96];
+  i->algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
+  i->trunc_size = 12;
+
+  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_256_96];
+  i->algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
+  i->trunc_size = 12;
+
+  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_256_128];
+  i->algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
+  i->trunc_size = 16;
+
+  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_384_192];
+  i->algo = RTE_CRYPTO_AUTH_SHA384_HMAC;
+  i->trunc_size = 24;
+
+  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_512_256];
+  i->algo = RTE_CRYPTO_AUTH_SHA512_HMAC;
+  i->trunc_size = 32;
+
+  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_AES_GCM_128];
+  i->algo = RTE_CRYPTO_AUTH_AES_GCM;
+  i->trunc_size = 16;
+}
+
+static_always_inline int
+translate_crypto_algo (ipsec_crypto_alg_t crypto_algo,
+		       struct rte_crypto_sym_xform *cipher_xform)
+{
+  switch (crypto_algo)
+    {
+    case IPSEC_CRYPTO_ALG_NONE:
+      cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
+      break;
+    case IPSEC_CRYPTO_ALG_AES_CBC_128:
+    case IPSEC_CRYPTO_ALG_AES_CBC_192:
+    case IPSEC_CRYPTO_ALG_AES_CBC_256:
+      cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
+      break;
+    case IPSEC_CRYPTO_ALG_AES_GCM_128:
+      cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_GCM;
+      break;
+    default:
+      return -1;
+    }
+
+  cipher_xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+
+  return 0;
+}
+
+static_always_inline int
+translate_integ_algo (ipsec_integ_alg_t integ_alg,
+		      struct rte_crypto_sym_xform *auth_xform, int use_esn)
+{
+  switch (integ_alg)
+    {
+    case IPSEC_INTEG_ALG_NONE:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_NULL;
+      auth_xform->auth.digest_length = 0;
+      break;
+    case IPSEC_INTEG_ALG_SHA1_96:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
+      auth_xform->auth.digest_length = 12;
+      break;
+    case IPSEC_INTEG_ALG_SHA_256_96:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
+      auth_xform->auth.digest_length = 12;
+      break;
+    case IPSEC_INTEG_ALG_SHA_256_128:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
+      auth_xform->auth.digest_length = 16;
+      break;
+    case IPSEC_INTEG_ALG_SHA_384_192:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA384_HMAC;
+      auth_xform->auth.digest_length = 24;
+      break;
+    case IPSEC_INTEG_ALG_SHA_512_256:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA512_HMAC;
+      auth_xform->auth.digest_length = 32;
+      break;
+    case IPSEC_INTEG_ALG_AES_GCM_128:
+      auth_xform->auth.algo = RTE_CRYPTO_AUTH_AES_GCM;
+      auth_xform->auth.digest_length = 16;
+      auth_xform->auth.add_auth_data_length = use_esn ? 12 : 8;
+      break;
+    default:
+      return -1;
+    }
+
+  auth_xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+
+  return 0;
+}
+
+static_always_inline int
+create_sym_sess (ipsec_sa_t * sa, crypto_sa_session_t * sa_sess,
+		 u8 is_outbound)
+{
+  u32 cpu_index = os_get_cpu_number ();
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
+  struct rte_crypto_sym_xform cipher_xform = { 0 };
+  struct rte_crypto_sym_xform auth_xform = { 0 };
+  struct rte_crypto_sym_xform *xfs;
+  uword key = 0, *data;
+  crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
+
+  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
+    {
+      sa->crypto_key_len -= 4;
+      clib_memcpy (&sa->salt, &sa->crypto_key[sa->crypto_key_len], 4);
+    }
+  else
+    {
+      u32 seed = (u32) clib_cpu_time_now ();
+      sa->salt = random_u32 (&seed);
+    }
+
+  cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  cipher_xform.cipher.key.data = sa->crypto_key;
+  cipher_xform.cipher.key.length = sa->crypto_key_len;
+
+  auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  auth_xform.auth.key.data = sa->integ_key;
+  auth_xform.auth.key.length = sa->integ_key_len;
+
+  if (translate_crypto_algo (sa->crypto_alg, &cipher_xform) < 0)
+    return -1;
+  p_key->cipher_algo = cipher_xform.cipher.algo;
+
+  if (translate_integ_algo (sa->integ_alg, &auth_xform, sa->use_esn) < 0)
+    return -1;
+  p_key->auth_algo = auth_xform.auth.algo;
+
+  if (is_outbound)
+    {
+      cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
+      auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
+      cipher_xform.next = &auth_xform;
+      xfs = &cipher_xform;
+    }
+  else
+    {
+      cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
+      auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
+      auth_xform.next = &cipher_xform;
+      xfs = &auth_xform;
+    }
+
+  p_key->is_outbound = is_outbound;
+
+  data = hash_get (cwm->algo_qp_map, key);
+  if (!data)
+    return -1;
+
+  sa_sess->sess =
+    rte_cryptodev_sym_session_create (cwm->qp_data[*data].dev_id, xfs);
+
+  if (!sa_sess->sess)
+    return -1;
+
+  sa_sess->qp_index = (u8) * data;
+
+  return 0;
+}
+
+#endif /* __DPDK_ESP_H__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c
new file mode 100644
index 00000000..286e03f8
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/esp_decrypt.c
@@ -0,0 +1,594 @@
+/*
+ * esp_decrypt.c : IPSec ESP Decrypt node using DPDK Cryptodev
+ *
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define foreach_esp_decrypt_next	       \
+_(DROP, "error-drop")			       \
+_(IP4_INPUT, "ip4-input")		       \
+_(IP6_INPUT, "ip6-input")
+
+#define _(v, s) ESP_DECRYPT_NEXT_##v,
+typedef enum {
+  foreach_esp_decrypt_next
+#undef _
+  ESP_DECRYPT_N_NEXT,
+} esp_decrypt_next_t;
+
+#define foreach_esp_decrypt_error		 \
+ _(RX_PKTS, "ESP pkts received")		 \
+ _(DECRYPTION_FAILED, "ESP decryption failed")   \
+ _(REPLAY, "SA replayed packet")	         \
+ _(NOT_IP, "Not IP packet (dropped)")	         \
+ _(ENQ_FAIL, "Enqueue failed (buffer full)")     \
+ _(NO_CRYPTODEV, "Cryptodev not configured")     \
+ _(BAD_LEN, "Invalid ciphertext length")         \
+ _(UNSUPPORTED, "Cipher/Auth not supported")
+
+
+typedef enum {
+#define _(sym,str) ESP_DECRYPT_ERROR_##sym,
+  foreach_esp_decrypt_error
+#undef _
+  ESP_DECRYPT_N_ERROR,
+} esp_decrypt_error_t;
+
+static char * esp_decrypt_error_strings[] = {
+#define _(sym,string) string,
+  foreach_esp_decrypt_error
+#undef _
+};
+
+vlib_node_registration_t dpdk_esp_decrypt_node;
+
+typedef struct {
+  ipsec_crypto_alg_t crypto_alg;
+  ipsec_integ_alg_t integ_alg;
+} esp_decrypt_trace_t;
+
+/* packet trace format function */
+static u8 * format_esp_decrypt_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *);
+
+  s = format (s, "esp: crypto %U integrity %U",
+	      format_ipsec_crypto_alg, t->crypto_alg,
+	      format_ipsec_integ_alg, t->integ_alg);
+  return s;
+}
+
+static uword
+dpdk_esp_decrypt_node_fn (vlib_main_t * vm,
+	     vlib_node_runtime_t * node,
+	     vlib_frame_t * from_frame)
+{
+  u32 n_left_from, *from, *to_next, next_index;
+  ipsec_main_t *im = &ipsec_main;
+  u32 cpu_index = os_get_cpu_number();
+  dpdk_crypto_main_t * dcm = &dpdk_crypto_main;
+  dpdk_esp_main_t * em = &dpdk_esp_main;
+  u32 i;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  if (PREDICT_FALSE(!dcm->workers_main))
+    {
+      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+	      ESP_DECRYPT_ERROR_NO_CRYPTODEV, n_left_from);
+      vlib_buffer_free(vm, from, n_left_from);
+      return n_left_from;
+    }
+
+  crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, cpu_index);
+  u32 n_qps = vec_len(cwm->qp_data);
+  struct rte_crypto_op ** cops_to_enq[n_qps];
+  u32 n_cop_qp[n_qps], * bi_to_enq[n_qps];
+
+  for (i = 0; i < n_qps; i++)
+    {
+      bi_to_enq[i] = cwm->qp_data[i].bi;
+      cops_to_enq[i] = cwm->qp_data[i].cops;
+    }
+
+  memset(n_cop_qp, 0, n_qps * sizeof(u32));
+
+  crypto_alloc_cops();
+
+  next_index = ESP_DECRYPT_NEXT_DROP;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0, sa_index0 = ~0, seq, icv_size, iv_size;
+	  vlib_buffer_t * b0;
+	  esp_header_t * esp0;
+	  ipsec_sa_t * sa0;
+	  struct rte_mbuf * mb0 = 0;
+	  const int BLOCK_SIZE = 16;
+	  crypto_sa_session_t * sa_sess;
+	  void * sess;
+	  u16 qp_index;
+	  struct rte_crypto_op * cop = 0;
+
+	  bi0 = from[0];
+	  from += 1;
+	  n_left_from -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  esp0 = vlib_buffer_get_current (b0);
+
+	  sa_index0 = vnet_buffer(b0)->ipsec.sad_index;
+	  sa0 = pool_elt_at_index (im->sad, sa_index0);
+
+	  seq = clib_host_to_net_u32(esp0->seq);
+
+	  /* anti-replay check */
+	  if (sa0->use_anti_replay)
+	    {
+	      int rv = 0;
+
+	      if (PREDICT_TRUE(sa0->use_esn))
+		rv = esp_replay_check_esn(sa0, seq);
+	      else
+		rv = esp_replay_check(sa0, seq);
+
+	      if (PREDICT_FALSE(rv))
+		{
+		  clib_warning ("anti-replay SPI %u seq %u", sa0->spi, seq);
+		  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+					       ESP_DECRYPT_ERROR_REPLAY, 1);
+		  to_next[0] = bi0;
+		  to_next += 1;
+		  n_left_to_next -= 1;
+		  goto trace;
+		}
+	    }
+
+	  sa0->total_data_size += b0->current_length;
+
+	  if (PREDICT_FALSE(sa0->integ_alg == IPSEC_INTEG_ALG_NONE) ||
+		  PREDICT_FALSE(sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE))
+	    {
+	      clib_warning ("SPI %u : only cipher + auth supported", sa0->spi);
+	      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+					   ESP_DECRYPT_ERROR_UNSUPPORTED, 1);
+	      to_next[0] = bi0;
+	      to_next += 1;
+	      n_left_to_next -= 1;
+	      goto trace;
+	    }
+
+	  sa_sess = pool_elt_at_index(cwm->sa_sess_d[0], sa_index0);
+
+	  if (PREDICT_FALSE(!sa_sess->sess))
+	    {
+	      int ret = create_sym_sess(sa0, sa_sess, 0);
+
+	      if (PREDICT_FALSE (ret))
+		{
+		  to_next[0] = bi0;
+		  to_next += 1;
+		  n_left_to_next -= 1;
+		  goto trace;
+		}
+	    }
+
+	  sess = sa_sess->sess;
+	  qp_index = sa_sess->qp_index;
+
+	  ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0);
+	  cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops);
+	  ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
+
+	  cops_to_enq[qp_index][0] = cop;
+	  cops_to_enq[qp_index] += 1;
+	  n_cop_qp[qp_index] += 1;
+	  bi_to_enq[qp_index][0] = bi0;
+	  bi_to_enq[qp_index] += 1;
+
+	  rte_crypto_op_attach_sym_session(cop, sess);
+
+	  icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
+	  iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len;
+
+	  /* Convert vlib buffer to mbuf */
+	  mb0 = rte_mbuf_from_vlib_buffer(b0);
+	  mb0->data_len = b0->current_length;
+	  mb0->pkt_len = b0->current_length;
+	  mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data;
+
+	  /* Outer IP header has already been stripped */
+	  u16 payload_len = rte_pktmbuf_pkt_len(mb0) - sizeof (esp_header_t) -
+	      iv_size - icv_size;
+
+	  if ((payload_len & (BLOCK_SIZE - 1)) || (payload_len <= 0))
+	    {
+	      clib_warning ("payload %u not multiple of %d\n",
+			    payload_len, BLOCK_SIZE);
+	      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+					   ESP_DECRYPT_ERROR_BAD_LEN, 1);
+	      vec_add (vec_elt (cwm->qp_data, qp_index).free_cops, &cop, 1);
+	      bi_to_enq[qp_index] -= 1;
+	      cops_to_enq[qp_index] -= 1;
+	      n_cop_qp[qp_index] -= 1;
+	      to_next[0] = bi0;
+	      to_next += 1;
+	      n_left_to_next -= 1;
+	      goto trace;
+	    }
+
+	  struct rte_crypto_sym_op *sym_cop = (struct rte_crypto_sym_op *)(cop + 1);
+
+	  sym_cop->m_src = mb0;
+	  sym_cop->cipher.data.offset = sizeof (esp_header_t) + iv_size;
+	  sym_cop->cipher.data.length = payload_len;
+
+          u8 *iv = rte_pktmbuf_mtod_offset(mb0, void*, sizeof (esp_header_t));
+          dpdk_cop_priv_t * priv = (dpdk_cop_priv_t *)(sym_cop + 1);
+
+          if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
+            {
+              dpdk_gcm_cnt_blk *icb = &priv->cb;
+              icb->salt = sa0->salt;
+              clib_memcpy(icb->iv, iv, 8);
+              icb->cnt = clib_host_to_net_u32(1);
+              sym_cop->cipher.iv.data = (u8 *)icb;
+              sym_cop->cipher.iv.phys_addr = cop->phys_addr +
+		(uintptr_t)icb - (uintptr_t)cop;
+              sym_cop->cipher.iv.length = 16;
+
+              u8 *aad = priv->aad;
+              clib_memcpy(aad, iv - sizeof(esp_header_t), 8);
+              sym_cop->auth.aad.data = aad;
+              sym_cop->auth.aad.phys_addr = cop->phys_addr +
+                  (uintptr_t)aad - (uintptr_t)cop;
+              if (sa0->use_esn)
+                {
+                  *((u32*)&aad[8]) = sa0->seq_hi;
+                  sym_cop->auth.aad.length = 12;
+                }
+              else
+                {
+                  sym_cop->auth.aad.length = 8;
+                }
+
+              sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*,
+                       rte_pktmbuf_pkt_len(mb0) - icv_size);
+              sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
+                       rte_pktmbuf_pkt_len(mb0) - icv_size);
+              sym_cop->auth.digest.length = icv_size;
+
+            }
+          else
+            {
+              sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(mb0, void*,
+                       sizeof (esp_header_t));
+              sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
+                       sizeof (esp_header_t));
+              sym_cop->cipher.iv.length = iv_size;
+
+              if (sa0->use_esn)
+                {
+                  dpdk_cop_priv_t* priv = (dpdk_cop_priv_t*) (sym_cop + 1);
+                  u8* payload_end = rte_pktmbuf_mtod_offset(
+                      mb0, u8*, sizeof(esp_header_t) + iv_size + payload_len);
+
+                  clib_memcpy (priv->icv, payload_end, icv_size);
+                  *((u32*) payload_end) = sa0->seq_hi;
+                  sym_cop->auth.data.offset = 0;
+                  sym_cop->auth.data.length = sizeof(esp_header_t) + iv_size
+                      + payload_len + sizeof(sa0->seq_hi);
+                  sym_cop->auth.digest.data = priv->icv;
+                  sym_cop->auth.digest.phys_addr = cop->phys_addr
+                      + (uintptr_t) priv->icv - (uintptr_t) cop;
+                  sym_cop->auth.digest.length = icv_size;
+                }
+              else
+                {
+                  sym_cop->auth.data.offset = 0;
+                  sym_cop->auth.data.length = sizeof(esp_header_t) +
+                           iv_size + payload_len;
+
+                  sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*,
+                           rte_pktmbuf_pkt_len(mb0) - icv_size);
+                  sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
+                           rte_pktmbuf_pkt_len(mb0) - icv_size);
+                  sym_cop->auth.digest.length = icv_size;
+                }
+            }
+
+trace:
+	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      tr->crypto_alg = sa0->crypto_alg;
+	      tr->integ_alg = sa0->integ_alg;
+	    }
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+			       ESP_DECRYPT_ERROR_RX_PKTS,
+			       from_frame->n_vectors);
+  crypto_qp_data_t *qpd;
+  /* *INDENT-OFF* */
+  vec_foreach_index (i, cwm->qp_data)
+    {
+      u32 enq;
+
+      qpd = vec_elt_at_index(cwm->qp_data, i);
+      enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id,
+					qpd->cops, n_cop_qp[i]);
+      qpd->inflights += enq;
+
+      if (PREDICT_FALSE(enq < n_cop_qp[i]))
+	{
+	  crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq);
+	  vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq);
+
+	  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+				       ESP_DECRYPT_ERROR_ENQ_FAIL,
+				       n_cop_qp[i] - enq);
+	}
+    }
+  /* *INDENT-ON* */
+
+  return from_frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = {
+  .function = dpdk_esp_decrypt_node_fn,
+  .name = "dpdk-esp-decrypt",
+  .vector_size = sizeof (u32),
+  .format_trace = format_esp_decrypt_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(esp_decrypt_error_strings),
+  .error_strings = esp_decrypt_error_strings,
+
+  .n_next_nodes = ESP_DECRYPT_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
+    foreach_esp_decrypt_next
+#undef _
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn)
+
+/*
+ * Decrypt Post Node
+ */
+
+#define foreach_esp_decrypt_post_error	      \
+ _(PKTS, "ESP post pkts")
+
+typedef enum {
+#define _(sym,str) ESP_DECRYPT_POST_ERROR_##sym,
+  foreach_esp_decrypt_post_error
+#undef _
+  ESP_DECRYPT_POST_N_ERROR,
+} esp_decrypt_post_error_t;
+
+static char * esp_decrypt_post_error_strings[] = {
+#define _(sym,string) string,
+  foreach_esp_decrypt_post_error
+#undef _
+};
+
+vlib_node_registration_t dpdk_esp_decrypt_post_node;
+
+static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args)
+{
+  return s;
+}
+
+static uword
+dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm,
+	     vlib_node_runtime_t * node,
+	     vlib_frame_t * from_frame)
+{
+  u32 n_left_from, *from, *to_next = 0, next_index;
+  ipsec_sa_t * sa0;
+  u32 sa_index0 = ~0;
+  ipsec_main_t *im = &ipsec_main;
+  dpdk_esp_main_t *em = &dpdk_esp_main;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  esp_footer_t * f0;
+	  u32 bi0, next0, icv_size, iv_size;
+	  vlib_buffer_t * b0 = 0;
+	  ip4_header_t *ih4 = 0, *oh4 = 0;
+	  ip6_header_t *ih6 = 0, *oh6 = 0;
+	  u8 tunnel_mode = 1;
+	  u8 transport_ip6 = 0;
+
+	  next0 = ESP_DECRYPT_NEXT_DROP;
+
+	  bi0 = from[0];
+	  from += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  sa_index0 = vnet_buffer(b0)->ipsec.sad_index;
+	  sa0 = pool_elt_at_index (im->sad, sa_index0);
+
+	  to_next[0] = bi0;
+	  to_next += 1;
+
+	  icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
+	  iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len;
+
+	  if (sa0->use_anti_replay)
+	    {
+	      esp_header_t * esp0 = vlib_buffer_get_current (b0);
+	      u32 seq;
+	      seq = clib_host_to_net_u32(esp0->seq);
+	      if (PREDICT_TRUE(sa0->use_esn))
+		esp_replay_advance_esn(sa0, seq);
+	      else
+		esp_replay_advance(sa0, seq);
+	    }
+
+	  ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t));
+	  vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size);
+
+	  b0->current_length -= (icv_size + 2);
+	  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
+	  f0 = (esp_footer_t *) ((u8 *) vlib_buffer_get_current (b0) +
+				 b0->current_length);
+	  b0->current_length -= f0->pad_length;
+
+	  /* transport mode */
+	  if (PREDICT_FALSE(!sa0->is_tunnel && !sa0->is_tunnel_ip6))
+	    {
+	      tunnel_mode = 0;
+
+	      if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) != 0x40))
+		{
+		  if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) == 0x60))
+		    transport_ip6 = 1;
+		  else
+		    {
+		      clib_warning("next header: 0x%x", f0->next_header);
+		      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+						   ESP_DECRYPT_ERROR_NOT_IP, 1);
+		      goto trace;
+		    }
+		}
+	    }
+
+	  if (PREDICT_TRUE (tunnel_mode))
+	    {
+	      if (PREDICT_TRUE(f0->next_header == IP_PROTOCOL_IP_IN_IP))
+		next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
+	      else if (f0->next_header == IP_PROTOCOL_IPV6)
+		next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
+	      else
+		{
+		  clib_warning("next header: 0x%x", f0->next_header);
+		  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
+					       ESP_DECRYPT_ERROR_DECRYPTION_FAILED,
+					       1);
+		  goto trace;
+		}
+	    }
+	  /* transport mode */
+	  else
+	    {
+	      if (PREDICT_FALSE(transport_ip6))
+		{
+		  ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t));
+		  vlib_buffer_advance (b0, -sizeof(ip6_header_t));
+		  oh6 = vlib_buffer_get_current (b0);
+		  memmove(oh6, ih6, sizeof(ip6_header_t));
+
+		  next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
+		  oh6->protocol = f0->next_header;
+		  oh6->payload_length =
+		      clib_host_to_net_u16 (
+			  vlib_buffer_length_in_chain(vm, b0) -
+			  sizeof (ip6_header_t));
+		}
+	      else
+		{
+		  vlib_buffer_advance (b0, -sizeof(ip4_header_t));
+		  oh4 = vlib_buffer_get_current (b0);
+		  memmove(oh4, ih4, sizeof(ip4_header_t));
+
+		  next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
+		  oh4->ip_version_and_header_length = 0x45;
+		  oh4->fragment_id = 0;
+		  oh4->flags_and_fragment_offset = 0;
+		  oh4->protocol = f0->next_header;
+		  oh4->length = clib_host_to_net_u16 (
+		      vlib_buffer_length_in_chain (vm, b0));
+		  oh4->checksum = ip4_header_checksum (oh4);
+		}
+	    }
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32)~0;
+
+trace:
+	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      tr->crypto_alg = sa0->crypto_alg;
+	      tr->integ_alg = sa0->integ_alg;
+	    }
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index,
+			       ESP_DECRYPT_POST_ERROR_PKTS,
+			       from_frame->n_vectors);
+
+  return from_frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = {
+  .function = dpdk_esp_decrypt_post_node_fn,
+  .name = "dpdk-esp-decrypt-post",
+  .vector_size = sizeof (u32),
+  .format_trace = format_esp_decrypt_post_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings),
+  .error_strings = esp_decrypt_post_error_strings,
+
+  .n_next_nodes = ESP_DECRYPT_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
+    foreach_esp_decrypt_next
+#undef _
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn)
diff --git a/src/plugins/dpdk/ipsec/esp_encrypt.c b/src/plugins/dpdk/ipsec/esp_encrypt.c
new file mode 100644
index 00000000..5b03de73
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/esp_encrypt.c
@@ -0,0 +1,609 @@
+/*
+ * esp_encrypt.c : IPSec ESP encrypt node using DPDK Cryptodev
+ *
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define foreach_esp_encrypt_next                   \
+_(DROP, "error-drop")                              \
+_(IP4_LOOKUP, "ip4-lookup")                        \
+_(IP6_LOOKUP, "ip6-lookup")                        \
+_(INTERFACE_OUTPUT, "interface-output")
+
+#define _(v, s) ESP_ENCRYPT_NEXT_##v,
+typedef enum
+{
+  foreach_esp_encrypt_next
+#undef _
+    ESP_ENCRYPT_N_NEXT,
+} esp_encrypt_next_t;
+
+#define foreach_esp_encrypt_error                   \
+ _(RX_PKTS, "ESP pkts received")                    \
+ _(SEQ_CYCLED, "sequence number cycled")            \
+ _(ENQ_FAIL, "Enqueue failed (buffer full)")        \
+ _(NO_CRYPTODEV, "Cryptodev not configured")        \
+ _(UNSUPPORTED, "Cipher/Auth not supported")
+
+
+typedef enum
+{
+#define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
+  foreach_esp_encrypt_error
+#undef _
+    ESP_ENCRYPT_N_ERROR,
+} esp_encrypt_error_t;
+
+static char *esp_encrypt_error_strings[] = {
+#define _(sym,string) string,
+  foreach_esp_encrypt_error
+#undef _
+};
+
+vlib_node_registration_t dpdk_esp_encrypt_node;
+
+typedef struct
+{
+  u32 spi;
+  u32 seq;
+  ipsec_crypto_alg_t crypto_alg;
+  ipsec_integ_alg_t integ_alg;
+} esp_encrypt_trace_t;
+
+/* packet trace format function */
+static u8 *
+format_esp_encrypt_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
+
+  s = format (s, "esp: spi %u seq %u crypto %U integrity %U",
+	      t->spi, t->seq,
+	      format_ipsec_crypto_alg, t->crypto_alg,
+	      format_ipsec_integ_alg, t->integ_alg);
+  return s;
+}
+
+static uword
+dpdk_esp_encrypt_node_fn (vlib_main_t * vm,
+			  vlib_node_runtime_t * node,
+			  vlib_frame_t * from_frame)
+{
+  u32 n_left_from, *from, *to_next, next_index;
+  ipsec_main_t *im = &ipsec_main;
+  u32 cpu_index = os_get_cpu_number ();
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  dpdk_esp_main_t *em = &dpdk_esp_main;
+  u32 i;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  if (PREDICT_FALSE (!dcm->workers_main))
+    {
+      /* Likely there are not enough cryptodevs, so drop frame */
+      vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
+				   ESP_ENCRYPT_ERROR_NO_CRYPTODEV,
+				   n_left_from);
+      vlib_buffer_free (vm, from, n_left_from);
+      return n_left_from;
+    }
+
+  crypto_worker_main_t *cwm = vec_elt_at_index (dcm->workers_main, cpu_index);
+  u32 n_qps = vec_len (cwm->qp_data);
+  struct rte_crypto_op **cops_to_enq[n_qps];
+  u32 n_cop_qp[n_qps], *bi_to_enq[n_qps];
+
+  for (i = 0; i < n_qps; i++)
+    {
+      bi_to_enq[i] = cwm->qp_data[i].bi;
+      cops_to_enq[i] = cwm->qp_data[i].cops;
+    }
+
+  memset (n_cop_qp, 0, n_qps * sizeof (u32));
+
+  crypto_alloc_cops ();
+
+  next_index = ESP_ENCRYPT_NEXT_DROP;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0, next0;
+	  vlib_buffer_t *b0 = 0;
+	  u32 sa_index0;
+	  ipsec_sa_t *sa0;
+	  ip4_and_esp_header_t *ih0, *oh0 = 0;
+	  ip6_and_esp_header_t *ih6_0, *oh6_0 = 0;
+	  struct rte_mbuf *mb0 = 0;
+	  esp_footer_t *f0;
+	  u8 is_ipv6;
+	  u8 ip_hdr_size;
+	  u8 next_hdr_type;
+	  u8 transport_mode = 0;
+	  const int BLOCK_SIZE = 16;
+	  u32 iv_size;
+	  u16 orig_sz;
+	  crypto_sa_session_t *sa_sess;
+	  void *sess;
+	  struct rte_crypto_op *cop = 0;
+	  u16 qp_index;
+
+	  bi0 = from[0];
+	  from += 1;
+	  n_left_from -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  sa_index0 = vnet_buffer (b0)->ipsec.sad_index;
+	  sa0 = pool_elt_at_index (im->sad, sa_index0);
+
+	  if (PREDICT_FALSE (esp_seq_advance (sa0)))
+	    {
+	      clib_warning ("sequence number counter has cycled SPI %u",
+			    sa0->spi);
+	      vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
+					   ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
+	      //TODO: rekey SA
+	      to_next[0] = bi0;
+	      to_next += 1;
+	      n_left_to_next -= 1;
+	      goto trace;
+	    }
+
+	  sa0->total_data_size += b0->current_length;
+
+	  sa_sess = pool_elt_at_index (cwm->sa_sess_d[1], sa_index0);
+	  if (PREDICT_FALSE (!sa_sess->sess))
+	    {
+	      int ret = create_sym_sess (sa0, sa_sess, 1);
+
+	      if (PREDICT_FALSE (ret))
+		{
+		  to_next[0] = bi0;
+		  to_next += 1;
+		  n_left_to_next -= 1;
+		  goto trace;
+		}
+	    }
+
+	  qp_index = sa_sess->qp_index;
+	  sess = sa_sess->sess;
+
+	  ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0);
+	  cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops);
+	  ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
+
+	  cops_to_enq[qp_index][0] = cop;
+	  cops_to_enq[qp_index] += 1;
+	  n_cop_qp[qp_index] += 1;
+	  bi_to_enq[qp_index][0] = bi0;
+	  bi_to_enq[qp_index] += 1;
+
+	  ssize_t adv;
+	  iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len;
+	  ih0 = vlib_buffer_get_current (b0);
+	  orig_sz = b0->current_length;
+	  is_ipv6 = (ih0->ip4.ip_version_and_header_length & 0xF0) == 0x60;
+	  /* is ipv6 */
+	  if (PREDICT_TRUE (sa0->is_tunnel))
+	    {
+	      if (PREDICT_TRUE (!is_ipv6))
+		adv = -sizeof (ip4_and_esp_header_t);
+	      else
+		adv = -sizeof (ip6_and_esp_header_t);
+	    }
+	  else
+	    {
+	      adv = -sizeof (esp_header_t);
+	      if (PREDICT_TRUE (!is_ipv6))
+		orig_sz -= sizeof (ip4_header_t);
+	      else
+		orig_sz -= sizeof (ip6_header_t);
+	    }
+
+	  /*transport mode save the eth header before it is overwritten */
+	  if (PREDICT_FALSE (!sa0->is_tunnel))
+	    {
+	      ethernet_header_t *ieh0 = (ethernet_header_t *)
+		((u8 *) vlib_buffer_get_current (b0) -
+		 sizeof (ethernet_header_t));
+	      ethernet_header_t *oeh0 =
+		(ethernet_header_t *) ((u8 *) ieh0 + (adv - iv_size));
+	      clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t));
+	    }
+
+	  vlib_buffer_advance (b0, adv - iv_size);
+
+	  /* XXX IP6/ip4 and IP4/IP6 not supported, only IP4/IP4 and IP6/IP6 */
+
+	  /* is ipv6 */
+	  if (PREDICT_FALSE (is_ipv6))
+	    {
+	      ih6_0 = (ip6_and_esp_header_t *) ih0;
+	      ip_hdr_size = sizeof (ip6_header_t);
+	      oh6_0 = vlib_buffer_get_current (b0);
+
+	      if (PREDICT_TRUE (sa0->is_tunnel))
+		{
+		  next_hdr_type = IP_PROTOCOL_IPV6;
+		  oh6_0->ip6.ip_version_traffic_class_and_flow_label =
+		    ih6_0->ip6.ip_version_traffic_class_and_flow_label;
+		}
+	      else
+		{
+		  next_hdr_type = ih6_0->ip6.protocol;
+		  memmove (oh6_0, ih6_0, sizeof (ip6_header_t));
+		}
+
+	      oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
+	      oh6_0->ip6.hop_limit = 254;
+	      oh6_0->esp.spi = clib_net_to_host_u32 (sa0->spi);
+	      oh6_0->esp.seq = clib_net_to_host_u32 (sa0->seq);
+	    }
+	  else
+	    {
+	      ip_hdr_size = sizeof (ip4_header_t);
+	      oh0 = vlib_buffer_get_current (b0);
+
+	      if (PREDICT_TRUE (sa0->is_tunnel))
+		{
+		  next_hdr_type = IP_PROTOCOL_IP_IN_IP;
+		  oh0->ip4.tos = ih0->ip4.tos;
+		}
+	      else
+		{
+		  next_hdr_type = ih0->ip4.protocol;
+		  memmove (oh0, ih0, sizeof (ip4_header_t));
+		}
+
+	      oh0->ip4.ip_version_and_header_length = 0x45;
+	      oh0->ip4.fragment_id = 0;
+	      oh0->ip4.flags_and_fragment_offset = 0;
+	      oh0->ip4.ttl = 254;
+	      oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
+	      oh0->esp.spi = clib_net_to_host_u32 (sa0->spi);
+	      oh0->esp.seq = clib_net_to_host_u32 (sa0->seq);
+	    }
+
+	  if (PREDICT_TRUE (sa0->is_tunnel && !sa0->is_tunnel_ip6))
+	    {
+	      oh0->ip4.src_address.as_u32 = sa0->tunnel_src_addr.ip4.as_u32;
+	      oh0->ip4.dst_address.as_u32 = sa0->tunnel_dst_addr.ip4.as_u32;
+
+	      /* in tunnel mode send it back to FIB */
+	      next0 = ESP_ENCRYPT_NEXT_IP4_LOOKUP;
+	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	    }
+	  else if (sa0->is_tunnel && sa0->is_tunnel_ip6)
+	    {
+	      oh6_0->ip6.src_address.as_u64[0] =
+		sa0->tunnel_src_addr.ip6.as_u64[0];
+	      oh6_0->ip6.src_address.as_u64[1] =
+		sa0->tunnel_src_addr.ip6.as_u64[1];
+	      oh6_0->ip6.dst_address.as_u64[0] =
+		sa0->tunnel_dst_addr.ip6.as_u64[0];
+	      oh6_0->ip6.dst_address.as_u64[1] =
+		sa0->tunnel_dst_addr.ip6.as_u64[1];
+
+	      /* in tunnel mode send it back to FIB */
+	      next0 = ESP_ENCRYPT_NEXT_IP6_LOOKUP;
+	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	    }
+	  else
+	    {
+	      next0 = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT;
+	      transport_mode = 1;
+	    }
+
+	  ASSERT (sa0->crypto_alg < IPSEC_CRYPTO_N_ALG);
+	  ASSERT (sa0->crypto_alg != IPSEC_CRYPTO_ALG_NONE);
+
+	  int blocks = 1 + (orig_sz + 1) / BLOCK_SIZE;
+
+	  /* pad packet in input buffer */
+	  u8 pad_bytes = BLOCK_SIZE * blocks - 2 - orig_sz;
+	  u8 i;
+	  u8 *padding = vlib_buffer_get_current (b0) + b0->current_length;
+
+	  for (i = 0; i < pad_bytes; ++i)
+	    padding[i] = i + 1;
+
+	  f0 = vlib_buffer_get_current (b0) + b0->current_length + pad_bytes;
+	  f0->pad_length = pad_bytes;
+	  f0->next_header = next_hdr_type;
+	  b0->current_length += pad_bytes + 2 +
+	    em->esp_integ_algs[sa0->integ_alg].trunc_size;
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
+	    vnet_buffer (b0)->sw_if_index[VLIB_RX];
+	  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
+
+	  struct rte_crypto_sym_op *sym_cop;
+	  sym_cop = (struct rte_crypto_sym_op *) (cop + 1);
+
+	  dpdk_cop_priv_t *priv = (dpdk_cop_priv_t *) (sym_cop + 1);
+
+	  vnet_buffer (b0)->unused[0] = next0;
+
+	  mb0 = rte_mbuf_from_vlib_buffer (b0);
+	  mb0->data_len = b0->current_length;
+	  mb0->pkt_len = b0->current_length;
+	  mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data;
+
+	  rte_crypto_op_attach_sym_session (cop, sess);
+
+	  sym_cop->m_src = mb0;
+
+	  dpdk_gcm_cnt_blk *icb = &priv->cb;
+	  icb->salt = sa0->salt;
+	  icb->iv[0] = sa0->seq;
+	  icb->iv[1] = sa0->seq_hi;
+
+	  if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
+	    {
+	      icb->cnt = clib_host_to_net_u32 (1);
+	      clib_memcpy (vlib_buffer_get_current (b0) + ip_hdr_size +
+			   sizeof (esp_header_t), icb->iv, 8);
+	      sym_cop->cipher.data.offset =
+		ip_hdr_size + sizeof (esp_header_t) + iv_size;
+	      sym_cop->cipher.data.length = BLOCK_SIZE * blocks;
+	      sym_cop->cipher.iv.length = 16;
+	    }
+	  else
+	    {
+	      sym_cop->cipher.data.offset =
+		ip_hdr_size + sizeof (esp_header_t);
+	      sym_cop->cipher.data.length = BLOCK_SIZE * blocks + iv_size;
+	      sym_cop->cipher.iv.length = iv_size;
+	    }
+
+	  sym_cop->cipher.iv.data = (u8 *) icb;
+	  sym_cop->cipher.iv.phys_addr = cop->phys_addr + (uintptr_t) icb
+	    - (uintptr_t) cop;
+
+
+	  ASSERT (sa0->integ_alg < IPSEC_INTEG_N_ALG);
+	  ASSERT (sa0->integ_alg != IPSEC_INTEG_ALG_NONE);
+
+	  if (PREDICT_FALSE (sa0->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128))
+	    {
+	      u8 *aad = priv->aad;
+	      clib_memcpy (aad, vlib_buffer_get_current (b0) + ip_hdr_size,
+			   8);
+	      sym_cop->auth.aad.data = aad;
+	      sym_cop->auth.aad.phys_addr = cop->phys_addr +
+		(uintptr_t) aad - (uintptr_t) cop;
+
+	      if (PREDICT_FALSE (sa0->use_esn))
+		{
+		  *((u32 *) & aad[8]) = sa0->seq_hi;
+		  sym_cop->auth.aad.length = 12;
+		}
+	      else
+		{
+		  sym_cop->auth.aad.length = 8;
+		}
+	    }
+	  else
+	    {
+	      sym_cop->auth.data.offset = ip_hdr_size;
+	      sym_cop->auth.data.length = b0->current_length - ip_hdr_size
+		- em->esp_integ_algs[sa0->integ_alg].trunc_size;
+
+	      if (PREDICT_FALSE (sa0->use_esn))
+		{
+		  u8 *payload_end =
+		    vlib_buffer_get_current (b0) + b0->current_length;
+		  *((u32 *) payload_end) = sa0->seq_hi;
+		  sym_cop->auth.data.length += sizeof (sa0->seq_hi);
+		}
+	    }
+	  sym_cop->auth.digest.data = vlib_buffer_get_current (b0) +
+	    b0->current_length -
+	    em->esp_integ_algs[sa0->integ_alg].trunc_size;
+	  sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset (mb0,
+								       b0->current_length
+								       -
+								       em->esp_integ_algs
+								       [sa0->integ_alg].trunc_size);
+	  sym_cop->auth.digest.length =
+	    em->esp_integ_algs[sa0->integ_alg].trunc_size;
+
+
+	  if (PREDICT_FALSE (is_ipv6))
+	    {
+	      oh6_0->ip6.payload_length =
+		clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
+				      sizeof (ip6_header_t));
+	    }
+	  else
+	    {
+	      oh0->ip4.length =
+		clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
+	      oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4);
+	    }
+
+	  if (transport_mode)
+	    vlib_buffer_advance (b0, -sizeof (ethernet_header_t));
+
+	trace:
+	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      esp_encrypt_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      tr->spi = sa0->spi;
+	      tr->seq = sa0->seq - 1;
+	      tr->crypto_alg = sa0->crypto_alg;
+	      tr->integ_alg = sa0->integ_alg;
+	    }
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
+			       ESP_ENCRYPT_ERROR_RX_PKTS,
+			       from_frame->n_vectors);
+  crypto_qp_data_t *qpd;
+  /* *INDENT-OFF* */
+  vec_foreach_index (i, cwm->qp_data)
+    {
+      u32 enq;
+
+      qpd = vec_elt_at_index(cwm->qp_data, i);
+      enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id,
+					qpd->cops, n_cop_qp[i]);
+      qpd->inflights += enq;
+
+      if (PREDICT_FALSE(enq < n_cop_qp[i]))
+	{
+	  crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq);
+	  vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq);
+
+          vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
+				       ESP_ENCRYPT_ERROR_ENQ_FAIL,
+				       n_cop_qp[i] - enq);
+        }
+    }
+  /* *INDENT-ON* */
+
+  return from_frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (dpdk_esp_encrypt_node) =
+{
+  .function = dpdk_esp_encrypt_node_fn,.name = "dpdk-esp-encrypt",.flags =
+    VLIB_NODE_FLAG_IS_OUTPUT,.vector_size = sizeof (u32),.format_trace =
+    format_esp_encrypt_trace,.n_errors =
+    ARRAY_LEN (esp_encrypt_error_strings),.error_strings =
+    esp_encrypt_error_strings,.n_next_nodes = 1,.next_nodes =
+  {
+  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",}
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_encrypt_node, dpdk_esp_encrypt_node_fn)
+/*
+ * ESP Encrypt Post Node
+ */
+#define foreach_esp_encrypt_post_error              \
+ _(PKTS, "ESP post pkts")
+     typedef enum
+     {
+#define _(sym,str) ESP_ENCRYPT_POST_ERROR_##sym,
+       foreach_esp_encrypt_post_error
+#undef _
+	 ESP_ENCRYPT_POST_N_ERROR,
+     } esp_encrypt_post_error_t;
+
+     static char *esp_encrypt_post_error_strings[] = {
+#define _(sym,string) string,
+       foreach_esp_encrypt_post_error
+#undef _
+     };
+
+vlib_node_registration_t dpdk_esp_encrypt_post_node;
+
+static u8 *
+format_esp_encrypt_post_trace (u8 * s, va_list * args)
+{
+  return s;
+}
+
+static uword
+dpdk_esp_encrypt_post_node_fn (vlib_main_t * vm,
+			       vlib_node_runtime_t * node,
+			       vlib_frame_t * from_frame)
+{
+  u32 n_left_from, *from, *to_next = 0, next_index;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0, next0;
+	  vlib_buffer_t *b0 = 0;
+
+	  bi0 = from[0];
+	  from += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  to_next[0] = bi0;
+	  to_next += 1;
+
+	  next0 = vnet_buffer (b0)->unused[0];
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next, bi0,
+					   next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, dpdk_esp_encrypt_post_node.index,
+			       ESP_ENCRYPT_POST_ERROR_PKTS,
+			       from_frame->n_vectors);
+
+  return from_frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (dpdk_esp_encrypt_post_node) =
+{
+  .function = dpdk_esp_encrypt_post_node_fn,.name =
+    "dpdk-esp-encrypt-post",.vector_size = sizeof (u32),.format_trace =
+    format_esp_encrypt_post_trace,.type = VLIB_NODE_TYPE_INTERNAL,.n_errors =
+    ARRAY_LEN (esp_encrypt_post_error_strings),.error_strings =
+    esp_encrypt_post_error_strings,.n_next_nodes =
+    ESP_ENCRYPT_N_NEXT,.next_nodes =
+  {
+#define _(s,n) [ESP_ENCRYPT_NEXT_##s] = n,
+    foreach_esp_encrypt_next
+#undef _
+  }
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_encrypt_post_node,
+			      dpdk_esp_encrypt_post_node_fn)
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/ipsec/ipsec.c b/src/plugins/dpdk/ipsec/ipsec.c
new file mode 100644
index 00000000..16bec20a
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/ipsec.c
@@ -0,0 +1,430 @@
+/*
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#define DPDK_CRYPTO_NB_SESS_OBJS  20000
+#define DPDK_CRYPTO_CACHE_SIZE	  512
+#define DPDK_CRYPTO_PRIV_SIZE	  128
+#define DPDK_CRYPTO_N_QUEUE_DESC  1024
+#define DPDK_CRYPTO_NB_COPS	  (1024 * 4)
+
+static int
+add_del_sa_sess (u32 sa_index, u8 is_add)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_worker_main_t *cwm;
+  u8 skip_master = vlib_num_workers () > 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (cwm, dcm->workers_main)
+    {
+      crypto_sa_session_t *sa_sess;
+      u8 is_outbound;
+
+      if (skip_master)
+	{
+	  skip_master = 0;
+	  continue;
+	}
+
+      for (is_outbound = 0; is_outbound < 2; is_outbound++)
+	{
+	  if (is_add)
+	    {
+	      pool_get (cwm->sa_sess_d[is_outbound], sa_sess);
+	    }
+	  else
+	    {
+	      u8 dev_id;
+
+	      sa_sess = pool_elt_at_index (cwm->sa_sess_d[is_outbound], sa_index);
+	      dev_id = cwm->qp_data[sa_sess->qp_index].dev_id;
+
+	      if (!sa_sess->sess)
+		continue;
+
+	      if (rte_cryptodev_sym_session_free(dev_id, sa_sess->sess))
+		{
+		  clib_warning("failed to free session");
+		  return -1;
+		}
+	      memset(sa_sess, 0, sizeof(sa_sess[0]));
+	    }
+	}
+    }
+  /* *INDENT-OFF* */
+
+  return 0;
+}
+
+static void
+update_qp_data (crypto_worker_main_t * cwm,
+		u8 cdev_id, u16 qp_id, u8 is_outbound, u16 * idx)
+{
+  crypto_qp_data_t *qpd;
+
+  /* *INDENT-OFF* */
+  vec_foreach_index (*idx, cwm->qp_data)
+    {
+      qpd = vec_elt_at_index(cwm->qp_data, *idx);
+
+      if (qpd->dev_id == cdev_id && qpd->qp_id == qp_id &&
+	  qpd->is_outbound == is_outbound)
+	  return;
+    }
+  /* *INDENT-ON* */
+
+  vec_add2 (cwm->qp_data, qpd, 1);
+
+  qpd->dev_id = cdev_id;
+  qpd->qp_id = qp_id;
+  qpd->is_outbound = is_outbound;
+}
+
+/*
+ * return:
+ * 	0: already exist
+ * 	1: mapped
+ */
+static int
+add_mapping (crypto_worker_main_t * cwm,
+	     u8 cdev_id, u16 qp, u8 is_outbound,
+	     const struct rte_cryptodev_capabilities *cipher_cap,
+	     const struct rte_cryptodev_capabilities *auth_cap)
+{
+  u16 qp_index;
+  uword key = 0, data, *ret;
+  crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
+
+  p_key->cipher_algo = (u8) cipher_cap->sym.cipher.algo;
+  p_key->auth_algo = (u8) auth_cap->sym.auth.algo;
+  p_key->is_outbound = is_outbound;
+
+  ret = hash_get (cwm->algo_qp_map, key);
+  if (ret)
+    return 0;
+
+  update_qp_data (cwm, cdev_id, qp, is_outbound, &qp_index);
+
+  data = (uword) qp_index;
+  hash_set (cwm->algo_qp_map, key, data);
+
+  return 1;
+}
+
+/*
+ * return:
+ * 	0: already exist
+ * 	1: mapped
+ */
+static int
+add_cdev_mapping (crypto_worker_main_t * cwm,
+		  struct rte_cryptodev_info *dev_info, u8 cdev_id,
+		  u16 qp, u8 is_outbound)
+{
+  const struct rte_cryptodev_capabilities *i, *j;
+  u32 mapped = 0;
+
+  for (i = dev_info->capabilities; i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++)
+    {
+      if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
+	continue;
+
+      if (check_algo_is_supported (i, NULL) != 0)
+	continue;
+
+      for (j = dev_info->capabilities; j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED;
+	   j++)
+	{
+	  if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
+	    continue;
+
+	  if (check_algo_is_supported (j, NULL) != 0)
+	    continue;
+
+	  mapped |= add_mapping (cwm, cdev_id, qp, is_outbound, i, j);
+	}
+    }
+
+  return mapped;
+}
+
+static int
+check_cryptodev_queues ()
+{
+  u32 n_qs = 0;
+  u8 cdev_id;
+  u32 n_req_qs = 2;
+
+  if (vlib_num_workers () > 0)
+    n_req_qs = vlib_num_workers () * 2;
+
+  for (cdev_id = 0; cdev_id < rte_cryptodev_count (); cdev_id++)
+    {
+      struct rte_cryptodev_info cdev_info;
+
+      rte_cryptodev_info_get (cdev_id, &cdev_info);
+
+      if (!
+	  (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
+	continue;
+
+      n_qs += cdev_info.max_nb_queue_pairs;
+    }
+
+  if (n_qs >= n_req_qs)
+    return 0;
+  else
+    return -1;
+}
+
+static clib_error_t *
+dpdk_ipsec_check_support (ipsec_sa_t * sa)
+{
+  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
+    {
+      if (sa->integ_alg != IPSEC_INTEG_ALG_NONE)
+	return clib_error_return (0, "unsupported integ-alg %U with "
+				  "crypto-algo aes-gcm-128",
+				  format_ipsec_integ_alg, sa->integ_alg);
+      sa->integ_alg = IPSEC_INTEG_ALG_AES_GCM_128;
+    }
+  else
+    {
+      if (sa->integ_alg == IPSEC_INTEG_ALG_NONE ||
+	  sa->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128)
+	return clib_error_return (0, "unsupported integ-alg %U",
+				  format_ipsec_integ_alg, sa->integ_alg);
+    }
+
+  return 0;
+}
+
+static uword
+dpdk_ipsec_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
+		    vlib_frame_t * f)
+{
+  dpdk_config_main_t *conf = &dpdk_config_main;
+  ipsec_main_t *im = &ipsec_main;
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  struct rte_cryptodev_config dev_conf;
+  struct rte_cryptodev_qp_conf qp_conf;
+  struct rte_cryptodev_info cdev_info;
+  struct rte_mempool *rmp;
+  i32 dev_id, ret;
+  u32 i, skip_master;
+
+  if (!conf->cryptodev)
+    {
+      clib_warning ("DPDK Cryptodev support is disabled, "
+		    "default to OpenSSL IPsec");
+      return 0;
+    }
+
+  if (check_cryptodev_queues () < 0)
+    {
+      conf->cryptodev = 0;
+      clib_warning ("not enough Cryptodevs, default to OpenSSL IPsec");
+      return 0;
+    }
+
+  vec_alloc (dcm->workers_main, tm->n_vlib_mains);
+  _vec_len (dcm->workers_main) = tm->n_vlib_mains;
+
+  fprintf (stdout, "DPDK Cryptodevs info:\n");
+  fprintf (stdout, "dev_id\tn_qp\tnb_obj\tcache_size\n");
+  /* HW cryptodevs have higher dev_id, use HW first */
+  for (dev_id = rte_cryptodev_count () - 1; dev_id >= 0; dev_id--)
+    {
+      u16 max_nb_qp, qp = 0;
+      skip_master = vlib_num_workers () > 0;
+
+      rte_cryptodev_info_get (dev_id, &cdev_info);
+
+      if (!
+	  (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
+	continue;
+
+      max_nb_qp = cdev_info.max_nb_queue_pairs;
+
+      for (i = 0; i < tm->n_vlib_mains; i++)
+	{
+	  u8 is_outbound;
+	  crypto_worker_main_t *cwm;
+	  uword *map;
+
+	  if (skip_master)
+	    {
+	      skip_master = 0;
+	      continue;
+	    }
+
+	  cwm = vec_elt_at_index (dcm->workers_main, i);
+	  map = cwm->algo_qp_map;
+
+	  if (!map)
+	    {
+	      map = hash_create (0, sizeof (crypto_worker_qp_key_t));
+	      if (!map)
+		{
+		  clib_warning ("unable to create hash table for worker %u",
+				vlib_mains[i]->cpu_index);
+		  goto error;
+		}
+	      cwm->algo_qp_map = map;
+	    }
+
+	  for (is_outbound = 0; is_outbound < 2 && qp < max_nb_qp;
+	       is_outbound++)
+	    qp += add_cdev_mapping (cwm, &cdev_info, dev_id, qp, is_outbound);
+	}
+
+      if (qp == 0)
+	continue;
+
+      dev_conf.socket_id = rte_cryptodev_socket_id (dev_id);
+      dev_conf.nb_queue_pairs = cdev_info.max_nb_queue_pairs;
+      dev_conf.session_mp.nb_objs = DPDK_CRYPTO_NB_SESS_OBJS;
+      dev_conf.session_mp.cache_size = DPDK_CRYPTO_CACHE_SIZE;
+
+      ret = rte_cryptodev_configure (dev_id, &dev_conf);
+      if (ret < 0)
+	{
+	  clib_warning ("cryptodev %u config error", dev_id);
+	  goto error;
+	}
+
+      qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
+      for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++)
+	{
+	  ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf,
+						dev_conf.socket_id);
+	  if (ret < 0)
+	    {
+	      clib_warning ("cryptodev %u qp %u setup error", dev_id, qp);
+	      goto error;
+	    }
+	}
+      vec_validate_aligned (dcm->cop_pools, dev_conf.socket_id,
+			    CLIB_CACHE_LINE_BYTES);
+
+      if (!vec_elt (dcm->cop_pools, dev_conf.socket_id))
+	{
+	  u8 *pool_name = format (0, "crypto_op_pool_socket%u%c",
+				  dev_conf.socket_id, 0);
+
+	  rmp = rte_crypto_op_pool_create ((char *) pool_name,
+					   RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+					   DPDK_CRYPTO_NB_COPS *
+					   (1 + vlib_num_workers ()),
+					   DPDK_CRYPTO_CACHE_SIZE,
+					   DPDK_CRYPTO_PRIV_SIZE,
+					   dev_conf.socket_id);
+	  vec_free (pool_name);
+
+	  if (!rmp)
+	    {
+	      clib_warning ("failed to allocate mempool on socket %u",
+			    dev_conf.socket_id);
+	      goto error;
+	    }
+	  vec_elt (dcm->cop_pools, dev_conf.socket_id) = rmp;
+	}
+
+      fprintf (stdout, "%u\t%u\t%u\t%u\n", dev_id, dev_conf.nb_queue_pairs,
+	       DPDK_CRYPTO_NB_SESS_OBJS, DPDK_CRYPTO_CACHE_SIZE);
+    }
+
+  dpdk_esp_init ();
+
+  /* Add new next node and set as default */
+  vlib_node_t *node, *next_node;
+
+  next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-encrypt");
+  ASSERT (next_node);
+  node = vlib_get_node_by_name (vm, (u8 *) "ipsec-output-ip4");
+  ASSERT (node);
+  im->esp_encrypt_node_index = next_node->index;
+  im->esp_encrypt_next_index =
+    vlib_node_add_next (vm, node->index, next_node->index);
+
+  next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-decrypt");
+  ASSERT (next_node);
+  node = vlib_get_node_by_name (vm, (u8 *) "ipsec-input-ip4");
+  ASSERT (node);
+  im->esp_decrypt_node_index = next_node->index;
+  im->esp_decrypt_next_index =
+    vlib_node_add_next (vm, node->index, next_node->index);
+
+  im->cb.check_support_cb = dpdk_ipsec_check_support;
+  im->cb.add_del_sa_sess_cb = add_del_sa_sess;
+
+  if (vec_len (vlib_mains) == 0)
+    vlib_node_set_state (&vlib_global_main, dpdk_crypto_input_node.index,
+			 VLIB_NODE_STATE_POLLING);
+  else
+    for (i = 1; i < tm->n_vlib_mains; i++)
+      vlib_node_set_state (vlib_mains[i], dpdk_crypto_input_node.index,
+			   VLIB_NODE_STATE_POLLING);
+
+  /* TODO cryptodev counters */
+
+  return 0;
+
+error:
+  ;
+  crypto_worker_main_t *cwm;
+  struct rte_mempool **mp;
+  /* *INDENT-OFF* */
+  vec_foreach (cwm, dcm->workers_main)
+    hash_free (cwm->algo_qp_map);
+
+  vec_foreach (mp, dcm->cop_pools)
+    {
+      if (mp)
+	rte_mempool_free (mp[0]);
+    }
+  /* *INDENT-ON* */
+  vec_free (dcm->workers_main);
+  vec_free (dcm->cop_pools);
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
+    .function = dpdk_ipsec_process,
+    .type = VLIB_NODE_TYPE_PROCESS,
+    .name = "dpdk-ipsec-process",
+    .process_log2_n_stack_bytes = 17,
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h
new file mode 100644
index 00000000..3465b361
--- /dev/null
+++ b/src/plugins/dpdk/ipsec/ipsec.h
@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __DPDK_IPSEC_H__
+#define __DPDK_IPSEC_H__
+
+#include 
+
+#undef always_inline
+#include 
+#include 
+
+#if CLIB_DEBUG > 0
+#define always_inline static inline
+#else
+#define always_inline static inline __attribute__ ((__always_inline__))
+#endif
+
+
+#define MAX_QP_PER_LCORE 16
+
+typedef struct
+{
+  u32 salt;
+  u32 iv[2];
+  u32 cnt;
+} dpdk_gcm_cnt_blk;
+
+typedef struct
+{
+  dpdk_gcm_cnt_blk cb;
+  union
+  {
+    u8 aad[12];
+    u8 icv[64];
+  };
+} dpdk_cop_priv_t;
+
+typedef struct
+{
+  u8 cipher_algo;
+  u8 auth_algo;
+  u8 is_outbound;
+} crypto_worker_qp_key_t;
+
+typedef struct
+{
+  u16 dev_id;
+  u16 qp_id;
+  u16 is_outbound;
+  i16 inflights;
+  u32 bi[VLIB_FRAME_SIZE];
+  struct rte_crypto_op *cops[VLIB_FRAME_SIZE];
+  struct rte_crypto_op **free_cops;
+} crypto_qp_data_t;
+
+typedef struct
+{
+  u8 qp_index;
+  void *sess;
+} crypto_sa_session_t;
+
+typedef struct
+{
+  crypto_sa_session_t *sa_sess_d[2];
+  crypto_qp_data_t *qp_data;
+  uword *algo_qp_map;
+} crypto_worker_main_t;
+
+typedef struct
+{
+  struct rte_mempool **cop_pools;
+  crypto_worker_main_t *workers_main;
+} dpdk_crypto_main_t;
+
+dpdk_crypto_main_t dpdk_crypto_main;
+
+extern vlib_node_registration_t dpdk_crypto_input_node;
+
+#define CRYPTO_N_FREE_COPS (VLIB_FRAME_SIZE * 3)
+
+static_always_inline void
+crypto_alloc_cops ()
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  u32 cpu_index = os_get_cpu_number ();
+  crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
+  unsigned socket_id = rte_socket_id ();
+  crypto_qp_data_t *qpd;
+
+  /* *INDENT-OFF* */
+  vec_foreach (qpd, cwm->qp_data)
+    {
+      u32 l = vec_len (qpd->free_cops);
+
+      if (PREDICT_FALSE (l < VLIB_FRAME_SIZE))
+	{
+	  u32 n_alloc;
+
+	  if (PREDICT_FALSE (!qpd->free_cops))
+	    vec_alloc (qpd->free_cops, CRYPTO_N_FREE_COPS);
+
+	  n_alloc = rte_crypto_op_bulk_alloc (dcm->cop_pools[socket_id],
+					      RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+					      &qpd->free_cops[l],
+					      CRYPTO_N_FREE_COPS - l - 1);
+
+	  _vec_len (qpd->free_cops) = l + n_alloc;
+	}
+    }
+  /* *INDENT-ON* */
+}
+
+static_always_inline void
+crypto_free_cop (crypto_qp_data_t * qpd, struct rte_crypto_op **cops, u32 n)
+{
+  u32 l = vec_len (qpd->free_cops);
+
+  if (l + n >= CRYPTO_N_FREE_COPS)
+    {
+      l -= VLIB_FRAME_SIZE;
+      rte_mempool_put_bulk (cops[0]->mempool,
+			    (void **) &qpd->free_cops[l], VLIB_FRAME_SIZE);
+    }
+  clib_memcpy (&qpd->free_cops[l], cops, sizeof (*cops) * n);
+
+  _vec_len (qpd->free_cops) = l + n;
+}
+
+static_always_inline int
+check_algo_is_supported (const struct rte_cryptodev_capabilities *cap,
+			 char *name)
+{
+  struct
+  {
+    uint8_t cipher_algo;
+    enum rte_crypto_sym_xform_type type;
+    union
+    {
+      enum rte_crypto_auth_algorithm auth;
+      enum rte_crypto_cipher_algorithm cipher;
+    };
+    char *name;
+  } supported_algo[] =
+  {
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
+	RTE_CRYPTO_CIPHER_NULL,.name = "NULL"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
+	RTE_CRYPTO_CIPHER_AES_CBC,.name = "AES_CBC"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
+	RTE_CRYPTO_CIPHER_AES_CTR,.name = "AES_CTR"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
+	RTE_CRYPTO_CIPHER_3DES_CBC,.name = "3DES-CBC"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
+	RTE_CRYPTO_CIPHER_AES_GCM,.name = "AES-GCM"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
+	RTE_CRYPTO_AUTH_SHA1_HMAC,.name = "HMAC-SHA1"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
+	RTE_CRYPTO_AUTH_SHA256_HMAC,.name = "HMAC-SHA256"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
+	RTE_CRYPTO_AUTH_SHA384_HMAC,.name = "HMAC-SHA384"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
+	RTE_CRYPTO_AUTH_SHA512_HMAC,.name = "HMAC-SHA512"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
+	RTE_CRYPTO_AUTH_AES_XCBC_MAC,.name = "AES-XCBC-MAC"},
+    {
+    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
+	RTE_CRYPTO_AUTH_AES_GCM,.name = "AES-GCM"},
+    {
+      /* tail */
+  .type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED},};
+  uint32_t i = 0;
+
+  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+    return -1;
+
+  while (supported_algo[i].type != RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED)
+    {
+      if (cap->sym.xform_type == supported_algo[i].type)
+	{
+	  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+	       cap->sym.cipher.algo == supported_algo[i].cipher) ||
+	      (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+	       cap->sym.auth.algo == supported_algo[i].auth))
+	    {
+	      if (name)
+		strcpy (name, supported_algo[i].name);
+	      return 0;
+	    }
+	}
+
+      i++;
+    }
+
+  return -1;
+}
+
+#endif /* __DPDK_IPSEC_H__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c
new file mode 100644
index 00000000..8073a50a
--- /dev/null
+++ b/src/plugins/dpdk/main.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+/*
+ * Called by the dpdk driver's rte_delay_us() function.
+ * Return 0 to have the dpdk do a regular delay loop.
+ * Return 1 if to skip the delay loop because we are suspending
+ * the calling vlib process instead.
+ */
+int
+rte_delay_us_override (unsigned us)
+{
+  vlib_main_t *vm;
+
+  /* Don't bother intercepting for short delays */
+  if (us < 10)
+    return 0;
+
+  /*
+   * Only intercept if we are in a vlib process.
+   * If we are called from a vlib worker thread or the vlib main
+   * thread then do not intercept. (Must not be called from an
+   * independent pthread).
+   */
+  if (os_get_cpu_number () == 0)
+    {
+      /*
+       * We're in the vlib main thread or a vlib process. Make sure
+       * the process is running and we're not still initializing.
+       */
+      vm = vlib_get_main ();
+      if (vlib_in_process_context (vm))
+	{
+	  /* Only suspend for the admin_down_process */
+	  vlib_process_t *proc = vlib_get_current_process (vm);
+	  if (!(proc->flags & VLIB_PROCESS_IS_RUNNING) ||
+	      (proc->node_runtime.function != admin_up_down_process))
+	    return 0;
+
+	  f64 delay = 1e-6 * us;
+	  vlib_process_suspend (vm, delay);
+	  return 1;
+	}
+    }
+  return 0;			// no override
+}
+
+static void
+rte_delay_us_override_cb (unsigned us)
+{
+  if (rte_delay_us_override (us) == 0)
+    rte_delay_us_block (us);
+}
+
+static clib_error_t * dpdk_main_init (vlib_main_t * vm)
+{
+  dpdk_main_t * dm = &dpdk_main;
+  clib_error_t * error = 0;
+
+  dm->vlib_main = vm;
+  dm->vnet_main = vnet_get_main ();
+
+  if ((error = vlib_call_init_function (vm, dpdk_init)))
+    return error;
+
+  /* register custom delay function */
+  rte_delay_us_callback_register (rte_delay_us_override_cb);
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (dpdk_main_init);
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+};
+/* *INDENT-ON* */
diff --git a/src/plugins/dpdk/thread.c b/src/plugins/dpdk/thread.c
new file mode 100644
index 00000000..3a3fcc6c
--- /dev/null
+++ b/src/plugins/dpdk/thread.c
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+static clib_error_t *
+dpdk_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id)
+{
+  int r;
+  r = rte_eal_remote_launch (fp, (void *) w, lcore_id);
+  if (r)
+    return clib_error_return (0, "Failed to launch thread %u", lcore_id);
+  return 0;
+}
+
+static clib_error_t *
+dpdk_thread_set_lcore (u32 thread, u16 lcore)
+{
+  return 0;
+}
+
+static vlib_thread_callbacks_t callbacks = {
+  .vlib_launch_thread_cb = &dpdk_launch_thread,
+  .vlib_thread_set_lcore_cb = &dpdk_thread_set_lcore,
+};
+
+static clib_error_t *
+dpdk_thread_init (vlib_main_t * vm)
+{
+  vlib_thread_cb_register (vm, &callbacks);
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (dpdk_thread_init);
+
+/** @endcond */
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index 999f9869..14e78817 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -3970,13 +3970,6 @@ _(feature_enable_disable_reply)				\
 _(sw_interface_tag_add_del_reply)			\
 _(sw_interface_set_mtu_reply)
 
-#if DPDK > 0
-#define foreach_standard_dpdk_reply_retval_handler      \
-_(sw_interface_set_dpdk_hqos_pipe_reply)                \
-_(sw_interface_set_dpdk_hqos_subport_reply)             \
-_(sw_interface_set_dpdk_hqos_tctbl_reply)
-#endif
-
 #define _(n)                                    \
     static void vl_api_##n##_t_handler          \
     (vl_api_##n##_t * mp)                       \
@@ -4008,39 +4001,6 @@ foreach_standard_reply_retval_handler;
 foreach_standard_reply_retval_handler;
 #undef _
 
-#if DPDK > 0
-#define _(n)                                    \
-    static void vl_api_##n##_t_handler          \
-    (vl_api_##n##_t * mp)                       \
-    {                                           \
-        vat_main_t * vam = &vat_main;           \
-        i32 retval = ntohl(mp->retval);         \
-        if (vam->async_mode) {                  \
-            vam->async_errors += (retval < 0);  \
-        } else {                                \
-            vam->retval = retval;               \
-            vam->result_ready = 1;              \
-        }                                       \
-    }
-foreach_standard_dpdk_reply_retval_handler;
-#undef _
-
-#define _(n)                                    \
-    static void vl_api_##n##_t_handler_json     \
-    (vl_api_##n##_t * mp)                       \
-    {                                           \
-        vat_main_t * vam = &vat_main;           \
-        vat_json_node_t node;                   \
-        vat_json_init_object(&node);            \
-        vat_json_object_add_int(&node, "retval", ntohl(mp->retval));    \
-        vat_json_print(vam->ofp, &node);        \
-        vam->retval = ntohl(mp->retval);        \
-        vam->result_ready = 1;                  \
-    }
-foreach_standard_dpdk_reply_retval_handler;
-#undef _
-#endif
-
 /*
  * Table of message reply handlers, must include boilerplate handlers
  * we just generated
@@ -4272,16 +4232,6 @@ _(SW_INTERFACE_SET_MTU_REPLY, sw_interface_set_mtu_reply)               \
 _(IP_NEIGHBOR_DETAILS, ip_neighbor_details)                             \
 _(SW_INTERFACE_GET_TABLE_REPLY, sw_interface_get_table_reply)
 
-#if DPDK > 0
-#define foreach_vpe_dpdk_api_reply_msg                                  \
-_(SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY,                                \
-  sw_interface_set_dpdk_hqos_pipe_reply)                                \
-_(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY,                             \
-  sw_interface_set_dpdk_hqos_subport_reply)                             \
-_(SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY,                               \
-  sw_interface_set_dpdk_hqos_tctbl_reply)
-#endif
-
 typedef struct
 {
   u8 *name;
@@ -5081,226 +5031,6 @@ api_sw_interface_clear_stats (vat_main_t * vam)
   return ret;
 }
 
-#if DPDK >0
-static int
-api_sw_interface_set_dpdk_hqos_pipe (vat_main_t * vam)
-{
-  unformat_input_t *i = vam->input;
-  vl_api_sw_interface_set_dpdk_hqos_pipe_t *mp;
-  u32 sw_if_index;
-  u8 sw_if_index_set = 0;
-  u32 subport;
-  u8 subport_set = 0;
-  u32 pipe;
-  u8 pipe_set = 0;
-  u32 profile;
-  u8 profile_set = 0;
-  int ret;
-
-  /* Parse args required to build the message */
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "rx %U", api_unformat_sw_if_index, vam, &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "sw_if_index %u", &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "subport %u", &subport))
-	subport_set = 1;
-      else
-	if (unformat (i, "%U", api_unformat_sw_if_index, vam, &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "pipe %u", &pipe))
-	pipe_set = 1;
-      else if (unformat (i, "profile %u", &profile))
-	profile_set = 1;
-      else
-	break;
-    }
-
-  if (sw_if_index_set == 0)
-    {
-      errmsg ("missing interface name or sw_if_index");
-      return -99;
-    }
-
-  if (subport_set == 0)
-    {
-      errmsg ("missing subport ");
-      return -99;
-    }
-
-  if (pipe_set == 0)
-    {
-      errmsg ("missing pipe");
-      return -99;
-    }
-
-  if (profile_set == 0)
-    {
-      errmsg ("missing profile");
-      return -99;
-    }
-
-  M (SW_INTERFACE_SET_DPDK_HQOS_PIPE, mp);
-
-  mp->sw_if_index = ntohl (sw_if_index);
-  mp->subport = ntohl (subport);
-  mp->pipe = ntohl (pipe);
-  mp->profile = ntohl (profile);
-
-
-  S (mp);
-  W (ret);
-  return ret;
-}
-
-static int
-api_sw_interface_set_dpdk_hqos_subport (vat_main_t * vam)
-{
-  unformat_input_t *i = vam->input;
-  vl_api_sw_interface_set_dpdk_hqos_subport_t *mp;
-  u32 sw_if_index;
-  u8 sw_if_index_set = 0;
-  u32 subport;
-  u8 subport_set = 0;
-  u32 tb_rate = 1250000000;	/* 10GbE */
-  u32 tb_size = 1000000;
-  u32 tc_rate[] = { 1250000000, 1250000000, 1250000000, 1250000000 };
-  u32 tc_period = 10;
-  int ret;
-
-  /* Parse args required to build the message */
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "rx %U", api_unformat_sw_if_index, vam, &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "sw_if_index %u", &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "subport %u", &subport))
-	subport_set = 1;
-      else
-	if (unformat (i, "%U", api_unformat_sw_if_index, vam, &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "rate %u", &tb_rate))
-	{
-	  u32 tc_id;
-
-	  for (tc_id = 0; tc_id < (sizeof (tc_rate) / sizeof (tc_rate[0]));
-	       tc_id++)
-	    tc_rate[tc_id] = tb_rate;
-	}
-      else if (unformat (i, "bktsize %u", &tb_size))
-	;
-      else if (unformat (i, "tc0 %u", &tc_rate[0]))
-	;
-      else if (unformat (i, "tc1 %u", &tc_rate[1]))
-	;
-      else if (unformat (i, "tc2 %u", &tc_rate[2]))
-	;
-      else if (unformat (i, "tc3 %u", &tc_rate[3]))
-	;
-      else if (unformat (i, "period %u", &tc_period))
-	;
-      else
-	break;
-    }
-
-  if (sw_if_index_set == 0)
-    {
-      errmsg ("missing interface name or sw_if_index");
-      return -99;
-    }
-
-  if (subport_set == 0)
-    {
-      errmsg ("missing subport ");
-      return -99;
-    }
-
-  M (SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, mp);
-
-  mp->sw_if_index = ntohl (sw_if_index);
-  mp->subport = ntohl (subport);
-  mp->tb_rate = ntohl (tb_rate);
-  mp->tb_size = ntohl (tb_size);
-  mp->tc_rate[0] = ntohl (tc_rate[0]);
-  mp->tc_rate[1] = ntohl (tc_rate[1]);
-  mp->tc_rate[2] = ntohl (tc_rate[2]);
-  mp->tc_rate[3] = ntohl (tc_rate[3]);
-  mp->tc_period = ntohl (tc_period);
-
-  S (mp);
-  W (ret);
-  return ret;
-}
-
-static int
-api_sw_interface_set_dpdk_hqos_tctbl (vat_main_t * vam)
-{
-  unformat_input_t *i = vam->input;
-  vl_api_sw_interface_set_dpdk_hqos_tctbl_t *mp;
-  u32 sw_if_index;
-  u8 sw_if_index_set = 0;
-  u8 entry_set = 0;
-  u8 tc_set = 0;
-  u8 queue_set = 0;
-  u32 entry, tc, queue;
-  int ret;
-
-  /* Parse args required to build the message */
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "rx %U", api_unformat_sw_if_index, vam, &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "sw_if_index %u", &sw_if_index))
-	sw_if_index_set = 1;
-      else if (unformat (i, "entry %d", &entry))
-	entry_set = 1;
-      else if (unformat (i, "tc %d", &tc))
-	tc_set = 1;
-      else if (unformat (i, "queue %d", &queue))
-	queue_set = 1;
-      else
-	break;
-    }
-
-  if (sw_if_index_set == 0)
-    {
-      errmsg ("missing interface name or sw_if_index");
-      return -99;
-    }
-
-  if (entry_set == 0)
-    {
-      errmsg ("missing entry ");
-      return -99;
-    }
-
-  if (tc_set == 0)
-    {
-      errmsg ("missing traffic class ");
-      return -99;
-    }
-
-  if (queue_set == 0)
-    {
-      errmsg ("missing queue ");
-      return -99;
-    }
-
-  M (SW_INTERFACE_SET_DPDK_HQOS_TCTBL, mp);
-
-  mp->sw_if_index = ntohl (sw_if_index);
-  mp->entry = ntohl (entry);
-  mp->tc = ntohl (tc);
-  mp->queue = ntohl (queue);
-
-  S (mp);
-  W (ret);
-  return ret;
-}
-#endif
-
 static int
 api_sw_interface_add_del_address (vat_main_t * vam)
 {
@@ -18656,18 +18386,6 @@ _(sw_interface_set_mtu, " | sw_if_index  mtu ")        \
 _(ip_neighbor_dump, "[ip6]  | sw_if_index ")                 \
 _(sw_interface_get_table, " | sw_if_index  [ipv6]")
 
-#if DPDK > 0
-#define foreach_vpe_dpdk_api_msg                                        \
-_(sw_interface_set_dpdk_hqos_pipe,                                      \
-  "rx  | sw_if_index  subport  pipe \n" \
-  "profile \n")                                             \
-_(sw_interface_set_dpdk_hqos_subport,                                   \
-  "rx  | sw_if_index  subport  [rate ]\n"     \
-  "[bktsize ] [tc0 ] [tc1 ] [tc2 ] [tc3 ] [period ]\n") \
-_(sw_interface_set_dpdk_hqos_tctbl,                                     \
-  "rx  | sw_if_index  entry  tc  queue \n")
-#endif
-
 /* List of command functions, CLI names map directly to functions */
 #define foreach_cli_function                                    \
 _(comment, "usage: comment ")		\
@@ -18705,22 +18423,6 @@ _(unset, "usage: unset ")
 foreach_vpe_api_reply_msg;
 #undef _
 
-#if DPDK > 0
-#define _(N,n)                                  \
-    static void vl_api_##n##_t_handler_uni      \
-    (vl_api_##n##_t * mp)                       \
-    {                                           \
-        vat_main_t * vam = &vat_main;           \
-        if (vam->json_output) {                 \
-            vl_api_##n##_t_handler_json(mp);    \
-        } else {                                \
-            vl_api_##n##_t_handler(mp);         \
-        }                                       \
-    }
-foreach_vpe_dpdk_api_reply_msg;
-#undef _
-#endif
-
 void
 vat_api_hookup (vat_main_t * vam)
 {
@@ -18734,18 +18436,6 @@ vat_api_hookup (vat_main_t * vam)
   foreach_vpe_api_reply_msg;
 #undef _
 
-#if DPDK > 0
-#define _(N,n)                                                  \
-    vl_msg_api_set_handlers(VL_API_##N, #n,                     \
-                           vl_api_##n##_t_handler_uni,          \
-                           vl_noop_handler,                     \
-                           vl_api_##n##_t_endian,               \
-                           vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
-  foreach_vpe_dpdk_api_reply_msg;
-#undef _
-#endif
-
 #if (VPP_API_TEST_BUILTIN==0)
   vl_msg_api_set_first_available_msg_id (VL_MSG_FIRST_AVAILABLE);
 #endif
@@ -18760,21 +18450,11 @@ vat_api_hookup (vat_main_t * vam)
 #define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
   foreach_vpe_api_msg;
 #undef _
-#if DPDK >0
-#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
-  foreach_vpe_dpdk_api_msg;
-#undef _
-#endif
 
   /* Help strings */
 #define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
   foreach_vpe_api_msg;
 #undef _
-#if DPDK >0
-#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
-  foreach_vpe_dpdk_api_msg;
-#undef _
-#endif
 
   /* CLI functions */
 #define _(n,h) hash_set_mem (vam->function_by_name, #n, n);
diff --git a/src/vnet.am b/src/vnet.am
index 923f61d8..84930f05 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -23,8 +23,7 @@ libvnet_la_DEPENDENCIES = \
 	libvlibmemory.la	\
 	libvlibmemoryclient.la
 
-libvnet_la_LIBADD = $(libvnet_la_DEPENDENCIES) -lm -lpthread -ldl -lrt $(DPDK_LD_ADD)
-libvnet_la_LDFLAGS = $(DPDK_LD_FLAGS)
+libvnet_la_LIBADD = $(libvnet_la_DEPENDENCIES) -lm -lpthread -ldl -lrt
 
 if WITH_LIBSSL
 libvnet_la_LIBADD += -lcrypto
@@ -396,15 +395,6 @@ libvnet_la_SOURCES +=				\
  vnet/ipsec/ipsec_api.c
 
 API_FILES += vnet/ipsec/ipsec.api
-
-if WITH_DPDK
-libvnet_la_SOURCES +=				\
- vnet/devices/dpdk/ipsec/esp_encrypt.c		\
- vnet/devices/dpdk/ipsec/esp_decrypt.c		\
- vnet/devices/dpdk/ipsec/crypto_node.c		\
- vnet/devices/dpdk/ipsec/cli.c			\
- vnet/devices/dpdk/ipsec/ipsec.c
-endif
 endif
 
 libvnet_la_SOURCES +=				\
@@ -416,11 +406,6 @@ nobase_include_HEADERS +=			\
  vnet/ipsec/ikev2.h				\
  vnet/ipsec/ikev2_priv.h			\
  vnet/ipsec/ipsec.api.h
-if WITH_DPDK
-nobase_include_HEADERS +=			\
- vnet/devices/dpdk/ipsec/ipsec.h		\
- vnet/devices/dpdk/ipsec/esp.h
-endif
 
 ########################################
 # Layer 3 protocol: osi
@@ -803,29 +788,7 @@ nobase_include_HEADERS +=			\
   vnet/pg/pg.h					\
   vnet/pg/edit.h
 
-########################################
-# DPDK
-########################################
-if WITH_DPDK
-libvnet_la_SOURCES +=				\
-  vnet/devices/dpdk/buffer.c			\
-  vnet/devices/dpdk/dpdk_priv.h		\
-  vnet/devices/dpdk/device.c		\
-  vnet/devices/dpdk/format.c		\
-  vnet/devices/dpdk/init.c			\
-  vnet/devices/dpdk/main.c			\
-  vnet/devices/dpdk/node.c			\
-  vnet/devices/dpdk/thread.c			\
-  vnet/devices/dpdk/hqos.c			\
-  vnet/devices/dpdk/cli.c			\
-  vnet/devices/dpdk/dpdk_api.c
-
-nobase_include_HEADERS +=			\
-  vnet/devices/dpdk/dpdk.h			\
-  vnet/devices/dpdk/dpdk.api.h
-
-API_FILES += vnet/devices/dpdk/dpdk.api
-else
+if !WITH_DPDK
 libvnet_la_SOURCES +=				\
   vnet/devices/nic/ixge.c                       \
   vnet/devices/nic/ixge.h                       \
diff --git a/src/vnet/devices/dpdk/buffer.c b/src/vnet/devices/dpdk/buffer.c
deleted file mode 100644
index f95d4cb5..00000000
--- a/src/vnet/devices/dpdk/buffer.c
+++ /dev/null
@@ -1,588 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- * buffer.c: allocate/free network buffers.
- *
- * Copyright (c) 2008 Eliot Dresselhaus
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/**
- * @file
- *
- * Allocate/free network buffers.
- */
-
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-
-
-STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
-	       "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
-
-static_always_inline void
-dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
-{
-  vlib_buffer_t *hb = b;
-  struct rte_mbuf *mb;
-  u32 next, flags;
-  mb = rte_mbuf_from_vlib_buffer (hb);
-
-next:
-  flags = b->flags;
-  next = b->next_buffer;
-  mb = rte_mbuf_from_vlib_buffer (b);
-
-  if (PREDICT_FALSE (b->n_add_refs))
-    {
-      rte_mbuf_refcnt_update (mb, b->n_add_refs);
-      b->n_add_refs = 0;
-    }
-
-  rte_pktmbuf_free_seg (mb);
-
-  if (flags & VLIB_BUFFER_NEXT_PRESENT)
-    {
-      b = vlib_get_buffer (vm, next);
-      goto next;
-    }
-}
-
-static void
-del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
-{
-  u32 i;
-  vlib_buffer_t *b;
-
-  for (i = 0; i < vec_len (f->buffers); i++)
-    {
-      b = vlib_get_buffer (vm, f->buffers[i]);
-      dpdk_rte_pktmbuf_free (vm, b);
-    }
-
-  vec_free (f->name);
-  vec_free (f->buffers);
-}
-
-/* Add buffer free list. */
-static void
-dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  vlib_buffer_free_list_t *f;
-  u32 merge_index;
-  int i;
-
-  ASSERT (os_get_cpu_number () == 0);
-
-  f = vlib_buffer_get_free_list (vm, free_list_index);
-
-  merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
-  if (merge_index != ~0 && merge_index != free_list_index)
-    {
-      vlib_buffer_merge_free_lists (pool_elt_at_index
-				    (bm->buffer_free_list_pool, merge_index),
-				    f);
-    }
-
-  del_free_list (vm, f);
-
-  /* Poison it. */
-  memset (f, 0xab, sizeof (f[0]));
-
-  pool_put (bm->buffer_free_list_pool, f);
-
-  for (i = 1; i < vec_len (vlib_mains); i++)
-    {
-      bm = vlib_mains[i]->buffer_main;
-      f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);;
-      memset (f, 0xab, sizeof (f[0]));
-      pool_put (bm->buffer_free_list_pool, f);
-    }
-}
-
-/* Make sure free list has at least given number of free buffers. */
-static uword
-fill_free_list (vlib_main_t * vm,
-		vlib_buffer_free_list_t * fl, uword min_free_buffers)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  vlib_buffer_t *b0, *b1, *b2, *b3;
-  int n, i;
-  u32 bi0, bi1, bi2, bi3;
-  unsigned socket_id = rte_socket_id ();
-  struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
-  struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
-
-  /* Too early? */
-  if (PREDICT_FALSE (rmp == 0))
-    return 0;
-
-  /* Already have enough free buffers on free list? */
-  n = min_free_buffers - vec_len (fl->buffers);
-  if (n <= 0)
-    return min_free_buffers;
-
-  /* Always allocate round number of buffers. */
-  n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
-
-  /* Always allocate new buffers in reasonably large sized chunks. */
-  n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
-
-  vec_validate (vm->mbuf_alloc_list, n - 1);
-
-  if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0)
-    return 0;
-
-  _vec_len (vm->mbuf_alloc_list) = n;
-
-  i = 0;
-
-  while (i < (n - 7))
-    {
-      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
-				   (vm->mbuf_alloc_list[i + 4]), STORE);
-      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
-				   (vm->mbuf_alloc_list[i + 5]), STORE);
-      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
-				   (vm->mbuf_alloc_list[i + 6]), STORE);
-      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
-				   (vm->mbuf_alloc_list[i + 7]), STORE);
-
-      mb0 = vm->mbuf_alloc_list[i];
-      mb1 = vm->mbuf_alloc_list[i + 1];
-      mb2 = vm->mbuf_alloc_list[i + 2];
-      mb3 = vm->mbuf_alloc_list[i + 3];
-
-      ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
-      ASSERT (rte_mbuf_refcnt_read (mb1) == 0);
-      ASSERT (rte_mbuf_refcnt_read (mb2) == 0);
-      ASSERT (rte_mbuf_refcnt_read (mb3) == 0);
-
-      rte_mbuf_refcnt_set (mb0, 1);
-      rte_mbuf_refcnt_set (mb1, 1);
-      rte_mbuf_refcnt_set (mb2, 1);
-      rte_mbuf_refcnt_set (mb3, 1);
-
-      b0 = vlib_buffer_from_rte_mbuf (mb0);
-      b1 = vlib_buffer_from_rte_mbuf (mb1);
-      b2 = vlib_buffer_from_rte_mbuf (mb2);
-      b3 = vlib_buffer_from_rte_mbuf (mb3);
-
-      bi0 = vlib_get_buffer_index (vm, b0);
-      bi1 = vlib_get_buffer_index (vm, b1);
-      bi2 = vlib_get_buffer_index (vm, b2);
-      bi3 = vlib_get_buffer_index (vm, b3);
-
-      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
-      vec_add1_aligned (fl->buffers, bi1, CLIB_CACHE_LINE_BYTES);
-      vec_add1_aligned (fl->buffers, bi2, CLIB_CACHE_LINE_BYTES);
-      vec_add1_aligned (fl->buffers, bi3, CLIB_CACHE_LINE_BYTES);
-
-      vlib_buffer_init_for_free_list (b0, fl);
-      vlib_buffer_init_for_free_list (b1, fl);
-      vlib_buffer_init_for_free_list (b2, fl);
-      vlib_buffer_init_for_free_list (b3, fl);
-
-      if (fl->buffer_init_function)
-	{
-	  fl->buffer_init_function (vm, fl, &bi0, 1);
-	  fl->buffer_init_function (vm, fl, &bi1, 1);
-	  fl->buffer_init_function (vm, fl, &bi2, 1);
-	  fl->buffer_init_function (vm, fl, &bi3, 1);
-	}
-      i += 4;
-    }
-
-  while (i < n)
-    {
-      mb0 = vm->mbuf_alloc_list[i];
-
-      ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
-      rte_mbuf_refcnt_set (mb0, 1);
-
-      b0 = vlib_buffer_from_rte_mbuf (mb0);
-      bi0 = vlib_get_buffer_index (vm, b0);
-
-      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
-
-      vlib_buffer_init_for_free_list (b0, fl);
-
-      if (fl->buffer_init_function)
-	fl->buffer_init_function (vm, fl, &bi0, 1);
-      i++;
-    }
-
-  fl->n_alloc += n;
-
-  return n;
-}
-
-static u32
-alloc_from_free_list (vlib_main_t * vm,
-		      vlib_buffer_free_list_t * free_list,
-		      u32 * alloc_buffers, u32 n_alloc_buffers)
-{
-  u32 *dst, *src;
-  uword len, n_filled;
-
-  dst = alloc_buffers;
-
-  n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
-  if (n_filled == 0)
-    return 0;
-
-  len = vec_len (free_list->buffers);
-  ASSERT (len >= n_alloc_buffers);
-
-  src = free_list->buffers + len - n_alloc_buffers;
-  clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
-  _vec_len (free_list->buffers) -= n_alloc_buffers;
-
-  return n_alloc_buffers;
-}
-
-/* Allocate a given number of buffers into given array.
-   Returns number actually allocated which will be either zero or
-   number requested. */
-u32
-dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  return alloc_from_free_list
-    (vm,
-     pool_elt_at_index (bm->buffer_free_list_pool,
-			VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
-     buffers, n_buffers);
-}
-
-
-u32
-dpdk_buffer_alloc_from_free_list (vlib_main_t * vm,
-				  u32 * buffers,
-				  u32 n_buffers, u32 free_list_index)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  vlib_buffer_free_list_t *f;
-  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
-  return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
-static_always_inline void
-vlib_buffer_free_inline (vlib_main_t * vm,
-			 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  vlib_buffer_free_list_t *fl;
-  u32 fi;
-  int i;
-  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
-	     u32 follow_buffer_next);
-
-  cb = bm->buffer_free_callback;
-
-  if (PREDICT_FALSE (cb != 0))
-    n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
-
-  if (!n_buffers)
-    return;
-
-  for (i = 0; i < n_buffers; i++)
-    {
-      vlib_buffer_t *b;
-
-      b = vlib_get_buffer (vm, buffers[i]);
-
-      fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
-
-      /* The only current use of this callback: multicast recycle */
-      if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
-	{
-	  int j;
-
-	  vlib_buffer_add_to_free_list
-	    (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-
-	  for (j = 0; j < vec_len (bm->announce_list); j++)
-	    {
-	      if (fl == bm->announce_list[j])
-		goto already_announced;
-	    }
-	  vec_add1 (bm->announce_list, fl);
-	already_announced:
-	  ;
-	}
-      else
-	{
-	  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
-	    dpdk_rte_pktmbuf_free (vm, b);
-	}
-    }
-  if (vec_len (bm->announce_list))
-    {
-      vlib_buffer_free_list_t *fl;
-      for (i = 0; i < vec_len (bm->announce_list); i++)
-	{
-	  fl = bm->announce_list[i];
-	  fl->buffers_added_to_freelist_function (vm, fl);
-	}
-      _vec_len (bm->announce_list) = 0;
-    }
-}
-
-static void
-dpdk_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
-  vlib_buffer_free_inline (vm, buffers, n_buffers,	/* follow_buffer_next */
-			   1);
-}
-
-static void
-dpdk_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
-  vlib_buffer_free_inline (vm, buffers, n_buffers,	/* follow_buffer_next */
-			   0);
-}
-
-static void
-dpdk_packet_template_init (vlib_main_t * vm,
-			   void *vt,
-			   void *packet_data,
-			   uword n_packet_data_bytes,
-			   uword min_n_buffers_each_physmem_alloc, u8 * name)
-{
-  vlib_packet_template_t *t = (vlib_packet_template_t *) vt;
-
-  vlib_worker_thread_barrier_sync (vm);
-  memset (t, 0, sizeof (t[0]));
-
-  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
-
-  vlib_worker_thread_barrier_release (vm);
-}
-
-clib_error_t *
-vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
-			 unsigned socket_id)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  vlib_physmem_main_t *vpm = &vm->physmem_main;
-  struct rte_mempool *rmp;
-  int i;
-
-  vec_validate_aligned (dm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
-
-  /* pool already exists, nothing to do */
-  if (dm->pktmbuf_pools[socket_id])
-    return 0;
-
-  u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0);
-
-  rmp = rte_pktmbuf_pool_create ((char *) pool_name,	/* pool name */
-				 num_mbufs,	/* number of mbufs */
-				 512,	/* cache size */
-				 VLIB_BUFFER_HDR_SIZE,	/* priv size */
-				 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE,	/* dataroom size */
-				 socket_id);	/* cpu socket */
-
-  if (rmp)
-    {
-      {
-	uword this_pool_end;
-	uword this_pool_start;
-	uword this_pool_size;
-	uword save_vpm_start, save_vpm_end, save_vpm_size;
-	struct rte_mempool_memhdr *memhdr;
-
-	this_pool_start = ~0ULL;
-	this_pool_end = 0LL;
-
-	STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
-	{
-	  if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
-	    this_pool_end = (uword) (memhdr->addr + memhdr->len);
-	  if (((uword) memhdr->addr) < this_pool_start)
-	    this_pool_start = (uword) (memhdr->addr);
-	}
-	ASSERT (this_pool_start < ~0ULL && this_pool_end > 0);
-	this_pool_size = this_pool_end - this_pool_start;
-
-	if (CLIB_DEBUG > 1)
-	  {
-	    clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
-			  pool_name, this_pool_start, this_pool_end,
-			  this_pool_size);
-	    clib_warning
-	      ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
-	       vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
-	  }
-
-	save_vpm_start = vpm->virtual.start;
-	save_vpm_end = vpm->virtual.end;
-	save_vpm_size = vpm->virtual.size;
-
-	if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
-	  vpm->virtual.start = this_pool_start;
-	if (this_pool_end > vpm->virtual.end)
-	  vpm->virtual.end = this_pool_end;
-
-	vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
-
-	if (CLIB_DEBUG > 1)
-	  {
-	    clib_warning
-	      ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
-	       vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
-	  }
-
-	/* check if fits into buffer index range */
-	if ((u64) vpm->virtual.size >
-	    ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
-	  {
-	    clib_warning ("physmem: virtual size out of range!");
-	    vpm->virtual.start = save_vpm_start;
-	    vpm->virtual.end = save_vpm_end;
-	    vpm->virtual.size = save_vpm_size;
-	    rmp = 0;
-	  }
-      }
-      if (rmp)
-	{
-	  dm->pktmbuf_pools[socket_id] = rmp;
-	  vec_free (pool_name);
-	  return 0;
-	}
-    }
-
-  vec_free (pool_name);
-
-  /* no usable pool for this socket, try to use pool from another one */
-  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
-    {
-      if (dm->pktmbuf_pools[i])
-	{
-	  clib_warning
-	    ("WARNING: Failed to allocate mempool for CPU socket %u. "
-	     "Threads running on socket %u will use socket %u mempool.",
-	     socket_id, socket_id, i);
-	  dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i];
-	  return 0;
-	}
-    }
-
-  return clib_error_return (0, "failed to allocate mempool on socket %u",
-			    socket_id);
-}
-
-#if CLIB_DEBUG > 0
-
-u32 *vlib_buffer_state_validation_lock;
-uword *vlib_buffer_state_validation_hash;
-void *vlib_buffer_state_heap;
-
-static clib_error_t *
-buffer_state_validation_init (vlib_main_t * vm)
-{
-  void *oldheap;
-
-  vlib_buffer_state_heap = mheap_alloc (0, 10 << 20);
-
-  oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
-
-  vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword));
-  vec_validate_aligned (vlib_buffer_state_validation_lock, 0,
-			CLIB_CACHE_LINE_BYTES);
-  clib_mem_set_heap (oldheap);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (buffer_state_validation_init);
-#endif
-
-static vlib_buffer_callbacks_t callbacks = {
-  .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
-  .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
-  .vlib_buffer_free_cb = &dpdk_buffer_free,
-  .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
-  .vlib_packet_template_init_cb = &dpdk_packet_template_init,
-  .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
-};
-
-static clib_error_t *
-dpdk_buffer_init (vlib_main_t * vm)
-{
-  vlib_buffer_cb_register (vm, &callbacks);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_buffer_init);
-
-/** @endcond */
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/cli.c b/src/vnet/devices/dpdk/cli.c
deleted file mode 100644
index 99998862..00000000
--- a/src/vnet/devices/dpdk/cli.c
+++ /dev/null
@@ -1,2079 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-
-#include "dpdk_priv.h"
-
-/**
- * @file
- * @brief CLI for DPDK Abstraction Layer and pcap Tx Trace.
- *
- * This file contains the source code for CLI for DPDK
- * Abstraction Layer and pcap Tx Trace.
- */
-
-
-static clib_error_t *
-get_hqos (u32 hw_if_index, u32 subport_id, dpdk_device_t ** xd,
-	  dpdk_device_config_t ** devconf)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  vnet_hw_interface_t *hw;
-  struct rte_eth_dev_info dev_info;
-  uword *p = 0;
-  clib_error_t *error = NULL;
-
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify valid interface name");
-      goto done;
-    }
-
-  if (subport_id != 0)
-    {
-      error = clib_error_return (0, "Invalid subport");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  *xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rte_eth_dev_info_get ((*xd)->device_index, &dev_info);
-  if (dev_info.pci_dev)
-    {				/* bonded interface has no pci info */
-      vlib_pci_addr_t pci_addr;
-
-      pci_addr.domain = dev_info.pci_dev->addr.domain;
-      pci_addr.bus = dev_info.pci_dev->addr.bus;
-      pci_addr.slot = dev_info.pci_dev->addr.devid;
-      pci_addr.function = dev_info.pci_dev->addr.function;
-
-      p =
-	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
-    }
-
-  if (p)
-    (*devconf) = pool_elt_at_index (dm->conf->dev_confs, p[0]);
-  else
-    (*devconf) = &dm->conf->default_devconf;
-
-done:
-  return error;
-}
-
-static clib_error_t *
-pcap_trace_command_fn (vlib_main_t * vm,
-		       unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-#define PCAP_DEF_PKT_TO_CAPTURE (100)
-
-  unformat_input_t _line_input, *line_input = &_line_input;
-  dpdk_main_t *dm = &dpdk_main;
-  u8 *filename;
-  u8 *chroot_filename = 0;
-  u32 max = 0;
-  int enabled = 0;
-  int errorFlag = 0;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "on"))
-	{
-	  if (dm->tx_pcap_enable == 0)
-	    {
-	      enabled = 1;
-	    }
-	  else
-	    {
-	      vlib_cli_output (vm, "pcap tx capture already on...");
-	      errorFlag = 1;
-	      break;
-	    }
-	}
-      else if (unformat (line_input, "off"))
-	{
-	  if (dm->tx_pcap_enable)
-	    {
-	      vlib_cli_output (vm, "captured %d pkts...",
-			       dm->pcap_main.n_packets_captured + 1);
-	      if (dm->pcap_main.n_packets_captured)
-		{
-		  dm->pcap_main.n_packets_to_capture =
-		    dm->pcap_main.n_packets_captured;
-		  error = pcap_write (&dm->pcap_main);
-		  if (error)
-		    clib_error_report (error);
-		  else
-		    vlib_cli_output (vm, "saved to %s...", dm->pcap_filename);
-		}
-
-	      dm->tx_pcap_enable = 0;
-	    }
-	  else
-	    {
-	      vlib_cli_output (vm, "pcap tx capture already off...");
-	      errorFlag = 1;
-	      break;
-	    }
-	}
-      else if (unformat (line_input, "max %d", &max))
-	{
-	  if (dm->tx_pcap_enable)
-	    {
-	      vlib_cli_output (vm,
-			       "can't change max value while pcap tx capture active...");
-	      errorFlag = 1;
-	      break;
-	    }
-	}
-      else if (unformat (line_input, "intfc %U",
-			 unformat_vnet_sw_interface, dm->vnet_main,
-			 &dm->pcap_sw_if_index))
-	;
-
-      else if (unformat (line_input, "intfc any"))
-	{
-	  dm->pcap_sw_if_index = 0;
-	}
-      else if (unformat (line_input, "file %s", &filename))
-	{
-	  if (dm->tx_pcap_enable)
-	    {
-	      vlib_cli_output (vm,
-			       "can't change file while pcap tx capture active...");
-	      errorFlag = 1;
-	      break;
-	    }
-
-	  /* Brain-police user path input */
-	  if (strstr ((char *) filename, "..")
-	      || index ((char *) filename, '/'))
-	    {
-	      vlib_cli_output (vm, "illegal characters in filename '%s'",
-			       filename);
-	      vlib_cli_output (vm,
-			       "Hint: Only filename, do not enter directory structure.");
-	      vec_free (filename);
-	      errorFlag = 1;
-	      break;
-	    }
-
-	  chroot_filename = format (0, "/tmp/%s%c", filename, 0);
-	  vec_free (filename);
-	}
-      else if (unformat (line_input, "status"))
-	{
-	  if (dm->pcap_sw_if_index == 0)
-	    {
-	      vlib_cli_output (vm, "max is %d for any interface to file %s",
-			       dm->
-			       pcap_pkts_to_capture ? dm->pcap_pkts_to_capture
-			       : PCAP_DEF_PKT_TO_CAPTURE,
-			       dm->
-			       pcap_filename ? dm->pcap_filename : (u8 *)
-			       "/tmp/vpe.pcap");
-	    }
-	  else
-	    {
-	      vlib_cli_output (vm, "max is %d for interface %U to file %s",
-			       dm->
-			       pcap_pkts_to_capture ? dm->pcap_pkts_to_capture
-			       : PCAP_DEF_PKT_TO_CAPTURE,
-			       format_vnet_sw_if_index_name, dm->vnet_main,
-			       dm->pcap_sw_if_index,
-			       dm->
-			       pcap_filename ? dm->pcap_filename : (u8 *)
-			       "/tmp/vpe.pcap");
-	    }
-
-	  if (dm->tx_pcap_enable == 0)
-	    {
-	      vlib_cli_output (vm, "pcap tx capture is off...");
-	    }
-	  else
-	    {
-	      vlib_cli_output (vm, "pcap tx capture is on: %d of %d pkts...",
-			       dm->pcap_main.n_packets_captured,
-			       dm->pcap_main.n_packets_to_capture);
-	    }
-	  break;
-	}
-
-      else
-	{
-	  error = clib_error_return (0, "unknown input `%U'",
-				     format_unformat_error, line_input);
-	  errorFlag = 1;
-	  break;
-	}
-    }
-  unformat_free (line_input);
-
-
-  if (errorFlag == 0)
-    {
-      /* Since no error, save configured values. */
-      if (chroot_filename)
-	{
-	  if (dm->pcap_filename)
-	    vec_free (dm->pcap_filename);
-	  vec_add1 (chroot_filename, 0);
-	  dm->pcap_filename = chroot_filename;
-	}
-
-      if (max)
-	dm->pcap_pkts_to_capture = max;
-
-
-      if (enabled)
-	{
-	  if (dm->pcap_filename == 0)
-	    dm->pcap_filename = format (0, "/tmp/vpe.pcap%c", 0);
-
-	  memset (&dm->pcap_main, 0, sizeof (dm->pcap_main));
-	  dm->pcap_main.file_name = (char *) dm->pcap_filename;
-	  dm->pcap_main.n_packets_to_capture = PCAP_DEF_PKT_TO_CAPTURE;
-	  if (dm->pcap_pkts_to_capture)
-	    dm->pcap_main.n_packets_to_capture = dm->pcap_pkts_to_capture;
-
-	  dm->pcap_main.packet_type = PCAP_PACKET_TYPE_ethernet;
-	  dm->tx_pcap_enable = 1;
-	  vlib_cli_output (vm, "pcap tx capture on...");
-	}
-    }
-  else if (chroot_filename)
-    vec_free (chroot_filename);
-
-
-  return error;
-}
-
-/*?
- * This command is used to start or stop a packet capture, or show
- * the status of packet capture.
- *
- * This command has the following optional parameters:
- *
- * - on|off - Used to start or stop a packet capture.
- *
- * - max  - Depth of local buffer. Once 'nn' number
- *   of packets have been received, buffer is flushed to file. Once another
- *   'nn' number of packets have been received, buffer is flushed
- *   to file, overwriting previous write. If not entered, value defaults
- *   to 100. Can only be updated if packet capture is off.
- *
- * - intfc |any - Used to specify a given interface,
- *   or use 'any' to run packet capture on all interfaces.
- *   'any' is the default if not provided. Settings from a previous
- *   packet capture are preserved, so 'any' can be used to reset
- *   the interface setting.
- *
- * - file  - Used to specify the output filename. The file will
- *   be placed in the '/tmp' directory, so only the filename is
- *   supported. Directory should not be entered. If file already exists, file
- *   will be overwritten. If no filename is provided, '/tmp/vpe.pcap'
- *   will be used. Can only be updated if packet capture is off.
- *
- * - status - Displays the current status and configured attributes
- *   associated with a packet capture. If packet capture is in progress,
- *   'status' also will return the number of packets currently in
- *   the local buffer. All additional attributes entered on command line
- *   with 'status' will be ingnored and not applied.
- *
- * @cliexpar
- * Example of how to display the status of a tx packet capture when off:
- * @cliexstart{pcap tx trace status}
- * max is 100, for any interface to file /tmp/vpe.pcap
- * pcap tx capture is off...
- * @cliexend
- * Example of how to start a tx packet capture:
- * @cliexstart{pcap tx trace on max 35 intfc GigabitEthernet0/8/0 file vppTest.pcap}
- * pcap tx capture on...
- * @cliexend
- * Example of how to display the status of a tx packet capture in progress:
- * @cliexstart{pcap tx trace status}
- * max is 35, for interface GigabitEthernet0/8/0 to file /tmp/vppTest.pcap
- * pcap tx capture is on: 20 of 35 pkts...
- * @cliexend
- * Example of how to stop a tx packet capture:
- * @cliexstart{vppctl pcap tx trace off}
- * captured 21 pkts...
- * saved to /tmp/vppTest.pcap...
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (pcap_trace_command, static) = {
-    .path = "pcap tx trace",
-    .short_help =
-    "pcap tx trace [on|off] [max ] [intfc |any] [file ] [status]",
-    .function = pcap_trace_command_fn,
-};
-/* *INDENT-ON* */
-
-
-static clib_error_t *
-show_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input,
-		  vlib_cli_command_t * cmd)
-{
-  struct rte_mempool *rmp;
-  int i;
-
-  for (i = 0; i < vec_len (dpdk_main.pktmbuf_pools); i++)
-    {
-      rmp = dpdk_main.pktmbuf_pools[i];
-      if (rmp)
-	{
-	  unsigned count = rte_mempool_avail_count (rmp);
-	  unsigned free_count = rte_mempool_in_use_count (rmp);
-
-	  vlib_cli_output (vm,
-			   "name=\"%s\"  available = %7d allocated = %7d total = %7d\n",
-			   rmp->name, (u32) count, (u32) free_count,
-			   (u32) (count + free_count));
-	}
-      else
-	{
-	  vlib_cli_output (vm, "rte_mempool is NULL (!)\n");
-	}
-    }
-  return 0;
-}
-
-/*?
- * This command displays statistics of each DPDK mempool.
- *
- * @cliexpar
- * Example of how to display DPDK buffer data:
- * @cliexstart{show dpdk buffer}
- * name="mbuf_pool_socket0"  available =   15104 allocated =    1280 total =   16384
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_show_dpdk_bufferr,static) = {
-    .path = "show dpdk buffer",
-    .short_help = "show dpdk buffer",
-    .function = show_dpdk_buffer,
-    .is_mp_safe = 1,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-test_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input,
-		  vlib_cli_command_t * cmd)
-{
-  static u32 *allocated_buffers;
-  u32 n_alloc = 0;
-  u32 n_free = 0;
-  u32 first, actual_alloc;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "allocate %d", &n_alloc))
-	;
-      else if (unformat (input, "free %d", &n_free))
-	;
-      else
-	break;
-    }
-
-  if (n_free)
-    {
-      if (vec_len (allocated_buffers) < n_free)
-	return clib_error_return (0, "Can't free %d, only %d allocated",
-				  n_free, vec_len (allocated_buffers));
-
-      first = vec_len (allocated_buffers) - n_free;
-      vlib_buffer_free (vm, allocated_buffers + first, n_free);
-      _vec_len (allocated_buffers) = first;
-    }
-  if (n_alloc)
-    {
-      first = vec_len (allocated_buffers);
-      vec_validate (allocated_buffers,
-		    vec_len (allocated_buffers) + n_alloc - 1);
-
-      actual_alloc = vlib_buffer_alloc (vm, allocated_buffers + first,
-					n_alloc);
-      _vec_len (allocated_buffers) = first + actual_alloc;
-
-      if (actual_alloc < n_alloc)
-	vlib_cli_output (vm, "WARNING: only allocated %d buffers",
-			 actual_alloc);
-    }
-
-  vlib_cli_output (vm, "Currently %d buffers allocated",
-		   vec_len (allocated_buffers));
-
-  if (allocated_buffers && vec_len (allocated_buffers) == 0)
-    vec_free (allocated_buffers);
-
-  return 0;
-}
-
-/*?
- * This command tests the allocation and freeing of DPDK buffers.
- * If both 'allocate' and 'free' are entered on the
- * same command, the 'free' is executed first. If no
- * parameters are provided, this command display how many DPDK buffers
- * the test command has allocated.
- *
- * @cliexpar
- * @parblock
- *
- * Example of how to display how many DPDK buffer test command has allcoated:
- * @cliexstart{test dpdk buffer}
- * Currently 0 buffers allocated
- * @cliexend
- *
- * Example of how to allocate DPDK buffers using the test command:
- * @cliexstart{test dpdk buffer allocate 10}
- * Currently 10 buffers allocated
- * @cliexend
- *
- * Example of how to free DPDK buffers allocated by the test command:
- * @cliexstart{test dpdk buffer free 10}
- * Currently 0 buffers allocated
- * @cliexend
- * @endparblock
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_test_dpdk_buffer,static) = {
-    .path = "test dpdk buffer",
-    .short_help = "test dpdk buffer [allocate ] [free ]",
-    .function = test_dpdk_buffer,
-    .is_mp_safe = 1,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input,
-		  vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  dpdk_main_t *dm = &dpdk_main;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 nb_rx_desc = (u32) ~ 0;
-  u32 nb_tx_desc = (u32) ~ 0;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "tx %d", &nb_tx_desc))
-	;
-      else if (unformat (line_input, "rx %d", &nb_rx_desc))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify valid interface name");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
-    {
-      error =
-	clib_error_return (0,
-			   "number of descriptors can be set only for "
-			   "physical devices");
-      goto done;
-    }
-
-  if ((nb_rx_desc == (u32) ~ 0 || nb_rx_desc == xd->nb_rx_desc) &&
-      (nb_tx_desc == (u32) ~ 0 || nb_tx_desc == xd->nb_tx_desc))
-    {
-      error = clib_error_return (0, "nothing changed");
-      goto done;
-    }
-
-  if (nb_rx_desc != (u32) ~ 0)
-    xd->nb_rx_desc = nb_rx_desc;
-
-  if (nb_tx_desc != (u32) ~ 0)
-    xd->nb_tx_desc = nb_tx_desc;
-
-  error = dpdk_port_setup (dm, xd);
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command sets the number of DPDK 'rx' and
- * 'tx' descriptors for the given physical interface. Use
- * the command 'show hardware-interface' to display the
- * current descriptor allocation.
- *
- * @cliexpar
- * Example of how to set the DPDK interface descriptors:
- * @cliexcmd{set dpdk interface descriptors GigabitEthernet0/8/0 rx 512 tx 512}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_desc,static) = {
-    .path = "set dpdk interface descriptors",
-    .short_help = "set dpdk interface descriptors  [rx ] [tx ]",
-    .function = set_dpdk_if_desc,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-show_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
-			vlib_cli_command_t * cmd)
-{
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_and_queue_t *dq;
-  int cpu;
-
-  if (tm->n_vlib_mains == 1)
-    vlib_cli_output (vm, "All interfaces are handled by main thread");
-
-  for (cpu = 0; cpu < vec_len (dm->devices_by_cpu); cpu++)
-    {
-      if (cpu >= dm->input_cpu_first_index &&
-	  cpu < (dm->input_cpu_first_index + dm->input_cpu_count))
-	vlib_cli_output (vm, "Thread %u (%s at lcore %u):", cpu,
-			 vlib_worker_threads[cpu].name,
-			 vlib_worker_threads[cpu].lcore_id);
-
-      /* *INDENT-OFF* */
-      vec_foreach(dq, dm->devices_by_cpu[cpu])
-        {
-          u32 hw_if_index = dm->devices[dq->device].vlib_hw_if_index;
-          vnet_hw_interface_t * hi =  vnet_get_hw_interface(dm->vnet_main, hw_if_index);
-          vlib_cli_output(vm, "  %v queue %u", hi->name, dq->queue_id);
-        }
-      /* *INDENT-ON* */
-    }
-  return 0;
-}
-
-/*?
- * This command is used to display the thread and core each
- * DPDK interface and queue is assigned too.
- *
- * @cliexpar
- * Example of how to display the DPDK interface placement:
- * @cliexstart{show dpdk interface placement}
- * Thread 1 (vpp_wk_0 at lcore 1):
- *   GigabitEthernet0/8/0 queue 0
- *   GigabitEthernet0/9/0 queue 0
- * Thread 2 (vpp_wk_1 at lcore 2):
- *   GigabitEthernet0/8/0 queue 1
- *   GigabitEthernet0/9/0 queue 1
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_show_dpdk_if_placement,static) = {
-    .path = "show dpdk interface placement",
-    .short_help = "show dpdk interface placement",
-    .function = show_dpdk_if_placement,
-};
-/* *INDENT-ON* */
-
-static int
-dpdk_device_queue_sort (void *a1, void *a2)
-{
-  dpdk_device_and_queue_t *dq1 = a1;
-  dpdk_device_and_queue_t *dq2 = a2;
-
-  if (dq1->device > dq2->device)
-    return 1;
-  else if (dq1->device < dq2->device)
-    return -1;
-  else if (dq1->queue_id > dq2->queue_id)
-    return 1;
-  else if (dq1->queue_id < dq2->queue_id)
-    return -1;
-  else
-    return 0;
-}
-
-static clib_error_t *
-set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
-		       vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_and_queue_t *dq;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 queue = (u32) 0;
-  u32 cpu = (u32) ~ 0;
-  int i;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "queue %d", &queue))
-	;
-      else if (unformat (line_input, "thread %d", &cpu))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify valid interface name");
-      goto done;
-    }
-
-  if (cpu < dm->input_cpu_first_index ||
-      cpu >= (dm->input_cpu_first_index + dm->input_cpu_count))
-    {
-      error = clib_error_return (0, "please specify valid thread id");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  for (i = 0; i < vec_len (dm->devices_by_cpu); i++)
-    {
-      /* *INDENT-OFF* */
-      vec_foreach(dq, dm->devices_by_cpu[i])
-        {
-          if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index &&
-              queue == dq->queue_id)
-            {
-              if (cpu == i) /* nothing to do */
-                goto done;
-
-              vec_del1(dm->devices_by_cpu[i], dq - dm->devices_by_cpu[i]);
-              vec_add2(dm->devices_by_cpu[cpu], dq, 1);
-              dq->queue_id = queue;
-              dq->device = xd->device_index;
-              xd->cpu_socket_id_by_queue[queue] =
-                rte_lcore_to_socket_id(vlib_worker_threads[cpu].lcore_id);
-
-              vec_sort_with_function(dm->devices_by_cpu[i],
-		                     dpdk_device_queue_sort);
-
-              vec_sort_with_function(dm->devices_by_cpu[cpu],
-		                     dpdk_device_queue_sort);
-
-              if (vec_len(dm->devices_by_cpu[i]) == 0)
-                vlib_node_set_state (vlib_mains[i], dpdk_input_node.index,
-		                     VLIB_NODE_STATE_DISABLED);
-
-              if (vec_len(dm->devices_by_cpu[cpu]) == 1)
-                vlib_node_set_state (vlib_mains[cpu], dpdk_input_node.index,
-		                     VLIB_NODE_STATE_POLLING);
-
-              goto done;
-            }
-        }
-      /* *INDENT-ON* */
-    }
-
-  error = clib_error_return (0, "not found");
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to assign a given interface, and optionally a
- * given queue, to a different thread. This will not create a thread,
- * so the thread must already exist. Use '/etc/vpp/startup.conf'
- * for the initial thread creation. If the 'queue' is not provided,
- * it defaults to 0.
- *
- * @cliexpar
- * Example of how to display the DPDK interface placement:
- * @cliexstart{show dpdk interface placement}
- * Thread 1 (vpp_wk_0 at lcore 1):
- *   GigabitEthernet0/8/0 queue 0
- *   GigabitEthernet0/9/0 queue 0
- * Thread 2 (vpp_wk_1 at lcore 2):
- *   GigabitEthernet0/8/0 queue 1
- *   GigabitEthernet0/9/0 queue 1
- * @cliexend
- * Example of how to assign a DPDK interface and queue to a thread:
- * @cliexcmd{set dpdk interface placement GigabitEthernet0/8/0 queue 1 thread 1}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_placement,static) = {
-    .path = "set dpdk interface placement",
-    .short_help = "set dpdk interface placement  [queue ] thread ",
-    .function = set_dpdk_if_placement,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-show_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
-			     vlib_cli_command_t * cmd)
-{
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_and_queue_t *dq;
-  int cpu;
-
-  if (tm->n_vlib_mains == 1)
-    vlib_cli_output (vm, "All interfaces are handled by main thread");
-
-  for (cpu = 0; cpu < vec_len (dm->devices_by_hqos_cpu); cpu++)
-    {
-      if (cpu >= dm->hqos_cpu_first_index &&
-	  cpu < (dm->hqos_cpu_first_index + dm->hqos_cpu_count))
-	vlib_cli_output (vm, "Thread %u (%s at lcore %u):", cpu,
-			 vlib_worker_threads[cpu].name,
-			 vlib_worker_threads[cpu].lcore_id);
-
-      vec_foreach (dq, dm->devices_by_hqos_cpu[cpu])
-      {
-	u32 hw_if_index = dm->devices[dq->device].vlib_hw_if_index;
-	vnet_hw_interface_t *hi =
-	  vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-	vlib_cli_output (vm, "  %v queue %u", hi->name, dq->queue_id);
-      }
-    }
-  return 0;
-}
-
-/*?
- * This command is used to display the thread and core each
- * DPDK output interface and HQoS queue is assigned too.
- *
- * @cliexpar
- * Example of how to display the DPDK output interface and HQoS queue placement:
- * @cliexstart{show dpdk interface hqos placement}
- * Thread 1 (vpp_hqos-threads_0 at lcore 3):
- *   GigabitEthernet0/8/0 queue 0
- * Thread 2 (vpp_hqos-threads_1 at lcore 4):
- *   GigabitEthernet0/9/0 queue 0
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_show_dpdk_if_hqos_placement, static) = {
-  .path = "show dpdk interface hqos placement",
-  .short_help = "show dpdk interface hqos placement",
-  .function = show_dpdk_if_hqos_placement,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
-			    vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_and_queue_t *dq;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 cpu = (u32) ~ 0;
-  int i;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "thread %d", &cpu))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
-
-  if (cpu < dm->hqos_cpu_first_index ||
-      cpu >= (dm->hqos_cpu_first_index + dm->hqos_cpu_count))
-    {
-      error = clib_error_return (0, "please specify valid thread id");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  for (i = 0; i < vec_len (dm->devices_by_hqos_cpu); i++)
-    {
-      vec_foreach (dq, dm->devices_by_hqos_cpu[i])
-      {
-	if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index)
-	  {
-	    if (cpu == i)	/* nothing to do */
-	      goto done;
-
-	    vec_del1 (dm->devices_by_hqos_cpu[i],
-		      dq - dm->devices_by_hqos_cpu[i]);
-	    vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
-	    dq->queue_id = 0;
-	    dq->device = xd->device_index;
-
-	    vec_sort_with_function (dm->devices_by_hqos_cpu[i],
-				    dpdk_device_queue_sort);
-
-	    vec_sort_with_function (dm->devices_by_hqos_cpu[cpu],
-				    dpdk_device_queue_sort);
-
-	    goto done;
-	  }
-      }
-    }
-
-  error = clib_error_return (0, "not found");
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to assign a given DPDK output interface and
- * HQoS queue to a different thread. This will not create a thread,
- * so the thread must already exist. Use '/etc/vpp/startup.conf'
- * for the initial thread creation. See @ref qos_doc for more details.
- *
- * @cliexpar
- * Example of how to display the DPDK output interface and HQoS queue placement:
- * @cliexstart{show dpdk interface hqos placement}
- * Thread 1 (vpp_hqos-threads_0 at lcore 3):
- *   GigabitEthernet0/8/0 queue 0
- * Thread 2 (vpp_hqos-threads_1 at lcore 4):
- *   GigabitEthernet0/9/0 queue 0
- * @cliexend
- * Example of how to assign a DPDK output interface and HQoS queue to a thread:
- * @cliexcmd{set dpdk interface hqos placement GigabitEthernet0/8/0 thread 2}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_placement, static) = {
-  .path = "set dpdk interface hqos placement",
-  .short_help = "set dpdk interface hqos placement  thread ",
-  .function = set_dpdk_if_hqos_placement,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input,
-		       vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  dpdk_main_t *dm = &dpdk_main;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 subport_id = (u32) ~ 0;
-  u32 pipe_id = (u32) ~ 0;
-  u32 profile_id = (u32) ~ 0;
-  int rv;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "subport %d", &subport_id))
-	;
-      else if (unformat (line_input, "pipe %d", &pipe_id))
-	;
-      else if (unformat (line_input, "profile %d", &profile_id))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify valid interface name");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rv =
-    rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
-			   profile_id);
-  if (rv)
-    {
-      error = clib_error_return (0, "pipe configuration failed");
-      goto done;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to change the profile associate with a HQoS pipe. The
- * '' is zero based. Use the command
- * 'show dpdk interface hqos' to display the content of each profile.
- * See @ref qos_doc for more details.
- *
- * @note
- * Currently there is not an API to create a new HQoS pipe profile. One is
- * created by default in the code (search for 'hqos_pipe_params_default'').
- * Additional profiles can be created in code and code recompiled. Then use this
- * command to assign it.
- *
- * @cliexpar
- * Example of how to assign a new profile to a HQoS pipe:
- * @cliexcmd{set dpdk interface hqos pipe GigabitEthernet0/8/0 subport 0 pipe 2 profile 1}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_pipe, static) =
-{
-  .path = "set dpdk interface hqos pipe",
-  .short_help = "set dpdk interface hqos pipe  subport  pipe  "
-                  "profile ",
-  .function = set_dpdk_if_hqos_pipe,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input,
-			  vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = NULL;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 subport_id = (u32) ~ 0;
-  struct rte_sched_subport_params p;
-  int rv;
-  clib_error_t *error = NULL;
-  u32 tb_rate = (u32) ~ 0;
-  u32 tb_size = (u32) ~ 0;
-  u32 tc_rate[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] =
-    { (u32) ~ 0, (u32) ~ 0, (u32) ~ 0, (u32) ~ 0 };
-  u32 tc_period = (u32) ~ 0;
-  dpdk_device_config_t *devconf = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "subport %d", &subport_id))
-	;
-      else if (unformat (line_input, "rate %d", &tb_rate))
-	;
-      else if (unformat (line_input, "bktsize %d", &tb_size))
-	;
-      else if (unformat (line_input, "tc0 %d", &tc_rate[0]))
-	;
-      else if (unformat (line_input, "tc1 %d", &tc_rate[1]))
-	;
-      else if (unformat (line_input, "tc2 %d", &tc_rate[2]))
-	;
-      else if (unformat (line_input, "tc3 %d", &tc_rate[3]))
-	;
-      else if (unformat (line_input, "period %d", &tc_period))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  error = get_hqos (hw_if_index, subport_id, &xd, &devconf);
-
-  if (error == NULL)
-    {
-      /* Copy the current values over to local structure. */
-      memcpy (&p, &devconf->hqos.subport[subport_id], sizeof (p));
-
-      /* Update local structure with input values. */
-      if (tb_rate != (u32) ~ 0)
-	{
-	  p.tb_rate = tb_rate;
-	  p.tc_rate[0] = tb_rate;
-	  p.tc_rate[1] = tb_rate;
-	  p.tc_rate[2] = tb_rate;
-	  p.tc_rate[3] = tb_rate;
-	}
-      if (tb_size != (u32) ~ 0)
-	{
-	  p.tb_size = tb_size;
-	}
-      if (tc_rate[0] != (u32) ~ 0)
-	{
-	  p.tc_rate[0] = tc_rate[0];
-	}
-      if (tc_rate[1] != (u32) ~ 0)
-	{
-	  p.tc_rate[1] = tc_rate[1];
-	}
-      if (tc_rate[2] != (u32) ~ 0)
-	{
-	  p.tc_rate[2] = tc_rate[2];
-	}
-      if (tc_rate[3] != (u32) ~ 0)
-	{
-	  p.tc_rate[3] = tc_rate[3];
-	}
-      if (tc_period != (u32) ~ 0)
-	{
-	  p.tc_period = tc_period;
-	}
-
-      /* Apply changes. */
-      rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, &p);
-      if (rv)
-	{
-	  error = clib_error_return (0, "subport configuration failed");
-	  goto done;
-	}
-      else
-	{
-	  /* Successfully applied, so save of the input values. */
-	  memcpy (&devconf->hqos.subport[subport_id], &p, sizeof (p));
-	}
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to set the subport level parameters such as token
- * bucket rate (bytes per seconds), token bucket size (bytes), traffic class
- * rates (bytes per seconds) and token update period (Milliseconds).
- *
- * By default, the 'rate' is set to 1250000000 bytes/second (10GbE
- * rate) and each of the four traffic classes is set to 100% of the port rate.
- * If the 'rate' is updated by this command, all four traffic classes
- * are assigned the same value. Each of the four traffic classes can be updated
- * individually.
- *
- * @cliexpar
- * Example of how modify the subport attributes for a 1GbE link:
- * @cliexcmd{set dpdk interface hqos subport GigabitEthernet0/8/0 subport 0 rate 125000000}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_subport, static) = {
-  .path = "set dpdk interface hqos subport",
-  .short_help = "set dpdk interface hqos subport  subport  "
-                 "[rate ] [bktsize ] [tc0 ] [tc1 ] [tc2 ] [tc3 ] "
-                 "[period ]",
-  .function = set_dpdk_if_hqos_subport,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input,
-			vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_main_t *dm = &dpdk_main;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 tc = (u32) ~ 0;
-  u32 queue = (u32) ~ 0;
-  u32 entry = (u32) ~ 0;
-  u32 val, i;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "entry %d", &entry))
-	;
-      else if (unformat (line_input, "tc %d", &tc))
-	;
-      else if (unformat (line_input, "queue %d", &queue))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify valid interface name");
-      goto done;
-    }
-  if (entry >= 64)
-    {
-      error = clib_error_return (0, "invalid entry");
-      goto done;
-    }
-  if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
-    {
-      error = clib_error_return (0, "invalid traffic class");
-      goto done;
-    }
-  if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS)
-    {
-      error = clib_error_return (0, "invalid traffic class queue");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  /* Detect the set of worker threads */
-  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-  /* Should never happen, shut up Coverity warning */
-  if (p == 0)
-    {
-      error = clib_error_return (0, "no worker registrations?");
-      goto done;
-    }
-
-  vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
-  int worker_thread_first = tr->first_index;
-  int worker_thread_count = tr->count;
-
-  val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue;
-  for (i = 0; i < worker_thread_count; i++)
-    xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val;
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to set the traffic class translation table. The
- * traffic class translation table is used to map 64 values (0-63) to one of
- * four traffic class and one of four HQoS input queue. Use the 'show
- * dpdk interface hqos' command to display the traffic class translation
- * table. See @ref qos_doc for more details.
- *
- * This command has the following parameters:
- *
- * -  - Used to specify the output interface.
- *
- * - entry  - Mapped value (0-63) to assign traffic class and queue to.
- *
- * - tc  - Traffic class (0-3) to be used by the provided mapped value.
- *
- * - queue  - HQoS input queue (0-3) to be used by the provided mapped value.
- *
- * @cliexpar
- * Example of how modify the traffic class translation table:
- * @cliexcmd{set dpdk interface hqos tctbl GigabitEthernet0/8/0 entry 16 tc 2 queue 2}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_tctbl, static) = {
-  .path = "set dpdk interface hqos tctbl",
-  .short_help = "set dpdk interface hqos tctbl  entry  tc  queue ",
-  .function = set_dpdk_if_hqos_tctbl,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
-			   vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_main_t *dm = &dpdk_main;
-  clib_error_t *error = NULL;
-
-  /* Device specific data */
-  struct rte_eth_dev_info dev_info;
-  dpdk_device_config_t *devconf = 0;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  u32 hw_if_index = (u32) ~ 0;
-
-  /* Detect the set of worker threads */
-  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-  /* Should never happen, shut up Coverity warning */
-  if (p == 0)
-    return clib_error_return (0, "no worker registrations?");
-
-  vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
-  int worker_thread_first = tr->first_index;
-  int worker_thread_count = tr->count;
-
-  /* Packet field configuration */
-  u64 mask = (u64) ~ 0;
-  u32 id = (u32) ~ 0;
-  u32 offset = (u32) ~ 0;
-
-  /* HQoS params */
-  u32 n_subports_per_port, n_pipes_per_subport, tctbl_size;
-
-  u32 i;
-
-  /* Parse input arguments */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else if (unformat (line_input, "id subport"))
-	id = 0;
-      else if (unformat (line_input, "id pipe"))
-	id = 1;
-      else if (unformat (line_input, "id tc"))
-	id = 2;
-      else if (unformat (line_input, "id %d", &id))
-	;
-      else if (unformat (line_input, "offset %d", &offset))
-	;
-      else if (unformat (line_input, "mask %llx", &mask))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  /* Get interface */
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify valid interface name");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rte_eth_dev_info_get (xd->device_index, &dev_info);
-  if (dev_info.pci_dev)
-    {				/* bonded interface has no pci info */
-      vlib_pci_addr_t pci_addr;
-
-      pci_addr.domain = dev_info.pci_dev->addr.domain;
-      pci_addr.bus = dev_info.pci_dev->addr.bus;
-      pci_addr.slot = dev_info.pci_dev->addr.devid;
-      pci_addr.function = dev_info.pci_dev->addr.function;
-
-      p =
-	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
-    }
-
-  if (p)
-    devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
-  else
-    devconf = &dm->conf->default_devconf;
-
-  if (devconf->hqos_enabled == 0)
-    {
-      vlib_cli_output (vm, "HQoS disabled for this interface");
-      goto done;
-    }
-
-  n_subports_per_port = devconf->hqos.port.n_subports_per_port;
-  n_pipes_per_subport = devconf->hqos.port.n_pipes_per_subport;
-  tctbl_size = RTE_DIM (devconf->hqos.tc_table);
-
-  /* Validate packet field configuration: id, offset and mask */
-  if (id >= 3)
-    {
-      error = clib_error_return (0, "invalid packet field id");
-      goto done;
-    }
-
-  switch (id)
-    {
-    case 0:
-      if (dpdk_hqos_validate_mask (mask, n_subports_per_port) != 0)
-	{
-	  error = clib_error_return (0, "invalid subport ID mask "
-				     "(n_subports_per_port = %u)",
-				     n_subports_per_port);
-	  goto done;
-	}
-      break;
-    case 1:
-      if (dpdk_hqos_validate_mask (mask, n_pipes_per_subport) != 0)
-	{
-	  error = clib_error_return (0, "invalid pipe ID mask "
-				     "(n_pipes_per_subport = %u)",
-				     n_pipes_per_subport);
-	  goto done;
-	}
-      break;
-    case 2:
-    default:
-      if (dpdk_hqos_validate_mask (mask, tctbl_size) != 0)
-	{
-	  error = clib_error_return (0, "invalid TC table index mask "
-				     "(TC table size = %u)", tctbl_size);
-	  goto done;
-	}
-    }
-
-  /* Propagate packet field configuration to all workers */
-  for (i = 0; i < worker_thread_count; i++)
-    switch (id)
-      {
-      case 0:
-	xd->hqos_wt[worker_thread_first + i].hqos_field0_slabpos = offset;
-	xd->hqos_wt[worker_thread_first + i].hqos_field0_slabmask = mask;
-	xd->hqos_wt[worker_thread_first + i].hqos_field0_slabshr =
-	  __builtin_ctzll (mask);
-	break;
-      case 1:
-	xd->hqos_wt[worker_thread_first + i].hqos_field1_slabpos = offset;
-	xd->hqos_wt[worker_thread_first + i].hqos_field1_slabmask = mask;
-	xd->hqos_wt[worker_thread_first + i].hqos_field1_slabshr =
-	  __builtin_ctzll (mask);
-	break;
-      case 2:
-      default:
-	xd->hqos_wt[worker_thread_first + i].hqos_field2_slabpos = offset;
-	xd->hqos_wt[worker_thread_first + i].hqos_field2_slabmask = mask;
-	xd->hqos_wt[worker_thread_first + i].hqos_field2_slabshr =
-	  __builtin_ctzll (mask);
-      }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to set the packet fields required for classifiying the
- * incoming packet. As a result of classification process, packet field
- * information will be mapped to 5 tuples (subport, pipe, traffic class, pipe,
- * color) and stored in packet mbuf.
- *
- * This command has the following parameters:
- *
- * -  - Used to specify the output interface.
- *
- * - id subport|pipe|tc - Classification occurs across three fields.
- * This parameter indicates which of the three masks are being configured. Legacy
- * code used 0-2 to represent these three fields, so 0-2 is still accepted.
- *   - subport|0 - Currently only one subport is supported, so only
- * an empty mask is supported for the subport classification.
- *   - pipe|1 - Currently, 4096 pipes per subport are supported, so a
- * 12-bit mask should be configure to map to the 0-4095 pipes.
- *   - tc|2 - The translation table (see 'set dpdk interface hqos
- * tctbl' command) maps each value (0-63) into one of the 4 traffic classes
- * per pipe. A 6-bit mask should be configure to map this field to a traffic class.
- *
- * - offset  - Offset in the packet to apply the 64-bit mask for classification.
- * The offset should be on an 8-byte boundary (0,8,16,24..).
- *
- * - mask  - 64-bit mask to apply to packet at the given 'offset'.
- * Bits must be contiguous and should not include '0x'.
- *
- * The default values for the 'pktfield' assumes Ethernet/IPv4/UDP packets with
- * no VLAN. Adjust based on expected packet format and desired classification field.
- * - 'subport' is always empty (offset 0 mask 0000000000000000)
- * - By default, 'pipe' maps to the UDP payload bits 12 .. 23 (offset 40
- * mask 0000000fff000000)
- * - By default, 'tc' maps to the DSCP field in IP header (offset 48 mask
- * 00000000000000fc)
- *
- * @cliexpar
- * Example of how modify the 'pipe' classification filter to match VLAN:
- * @cliexcmd{set dpdk interface hqos pktfield GigabitEthernet0/8/0 id pipe offset 8 mask 0000000000000FFF}
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_pktfield, static) = {
-  .path = "set dpdk interface hqos pktfield",
-  .short_help = "set dpdk interface hqos pktfield  id subport|pipe|tc offset  "
-                 "mask ",
-  .function = set_dpdk_if_hqos_pktfield,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
-		   vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_main_t *dm = &dpdk_main;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  dpdk_device_config_hqos_t *cfg;
-  dpdk_device_hqos_per_hqos_thread_t *ht;
-  dpdk_device_hqos_per_worker_thread_t *wk;
-  u32 *tctbl;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 profile_id, subport_id, i;
-  struct rte_eth_dev_info dev_info;
-  dpdk_device_config_t *devconf = 0;
-  vlib_thread_registration_t *tr;
-  uword *p = 0;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify interface name!!");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rte_eth_dev_info_get (xd->device_index, &dev_info);
-  if (dev_info.pci_dev)
-    {				/* bonded interface has no pci info */
-      vlib_pci_addr_t pci_addr;
-
-      pci_addr.domain = dev_info.pci_dev->addr.domain;
-      pci_addr.bus = dev_info.pci_dev->addr.bus;
-      pci_addr.slot = dev_info.pci_dev->addr.devid;
-      pci_addr.function = dev_info.pci_dev->addr.function;
-
-      p =
-	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
-    }
-
-  if (p)
-    devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
-  else
-    devconf = &dm->conf->default_devconf;
-
-  if (devconf->hqos_enabled == 0)
-    {
-      vlib_cli_output (vm, "HQoS disabled for this interface");
-      goto done;
-    }
-
-  /* Detect the set of worker threads */
-  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-
-  /* Should never happen, shut up Coverity warning */
-  if (p == 0)
-    {
-      error = clib_error_return (0, "no worker registrations?");
-      goto done;
-    }
-
-  tr = (vlib_thread_registration_t *) p[0];
-
-  cfg = &devconf->hqos;
-  ht = xd->hqos_ht;
-  wk = &xd->hqos_wt[tr->first_index];
-  tctbl = wk->hqos_tc_table;
-
-  vlib_cli_output (vm, " Thread:");
-  vlib_cli_output (vm, "   Input SWQ size = %u packets", cfg->swq_size);
-  vlib_cli_output (vm, "   Enqueue burst size = %u packets",
-		   ht->hqos_burst_enq);
-  vlib_cli_output (vm, "   Dequeue burst size = %u packets",
-		   ht->hqos_burst_deq);
-
-  vlib_cli_output (vm,
-		   "   Packet field 0: slab position = %4u, slab bitmask = 0x%016llx   (subport)",
-		   wk->hqos_field0_slabpos, wk->hqos_field0_slabmask);
-  vlib_cli_output (vm,
-		   "   Packet field 1: slab position = %4u, slab bitmask = 0x%016llx   (pipe)",
-		   wk->hqos_field1_slabpos, wk->hqos_field1_slabmask);
-  vlib_cli_output (vm,
-		   "   Packet field 2: slab position = %4u, slab bitmask = 0x%016llx   (tc)",
-		   wk->hqos_field2_slabpos, wk->hqos_field2_slabmask);
-  vlib_cli_output (vm,
-		   "   Packet field 2  tc translation table: ([Mapped Value Range]: tc/queue tc/queue ...)");
-  vlib_cli_output (vm,
-		   "     [ 0 .. 15]: "
-		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
-		   tctbl[0] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[0] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[1] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[1] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[2] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[2] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[3] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[3] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[4] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[4] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[5] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[5] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[6] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[6] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[7] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[7] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[8] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[8] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[9] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[9] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[10] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[10] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[11] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[11] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[12] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[12] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[13] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[13] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[14] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[14] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[15] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[15] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
-  vlib_cli_output (vm,
-		   "     [16 .. 31]: "
-		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
-		   tctbl[16] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[16] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[17] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[17] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[18] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[18] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[19] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[19] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[20] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[20] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[21] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[21] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[22] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[22] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[23] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[23] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[24] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[24] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[25] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[25] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[26] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[26] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[27] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[27] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[28] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[28] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[29] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[29] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[30] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[30] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[31] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[31] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
-  vlib_cli_output (vm,
-		   "     [32 .. 47]: "
-		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
-		   tctbl[32] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[32] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[33] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[33] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[34] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[34] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[35] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[35] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[36] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[36] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[37] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[37] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[38] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[38] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[39] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[39] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[40] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[40] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[41] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[41] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[42] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[42] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[43] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[43] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[44] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[44] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[45] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[45] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[46] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[46] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[47] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[47] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
-  vlib_cli_output (vm,
-		   "     [48 .. 63]: "
-		   "%u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u %u/%u",
-		   tctbl[48] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[48] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[49] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[49] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[50] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[50] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[51] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[51] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[52] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[52] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[53] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[53] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[54] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[54] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[55] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[55] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[56] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[56] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[57] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[57] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[58] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[58] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[59] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[59] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[60] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[60] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[61] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[61] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[62] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[62] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[63] / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
-		   tctbl[63] % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
-  vlib_cli_output (vm, " Port:");
-  vlib_cli_output (vm, "   Rate = %u bytes/second", cfg->port.rate);
-  vlib_cli_output (vm, "   MTU = %u bytes", cfg->port.mtu);
-  vlib_cli_output (vm, "   Frame overhead = %u bytes",
-		   cfg->port.frame_overhead);
-  vlib_cli_output (vm, "   Number of subports = %u",
-		   cfg->port.n_subports_per_port);
-  vlib_cli_output (vm, "   Number of pipes per subport = %u",
-		   cfg->port.n_pipes_per_subport);
-  vlib_cli_output (vm,
-		   "   Packet queue size: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u packets",
-		   cfg->port.qsize[0], cfg->port.qsize[1], cfg->port.qsize[2],
-		   cfg->port.qsize[3]);
-  vlib_cli_output (vm, "   Number of pipe profiles = %u",
-		   cfg->port.n_pipe_profiles);
-
-  for (subport_id = 0; subport_id < vec_len (cfg->subport); subport_id++)
-    {
-      vlib_cli_output (vm, " Subport %u:", subport_id);
-      vlib_cli_output (vm, "   Rate = %u bytes/second",
-		       cfg->subport[subport_id].tb_rate);
-      vlib_cli_output (vm, "   Token bucket size = %u bytes",
-		       cfg->subport[subport_id].tb_size);
-      vlib_cli_output (vm,
-		       "   Traffic class rate: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u bytes/second",
-		       cfg->subport[subport_id].tc_rate[0],
-		       cfg->subport[subport_id].tc_rate[1],
-		       cfg->subport[subport_id].tc_rate[2],
-		       cfg->subport[subport_id].tc_rate[3]);
-      vlib_cli_output (vm, "   TC period = %u milliseconds",
-		       cfg->subport[subport_id].tc_period);
-    }
-
-  for (profile_id = 0; profile_id < vec_len (cfg->pipe); profile_id++)
-    {
-      vlib_cli_output (vm, " Pipe profile %u:", profile_id);
-      vlib_cli_output (vm, "   Rate = %u bytes/second",
-		       cfg->pipe[profile_id].tb_rate);
-      vlib_cli_output (vm, "   Token bucket size = %u bytes",
-		       cfg->pipe[profile_id].tb_size);
-      vlib_cli_output (vm,
-		       "   Traffic class rate: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u bytes/second",
-		       cfg->pipe[profile_id].tc_rate[0],
-		       cfg->pipe[profile_id].tc_rate[1],
-		       cfg->pipe[profile_id].tc_rate[2],
-		       cfg->pipe[profile_id].tc_rate[3]);
-      vlib_cli_output (vm, "   TC period = %u milliseconds",
-		       cfg->pipe[profile_id].tc_period);
-#ifdef RTE_SCHED_SUBPORT_TC_OV
-      vlib_cli_output (vm, "   TC3 oversubscription_weight = %u",
-		       cfg->pipe[profile_id].tc_ov_weight);
-#endif
-
-      for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
-	{
-	  vlib_cli_output (vm,
-			   "   TC%u WRR weights: Q0 = %u, Q1 = %u, Q2 = %u, Q3 = %u",
-			   i, cfg->pipe[profile_id].wrr_weights[i * 4],
-			   cfg->pipe[profile_id].wrr_weights[i * 4 + 1],
-			   cfg->pipe[profile_id].wrr_weights[i * 4 + 2],
-			   cfg->pipe[profile_id].wrr_weights[i * 4 + 3]);
-	}
-    }
-
-#ifdef RTE_SCHED_RED
-  vlib_cli_output (vm, " Weighted Random Early Detection (WRED):");
-  for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
-    {
-      vlib_cli_output (vm, "   TC%u min: G = %u, Y = %u, R = %u", i,
-		       cfg->port.red_params[i][e_RTE_METER_GREEN].min_th,
-		       cfg->port.red_params[i][e_RTE_METER_YELLOW].min_th,
-		       cfg->port.red_params[i][e_RTE_METER_RED].min_th);
-
-      vlib_cli_output (vm, "   TC%u max: G = %u, Y = %u, R = %u", i,
-		       cfg->port.red_params[i][e_RTE_METER_GREEN].max_th,
-		       cfg->port.red_params[i][e_RTE_METER_YELLOW].max_th,
-		       cfg->port.red_params[i][e_RTE_METER_RED].max_th);
-
-      vlib_cli_output (vm,
-		       "   TC%u inverted probability: G = %u, Y = %u, R = %u",
-		       i, cfg->port.red_params[i][e_RTE_METER_GREEN].maxp_inv,
-		       cfg->port.red_params[i][e_RTE_METER_YELLOW].maxp_inv,
-		       cfg->port.red_params[i][e_RTE_METER_RED].maxp_inv);
-
-      vlib_cli_output (vm, "   TC%u weight: R = %u, Y = %u, R = %u", i,
-		       cfg->port.red_params[i][e_RTE_METER_GREEN].wq_log2,
-		       cfg->port.red_params[i][e_RTE_METER_YELLOW].wq_log2,
-		       cfg->port.red_params[i][e_RTE_METER_RED].wq_log2);
-    }
-#endif
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to display details of an output interface's HQoS
- * settings.
- *
- * @cliexpar
- * Example of how to display HQoS settings for an interfaces:
- * @cliexstart{show dpdk interface hqos GigabitEthernet0/8/0}
- *  Thread:
- *    Input SWQ size = 4096 packets
- *    Enqueue burst size = 256 packets
- *    Dequeue burst size = 220 packets
- *    Packet field 0: slab position =    0, slab bitmask = 0x0000000000000000   (subport)
- *    Packet field 1: slab position =   40, slab bitmask = 0x0000000fff000000   (pipe)
- *    Packet field 2: slab position =    8, slab bitmask = 0x00000000000000fc   (tc)
- *    Packet field 2  tc translation table: ([Mapped Value Range]: tc/queue tc/queue ...)
- *      [ 0 .. 15]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
- *      [16 .. 31]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
- *      [32 .. 47]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
- *      [48 .. 63]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
- *  Port:
- *    Rate = 1250000000 bytes/second
- *    MTU = 1514 bytes
- *    Frame overhead = 24 bytes
- *    Number of subports = 1
- *    Number of pipes per subport = 4096
- *    Packet queue size: TC0 = 64, TC1 = 64, TC2 = 64, TC3 = 64 packets
- *    Number of pipe profiles = 2
- *  Subport 0:
- *    Rate = 1250000000 bytes/second
- *    Token bucket size = 1000000 bytes
- *    Traffic class rate: TC0 = 1250000000, TC1 = 1250000000, TC2 = 1250000000, TC3 = 1250000000 bytes/second
- *    TC period = 10 milliseconds
- *  Pipe profile 0:
- *    Rate = 305175 bytes/second
- *    Token bucket size = 1000000 bytes
- *    Traffic class rate: TC0 = 305175, TC1 = 305175, TC2 = 305175, TC3 = 305175 bytes/second
- *    TC period = 40 milliseconds
- *    TC0 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
- *    TC1 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
- *    TC2 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
- *    TC3 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_show_dpdk_if_hqos, static) = {
-  .path = "show dpdk interface hqos",
-  .short_help = "show dpdk interface hqos ",
-  .function = show_dpdk_if_hqos,
-};
-
-/* *INDENT-ON* */
-
-static clib_error_t *
-show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
-			    vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t *error = NULL;
-#ifdef RTE_SCHED_COLLECT_STATS
-  dpdk_main_t *dm = &dpdk_main;
-  u32 hw_if_index = (u32) ~ 0;
-  u32 subport = (u32) ~ 0;
-  u32 pipe = (u32) ~ 0;
-  u32 tc = (u32) ~ 0;
-  u32 tc_q = (u32) ~ 0;
-  vnet_hw_interface_t *hw;
-  dpdk_device_t *xd;
-  uword *p = 0;
-  struct rte_eth_dev_info dev_info;
-  dpdk_device_config_t *devconf = 0;
-  u32 qindex;
-  struct rte_sched_queue_stats stats;
-  u16 qlen;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main,
-	   &hw_if_index))
-	;
-
-      else if (unformat (line_input, "subport %d", &subport))
-	;
-
-      else if (unformat (line_input, "pipe %d", &pipe))
-	;
-
-      else if (unformat (line_input, "tc %d", &tc))
-	;
-
-      else if (unformat (line_input, "tc_q %d", &tc_q))
-	;
-
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (hw_if_index == (u32) ~ 0)
-    {
-      error = clib_error_return (0, "please specify interface name!!");
-      goto done;
-    }
-
-  hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rte_eth_dev_info_get (xd->device_index, &dev_info);
-  if (dev_info.pci_dev)
-    {				/* bonded interface has no pci info */
-      vlib_pci_addr_t pci_addr;
-
-      pci_addr.domain = dev_info.pci_dev->addr.domain;
-      pci_addr.bus = dev_info.pci_dev->addr.bus;
-      pci_addr.slot = dev_info.pci_dev->addr.devid;
-      pci_addr.function = dev_info.pci_dev->addr.function;
-
-      p =
-	hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32);
-    }
-
-  if (p)
-    devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
-  else
-    devconf = &dm->conf->default_devconf;
-
-  if (devconf->hqos_enabled == 0)
-    {
-      vlib_cli_output (vm, "HQoS disabled for this interface");
-      goto done;
-    }
-
-  /*
-   * Figure out which queue to query.  cf rte_sched_port_qindex.  (Not sure why
-   * that method isn't made public by DPDK - how _should_ we get the queue ID?)
-   */
-  qindex = subport * devconf->hqos.port.n_pipes_per_subport + pipe;
-  qindex = qindex * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE + tc;
-  qindex = qindex * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + tc_q;
-
-  if (rte_sched_queue_read_stats (xd->hqos_ht->hqos, qindex, &stats, &qlen) !=
-      0)
-    {
-      error = clib_error_return (0, "failed to read stats");
-      goto done;
-    }
-
-  vlib_cli_output (vm, "%=24s%=16s", "Stats Parameter", "Value");
-  vlib_cli_output (vm, "%=24s%=16d", "Packets", stats.n_pkts);
-  vlib_cli_output (vm, "%=24s%=16d", "Packets dropped", stats.n_pkts_dropped);
-#ifdef RTE_SCHED_RED
-  vlib_cli_output (vm, "%=24s%=16d", "Packets dropped (RED)",
-		   stats.n_pkts_red_dropped);
-#endif
-  vlib_cli_output (vm, "%=24s%=16d", "Bytes", stats.n_bytes);
-  vlib_cli_output (vm, "%=24s%=16d", "Bytes dropped", stats.n_bytes_dropped);
-
-#else
-
-  /* Get a line of input */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  vlib_cli_output (vm, "RTE_SCHED_COLLECT_STATS disabled in DPDK");
-  goto done;
-
-#endif
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * This command is used to display statistics associated with a HQoS traffic class
- * queue.
- *
- * @note
- * Statistic collection by the scheduler is disabled by default in DPDK. In order to
- * turn it on, add the following line to '../vpp/dpdk/Makefile':
- * - $(call set,RTE_SCHED_COLLECT_STATS,y)
- *
- * @cliexpar
- * Example of how to display statistics of HQoS a HQoS traffic class queue:
- * @cliexstart{show dpdk hqos queue GigabitEthernet0/9/0 subport 0 pipe 3181 tc 0 tc_q 0}
- *      Stats Parameter          Value
- *          Packets               140
- *      Packets dropped            0
- *           Bytes               8400
- *       Bytes dropped             0
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (cmd_show_dpdk_hqos_queue_stats, static) = {
-  .path = "show dpdk hqos queue",
-  .short_help = "show dpdk hqos queue  subport  pipe  tc  tc_q ",
-  .function = show_dpdk_hqos_queue_stats,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-show_dpdk_version_command_fn (vlib_main_t * vm,
-			      unformat_input_t * input,
-			      vlib_cli_command_t * cmd)
-{
-#define _(a,b,c) vlib_cli_output (vm, "%-25s " b, a ":", c);
-  _("DPDK Version", "%s", rte_version ());
-  _("DPDK EAL init args", "%s", dpdk_config_main.eal_init_args_str);
-#undef _
-  return 0;
-}
-
-/*?
- * This command is used to display the current DPDK version and
- * the list of arguments passed to DPDK when started.
- *
- * @cliexpar
- * Example of how to display how many DPDK buffer test command has allcoated:
- * @cliexstart{show dpdk version}
- * DPDK Version:        DPDK 16.11.0
- * DPDK EAL init args:  -c 1 -n 4 --huge-dir /run/vpp/hugepages --file-prefix vpp -w 0000:00:08.0 -w 0000:00:09.0 --master-lcore 0 --socket-mem 256
- * @cliexend
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (show_vpe_version_command, static) = {
-  .path = "show dpdk version",
-  .short_help = "show dpdk version",
-  .function = show_dpdk_version_command_fn,
-};
-/* *INDENT-ON* */
-
-clib_error_t *
-dpdk_cli_init (vlib_main_t * vm)
-{
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_cli_init);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/device.c b/src/vnet/devices/dpdk/device.c
deleted file mode 100644
index 17397900..00000000
--- a/src/vnet/devices/dpdk/device.c
+++ /dev/null
@@ -1,852 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-
-#include "dpdk_priv.h"
-#include 
-
-#define foreach_dpdk_tx_func_error			\
-  _(BAD_RETVAL, "DPDK tx function returned an error")	\
-  _(RING_FULL, "Tx packet drops (ring full)")	        \
-  _(PKT_DROP, "Tx packet drops (dpdk tx failure)")	\
-  _(REPL_FAIL, "Tx packet drops (replication failure)")
-
-typedef enum
-{
-#define _(f,s) DPDK_TX_FUNC_ERROR_##f,
-  foreach_dpdk_tx_func_error
-#undef _
-    DPDK_TX_FUNC_N_ERROR,
-} dpdk_tx_func_error_t;
-
-static char *dpdk_tx_func_error_strings[] = {
-#define _(n,s) s,
-  foreach_dpdk_tx_func_error
-#undef _
-};
-
-clib_error_t *
-dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address)
-{
-  int error;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
-
-  error = rte_eth_dev_default_mac_addr_set (xd->device_index,
-					    (struct ether_addr *) address);
-
-  if (error)
-    {
-      return clib_error_return (0, "mac address set failed: %d", error);
-    }
-  else
-    {
-      vec_reset_length (xd->default_mac_address);
-      vec_add (xd->default_mac_address, address, sizeof (address));
-      return NULL;
-    }
-}
-
-clib_error_t *
-dpdk_set_mc_filter (vnet_hw_interface_t * hi,
-		    struct ether_addr mc_addr_vec[], int naddr)
-{
-  int error;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
-
-  error = rte_eth_dev_set_mc_addr_list (xd->device_index, mc_addr_vec, naddr);
-
-  if (error)
-    {
-      return clib_error_return (0, "mc addr list failed: %d", error);
-    }
-  else
-    {
-      return NULL;
-    }
-}
-
-struct rte_mbuf *
-dpdk_replicate_packet_mb (vlib_buffer_t * b)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  struct rte_mbuf **mbufs = 0, *s, *d;
-  u8 nb_segs;
-  unsigned socket_id = rte_socket_id ();
-  int i;
-
-  ASSERT (dm->pktmbuf_pools[socket_id]);
-  s = rte_mbuf_from_vlib_buffer (b);
-  nb_segs = s->nb_segs;
-  vec_validate (mbufs, nb_segs - 1);
-
-  if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
-    {
-      vec_free (mbufs);
-      return 0;
-    }
-
-  d = mbufs[0];
-  d->nb_segs = s->nb_segs;
-  d->data_len = s->data_len;
-  d->pkt_len = s->pkt_len;
-  d->data_off = s->data_off;
-  clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
-
-  for (i = 1; i < nb_segs; i++)
-    {
-      d->next = mbufs[i];
-      d = mbufs[i];
-      s = s->next;
-      d->data_len = s->data_len;
-      clib_memcpy (d->buf_addr, s->buf_addr,
-		   RTE_PKTMBUF_HEADROOM + s->data_len);
-    }
-
-  d = mbufs[0];
-  vec_free (mbufs);
-  return d;
-}
-
-static void
-dpdk_tx_trace_buffer (dpdk_main_t * dm,
-		      vlib_node_runtime_t * node,
-		      dpdk_device_t * xd,
-		      u16 queue_id, u32 buffer_index, vlib_buffer_t * buffer)
-{
-  vlib_main_t *vm = vlib_get_main ();
-  dpdk_tx_dma_trace_t *t0;
-  struct rte_mbuf *mb;
-
-  mb = rte_mbuf_from_vlib_buffer (buffer);
-
-  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
-  t0->queue_index = queue_id;
-  t0->device_index = xd->device_index;
-  t0->buffer_index = buffer_index;
-  clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
-  clib_memcpy (&t0->buffer, buffer,
-	       sizeof (buffer[0]) - sizeof (buffer->pre_data));
-  clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data,
-	       sizeof (t0->buffer.pre_data));
-}
-
-static_always_inline void
-dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b,
-			int maybe_multiseg)
-{
-  struct rte_mbuf *mb, *first_mb, *last_mb;
-
-  /* buffer is coming from non-dpdk source so we need to init
-     rte_mbuf header */
-  if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0))
-    {
-      vlib_buffer_t *b2 = b;
-      last_mb = mb = rte_mbuf_from_vlib_buffer (b2);
-      rte_pktmbuf_reset (mb);
-      while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT))
-	{
-	  b2 = vlib_get_buffer (vm, b2->next_buffer);
-	  mb = rte_mbuf_from_vlib_buffer (b2);
-	  rte_pktmbuf_reset (mb);
-	}
-    }
-
-  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
-  first_mb->nb_segs = 1;
-  mb->data_len = b->current_length;
-  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
-    b->current_length;
-  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
-
-  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
-    {
-      b = vlib_get_buffer (vm, b->next_buffer);
-      mb = rte_mbuf_from_vlib_buffer (b);
-      last_mb->next = mb;
-      last_mb = mb;
-      mb->data_len = b->current_length;
-      mb->pkt_len = b->current_length;
-      mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
-      first_mb->nb_segs++;
-      if (PREDICT_FALSE (b->n_add_refs))
-	{
-	  rte_mbuf_refcnt_update (mb, b->n_add_refs);
-	  b->n_add_refs = 0;
-	}
-    }
-}
-
-/*
- * This function calls the dpdk's tx_burst function to transmit the packets
- * on the tx_vector. It manages a lock per-device if the device does not
- * support multiple queues. It returns the number of packets untransmitted
- * on the tx_vector. If all packets are transmitted (the normal case), the
- * function returns 0.
- *
- * The function assumes there is at least one packet on the tx_vector.
- */
-static_always_inline
-  u32 tx_burst_vector_internal (vlib_main_t * vm,
-				dpdk_device_t * xd,
-				struct rte_mbuf **tx_vector)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  u32 n_packets;
-  u32 tx_head;
-  u32 tx_tail;
-  u32 n_retry;
-  int rv;
-  int queue_id;
-  tx_ring_hdr_t *ring;
-
-  ring = vec_header (tx_vector, sizeof (*ring));
-
-  n_packets = ring->tx_head - ring->tx_tail;
-
-  tx_head = ring->tx_head % xd->nb_tx_desc;
-
-  /*
-   * Ensure rte_eth_tx_burst is not called with 0 packets, which can lead to
-   * unpredictable results.
-   */
-  ASSERT (n_packets > 0);
-
-  /*
-   * Check for tx_vector overflow. If this fails it is a system configuration
-   * error. The ring should be sized big enough to handle the largest un-flowed
-   * off burst from a traffic manager. A larger size also helps performance
-   * a bit because it decreases the probability of having to issue two tx_burst
-   * calls due to a ring wrap.
-   */
-  ASSERT (n_packets < xd->nb_tx_desc);
-  ASSERT (ring->tx_tail == 0);
-
-  n_retry = 16;
-  queue_id = vm->cpu_index;
-
-  do
-    {
-      /* start the burst at the tail */
-      tx_tail = ring->tx_tail % xd->nb_tx_desc;
-
-      /*
-       * This device only supports one TX queue,
-       * and we're running multi-threaded...
-       */
-      if (PREDICT_FALSE (xd->lockp != 0))
-	{
-	  queue_id = queue_id % xd->tx_q_used;
-	  while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
-	    /* zzzz */
-	    queue_id = (queue_id + 1) % xd->tx_q_used;
-	}
-
-      if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS))	/* HQoS ON */
-	{
-	  /* no wrap, transmit in one burst */
-	  dpdk_device_hqos_per_worker_thread_t *hqos =
-	    &xd->hqos_wt[vm->cpu_index];
-
-	  ASSERT (hqos->swq != NULL);
-
-	  dpdk_hqos_metadata_set (hqos,
-				  &tx_vector[tx_tail], tx_head - tx_tail);
-	  rv = rte_ring_sp_enqueue_burst (hqos->swq,
-					  (void **) &tx_vector[tx_tail],
-					  (uint16_t) (tx_head - tx_tail));
-	}
-      else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
-	{
-	  /* no wrap, transmit in one burst */
-	  rv = rte_eth_tx_burst (xd->device_index,
-				 (uint16_t) queue_id,
-				 &tx_vector[tx_tail],
-				 (uint16_t) (tx_head - tx_tail));
-	}
-      else
-	{
-	  ASSERT (0);
-	  rv = 0;
-	}
-
-      if (PREDICT_FALSE (xd->lockp != 0))
-	*xd->lockp[queue_id] = 0;
-
-      if (PREDICT_FALSE (rv < 0))
-	{
-	  // emit non-fatal message, bump counter
-	  vnet_main_t *vnm = dm->vnet_main;
-	  vnet_interface_main_t *im = &vnm->interface_main;
-	  u32 node_index;
-
-	  node_index = vec_elt_at_index (im->hw_interfaces,
-					 xd->vlib_hw_if_index)->tx_node_index;
-
-	  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
-	  clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index,
-			rv);
-	  return n_packets;	// untransmitted packets
-	}
-      ring->tx_tail += (u16) rv;
-      n_packets -= (uint16_t) rv;
-    }
-  while (rv && n_packets && (n_retry > 0));
-
-  return n_packets;
-}
-
-static_always_inline void
-dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
-{
-  vlib_buffer_t *b;
-  struct rte_mbuf *mb;
-  b = vlib_get_buffer (vm, bi);
-  mb = rte_mbuf_from_vlib_buffer (b);
-  CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, LOAD);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
-}
-
-static_always_inline void
-dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node,
-		     vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  u32 my_cpu = vm->cpu_index;
-  struct rte_mbuf *mb_new;
-
-  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
-    return;
-
-  mb_new = dpdk_replicate_packet_mb (b);
-  if (PREDICT_FALSE (mb_new == 0))
-    {
-      vlib_error_count (vm, node->node_index,
-			DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
-      b->flags |= VLIB_BUFFER_REPL_FAIL;
-    }
-  else
-    *mbp = mb_new;
-
-  vec_add1 (dm->recycle[my_cpu], bi);
-}
-
-/*
- * Transmits the packets on the frame to the interface associated with the
- * node. It first copies packets on the frame to a tx_vector containing the
- * rte_mbuf pointers. It then passes this vector to tx_burst_vector_internal
- * which calls the dpdk tx_burst function.
- */
-static uword
-dpdk_interface_tx (vlib_main_t * vm,
-		   vlib_node_runtime_t * node, vlib_frame_t * f)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, rd->dev_instance);
-  u32 n_packets = f->n_vectors;
-  u32 n_left;
-  u32 *from;
-  struct rte_mbuf **tx_vector;
-  u16 i;
-  u16 nb_tx_desc = xd->nb_tx_desc;
-  int queue_id;
-  u32 my_cpu;
-  u32 tx_pkts = 0;
-  tx_ring_hdr_t *ring;
-  u32 n_on_ring;
-
-  my_cpu = vm->cpu_index;
-
-  queue_id = my_cpu;
-
-  tx_vector = xd->tx_vectors[queue_id];
-  ring = vec_header (tx_vector, sizeof (*ring));
-
-  n_on_ring = ring->tx_head - ring->tx_tail;
-  from = vlib_frame_vector_args (f);
-
-  ASSERT (n_packets <= VLIB_FRAME_SIZE);
-
-  if (PREDICT_FALSE (n_on_ring + n_packets > nb_tx_desc))
-    {
-      /*
-       * Overflowing the ring should never happen.
-       * If it does then drop the whole frame.
-       */
-      vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_RING_FULL,
-			n_packets);
-
-      while (n_packets--)
-	{
-	  u32 bi0 = from[n_packets];
-	  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
-	  struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer (b0);
-	  rte_pktmbuf_free (mb0);
-	}
-      return n_on_ring;
-    }
-
-  if (PREDICT_FALSE (dm->tx_pcap_enable))
-    {
-      n_left = n_packets;
-      while (n_left > 0)
-	{
-	  u32 bi0 = from[0];
-	  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
-	  if (dm->pcap_sw_if_index == 0 ||
-	      dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX])
-	    pcap_add_buffer (&dm->pcap_main, vm, bi0, 512);
-	  from++;
-	  n_left--;
-	}
-    }
-
-  from = vlib_frame_vector_args (f);
-  n_left = n_packets;
-  i = ring->tx_head % nb_tx_desc;
-
-  while (n_left >= 8)
-    {
-      u32 bi0, bi1, bi2, bi3;
-      struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
-      vlib_buffer_t *b0, *b1, *b2, *b3;
-      u32 or_flags;
-
-      dpdk_prefetch_buffer_by_index (vm, from[4]);
-      dpdk_prefetch_buffer_by_index (vm, from[5]);
-      dpdk_prefetch_buffer_by_index (vm, from[6]);
-      dpdk_prefetch_buffer_by_index (vm, from[7]);
-
-      bi0 = from[0];
-      bi1 = from[1];
-      bi2 = from[2];
-      bi3 = from[3];
-      from += 4;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      b1 = vlib_get_buffer (vm, bi1);
-      b2 = vlib_get_buffer (vm, bi2);
-      b3 = vlib_get_buffer (vm, bi3);
-
-      or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
-
-      if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
-	{
-	  dpdk_validate_rte_mbuf (vm, b0, 1);
-	  dpdk_validate_rte_mbuf (vm, b1, 1);
-	  dpdk_validate_rte_mbuf (vm, b2, 1);
-	  dpdk_validate_rte_mbuf (vm, b3, 1);
-	}
-      else
-	{
-	  dpdk_validate_rte_mbuf (vm, b0, 0);
-	  dpdk_validate_rte_mbuf (vm, b1, 0);
-	  dpdk_validate_rte_mbuf (vm, b2, 0);
-	  dpdk_validate_rte_mbuf (vm, b3, 0);
-	}
-
-      mb0 = rte_mbuf_from_vlib_buffer (b0);
-      mb1 = rte_mbuf_from_vlib_buffer (b1);
-      mb2 = rte_mbuf_from_vlib_buffer (b2);
-      mb3 = rte_mbuf_from_vlib_buffer (b3);
-
-      if (PREDICT_FALSE (or_flags & VLIB_BUFFER_RECYCLE))
-	{
-	  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
-	  dpdk_buffer_recycle (vm, node, b1, bi1, &mb1);
-	  dpdk_buffer_recycle (vm, node, b2, bi2, &mb2);
-	  dpdk_buffer_recycle (vm, node, b3, bi3, &mb3);
-
-	  /* dont enqueue packets if replication failed as they must
-	     be sent back to recycle */
-	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
-	    tx_vector[i++ % nb_tx_desc] = mb0;
-	  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0))
-	    tx_vector[i++ % nb_tx_desc] = mb1;
-	  if (PREDICT_TRUE ((b2->flags & VLIB_BUFFER_REPL_FAIL) == 0))
-	    tx_vector[i++ % nb_tx_desc] = mb2;
-	  if (PREDICT_TRUE ((b3->flags & VLIB_BUFFER_REPL_FAIL) == 0))
-	    tx_vector[i++ % nb_tx_desc] = mb3;
-	}
-      else
-	{
-	  if (PREDICT_FALSE (i + 3 >= nb_tx_desc))
-	    {
-	      tx_vector[i++ % nb_tx_desc] = mb0;
-	      tx_vector[i++ % nb_tx_desc] = mb1;
-	      tx_vector[i++ % nb_tx_desc] = mb2;
-	      tx_vector[i++ % nb_tx_desc] = mb3;
-	      i %= nb_tx_desc;
-	    }
-	  else
-	    {
-	      tx_vector[i++] = mb0;
-	      tx_vector[i++] = mb1;
-	      tx_vector[i++] = mb2;
-	      tx_vector[i++] = mb3;
-	    }
-	}
-
-
-      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
-	{
-	  if (b0->flags & VLIB_BUFFER_IS_TRACED)
-	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
-	  if (b1->flags & VLIB_BUFFER_IS_TRACED)
-	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1);
-	  if (b2->flags & VLIB_BUFFER_IS_TRACED)
-	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi2, b2);
-	  if (b3->flags & VLIB_BUFFER_IS_TRACED)
-	    dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi3, b3);
-	}
-
-      n_left -= 4;
-    }
-  while (n_left > 0)
-    {
-      u32 bi0;
-      struct rte_mbuf *mb0;
-      vlib_buffer_t *b0;
-
-      bi0 = from[0];
-      from++;
-
-      b0 = vlib_get_buffer (vm, bi0);
-
-      dpdk_validate_rte_mbuf (vm, b0, 1);
-
-      mb0 = rte_mbuf_from_vlib_buffer (b0);
-      dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
-
-      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
-	if (b0->flags & VLIB_BUFFER_IS_TRACED)
-	  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
-
-      if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
-	{
-	  tx_vector[i % nb_tx_desc] = mb0;
-	  i++;
-	}
-      n_left--;
-    }
-
-  /* account for additional packets in the ring */
-  ring->tx_head += n_packets;
-  n_on_ring = ring->tx_head - ring->tx_tail;
-
-  /* transmit as many packets as possible */
-  n_packets = tx_burst_vector_internal (vm, xd, tx_vector);
-
-  /*
-   * tx_pkts is the number of packets successfully transmitted
-   * This is the number originally on ring minus the number remaining on ring
-   */
-  tx_pkts = n_on_ring - n_packets;
-
-  {
-    /* If there is no callback then drop any non-transmitted packets */
-    if (PREDICT_FALSE (n_packets))
-      {
-	vlib_simple_counter_main_t *cm;
-	vnet_main_t *vnm = vnet_get_main ();
-
-	cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
-			       VNET_INTERFACE_COUNTER_TX_ERROR);
-
-	vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
-				       n_packets);
-
-	vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
-			  n_packets);
-
-	while (n_packets--)
-	  rte_pktmbuf_free (tx_vector[ring->tx_tail + n_packets]);
-      }
-
-    /* Reset head/tail to avoid unnecessary wrap */
-    ring->tx_head = 0;
-    ring->tx_tail = 0;
-  }
-
-  /* Recycle replicated buffers */
-  if (PREDICT_FALSE (vec_len (dm->recycle[my_cpu])))
-    {
-      vlib_buffer_free (vm, dm->recycle[my_cpu],
-			vec_len (dm->recycle[my_cpu]));
-      _vec_len (dm->recycle[my_cpu]) = 0;
-    }
-
-  ASSERT (ring->tx_head >= ring->tx_tail);
-
-  return tx_pkts;
-}
-
-static void
-dpdk_clear_hw_interface_counters (u32 instance)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
-
-  /*
-   * Set the "last_cleared_stats" to the current stats, so that
-   * things appear to clear from a display perspective.
-   */
-  dpdk_update_counters (xd, vlib_time_now (dm->vlib_main));
-
-  clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
-  clib_memcpy (xd->last_cleared_xstats, xd->xstats,
-	       vec_len (xd->last_cleared_xstats) *
-	       sizeof (xd->last_cleared_xstats[0]));
-
-}
-
-static clib_error_t *
-dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
-{
-  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
-  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hif->dev_instance);
-  int rv = 0;
-
-  if (is_up)
-    {
-      f64 now = vlib_time_now (dm->vlib_main);
-
-      if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
-	{
-	  rv = rte_eth_dev_start (xd->device_index);
-	  if (!rv && xd->default_mac_address)
-	    rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
-						   (struct ether_addr *)
-						   xd->default_mac_address);
-	}
-
-      if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
-	rte_eth_promiscuous_enable (xd->device_index);
-      else
-	rte_eth_promiscuous_disable (xd->device_index);
-
-      rte_eth_allmulticast_enable (xd->device_index);
-      xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
-      dpdk_update_counters (xd, now);
-      dpdk_update_link_state (xd, now);
-    }
-  else
-    {
-      xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
-
-      rte_eth_allmulticast_disable (xd->device_index);
-      vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, 0);
-      rte_eth_dev_stop (xd->device_index);
-
-      /* For bonded interface, stop slave links */
-      if (xd->pmd == VNET_DPDK_PMD_BOND)
-	{
-	  u8 slink[16];
-	  int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16);
-	  while (nlink >= 1)
-	    {
-	      u8 dpdk_port = slink[--nlink];
-	      rte_eth_dev_stop (dpdk_port);
-	    }
-	}
-    }
-
-  if (rv < 0)
-    clib_warning ("rte_eth_dev_%s error: %d", is_up ? "start" : "stop", rv);
-
-  return /* no error */ 0;
-}
-
-/*
- * Dynamically redirect all pkts from a specific interface
- * to the specified node
- */
-static void
-dpdk_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
-			      u32 node_index)
-{
-  dpdk_main_t *xm = &dpdk_main;
-  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
-  dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
-
-  /* Shut off redirection */
-  if (node_index == ~0)
-    {
-      xd->per_interface_next_index = node_index;
-      return;
-    }
-
-  xd->per_interface_next_index =
-    vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
-}
-
-
-static clib_error_t *
-dpdk_subif_add_del_function (vnet_main_t * vnm,
-			     u32 hw_if_index,
-			     struct vnet_sw_interface_t *st, int is_add)
-{
-  dpdk_main_t *xm = &dpdk_main;
-  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
-  dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
-  vnet_sw_interface_t *t = (vnet_sw_interface_t *) st;
-  int r, vlan_offload;
-  u32 prev_subifs = xd->num_subifs;
-  clib_error_t *err = 0;
-
-  if (is_add)
-    xd->num_subifs++;
-  else if (xd->num_subifs)
-    xd->num_subifs--;
-
-  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
-    goto done;
-
-  /* currently we program VLANS only for IXGBE VF and I40E VF */
-  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
-    goto done;
-
-  if (t->sub.eth.flags.no_tags == 1)
-    goto done;
-
-  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
-    {
-      xd->num_subifs = prev_subifs;
-      err = clib_error_return (0, "unsupported VLAN setup");
-      goto done;
-    }
-
-  vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index);
-  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
-
-  if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload)))
-    {
-      xd->num_subifs = prev_subifs;
-      err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
-			       xd->device_index, r);
-      goto done;
-    }
-
-
-  if ((r =
-       rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id,
-				is_add)))
-    {
-      xd->num_subifs = prev_subifs;
-      err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
-			       xd->device_index, r);
-      goto done;
-    }
-
-done:
-  if (xd->num_subifs)
-    xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
-  else
-    xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
-
-  return err;
-}
-
-/* *INDENT-OFF* */
-VNET_DEVICE_CLASS (dpdk_device_class) = {
-  .name = "dpdk",
-  .tx_function = dpdk_interface_tx,
-  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
-  .tx_function_error_strings = dpdk_tx_func_error_strings,
-  .format_device_name = format_dpdk_device_name,
-  .format_device = format_dpdk_device,
-  .format_tx_trace = format_dpdk_tx_dma_trace,
-  .clear_counters = dpdk_clear_hw_interface_counters,
-  .admin_up_down_function = dpdk_interface_admin_up_down,
-  .subif_add_del_function = dpdk_subif_add_del_function,
-  .rx_redirect_to_node = dpdk_set_interface_next_node,
-  .mac_addr_change_function = dpdk_set_mac_address,
-};
-
-VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class, dpdk_interface_tx)
-/* *INDENT-ON* */
-
-#define UP_DOWN_FLAG_EVENT 1
-
-uword
-admin_up_down_process (vlib_main_t * vm,
-		       vlib_node_runtime_t * rt, vlib_frame_t * f)
-{
-  clib_error_t *error = 0;
-  uword event_type;
-  uword *event_data = 0;
-  u32 sw_if_index;
-  u32 flags;
-
-  while (1)
-    {
-      vlib_process_wait_for_event (vm);
-
-      event_type = vlib_process_get_events (vm, &event_data);
-
-      dpdk_main.admin_up_down_in_progress = 1;
-
-      switch (event_type)
-	{
-	case UP_DOWN_FLAG_EVENT:
-	  {
-	    if (vec_len (event_data) == 2)
-	      {
-		sw_if_index = event_data[0];
-		flags = event_data[1];
-		error =
-		  vnet_sw_interface_set_flags (vnet_get_main (), sw_if_index,
-					       flags);
-		clib_error_report (error);
-	      }
-	  }
-	  break;
-	}
-
-      vec_reset_length (event_data);
-
-      dpdk_main.admin_up_down_in_progress = 0;
-
-    }
-  return 0;			/* or not */
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (admin_up_down_process_node,static) = {
-    .function = admin_up_down_process,
-    .type = VLIB_NODE_TYPE_PROCESS,
-    .name = "admin-up-down-process",
-    .process_log2_n_stack_bytes = 17,  // 256KB
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/dir.dox b/src/vnet/devices/dpdk/dir.dox
deleted file mode 100644
index 43e36753..00000000
--- a/src/vnet/devices/dpdk/dir.dox
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* Doxygen directory documentation */
-
-/**
-@dir
-@brief DPDK Abstraction Layer.
-
-This directory contains the source code for the DPDK abstraction layer.
-
-*/
-/*? %%clicmd:group_label DPDK and pcap tx %% ?*/
-/*? %%syscfg:group_label DPDK and pcap tx %% ?*/
diff --git a/src/vnet/devices/dpdk/dpdk.api b/src/vnet/devices/dpdk/dpdk.api
deleted file mode 100644
index 21215d45..00000000
--- a/src/vnet/devices/dpdk/dpdk.api
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** \brief DPDK interface HQoS pipe profile set request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - the interface
-    @param subport - subport ID
-    @param pipe - pipe ID within its subport
-    @param profile - pipe profile ID
-*/
-define sw_interface_set_dpdk_hqos_pipe {
-    u32 client_index;
-    u32 context;
-    u32 sw_if_index;
-    u32 subport;
-    u32 pipe;
-    u32 profile;
-};
-
-/** \brief DPDK interface HQoS pipe profile set reply
-    @param context - sender context, to match reply w/ request
-    @param retval - request return code
-*/
-define sw_interface_set_dpdk_hqos_pipe_reply {
-    u32 context;
-    i32 retval;
-};
-
-/** \brief DPDK interface HQoS subport parameters set request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - the interface
-    @param subport - subport ID
-    @param tb_rate - subport token bucket rate (measured in bytes/second)
-    @param tb_size - subport token bucket size (measured in credits)
-    @param tc_rate - subport traffic class 0 .. 3 rates (measured in bytes/second)
-    @param tc_period - enforcement period for rates (measured in milliseconds)
-*/
-define sw_interface_set_dpdk_hqos_subport {
-    u32 client_index;
-    u32 context;
-    u32 sw_if_index;
-    u32 subport;
-    u32 tb_rate;
-    u32 tb_size;
-    u32 tc_rate[4];
-    u32 tc_period;
-};
-
-/** \brief DPDK interface HQoS subport parameters set reply
-    @param context - sender context, to match reply w/ request
-    @param retval - request return code
-*/
-define sw_interface_set_dpdk_hqos_subport_reply {
-    u32 context;
-    i32 retval;
-};
-
-/** \brief DPDK interface HQoS tctbl entry set request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - the interface
-    @param entry - entry index ID
-    @param tc - traffic class (0 .. 3)
-    @param queue - traffic class queue (0 .. 3)
-*/
-define sw_interface_set_dpdk_hqos_tctbl {
-    u32 client_index;
-    u32 context;
-    u32 sw_if_index;
-    u32 entry;
-    u32 tc;
-    u32 queue;
-};
-
-/** \brief DPDK interface HQoS tctbl entry set reply
-    @param context - sender context, to match reply w/ request
-    @param retval - request return code
-*/
-define sw_interface_set_dpdk_hqos_tctbl_reply {
-    u32 context;
-    i32 retval;
-};
-
-/*
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
- 
\ No newline at end of file
diff --git a/src/vnet/devices/dpdk/dpdk.h b/src/vnet/devices/dpdk/dpdk.h
deleted file mode 100644
index bf9f2768..00000000
--- a/src/vnet/devices/dpdk/dpdk.h
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_dpdk_h__
-#define __included_dpdk_h__
-
-/* $$$$ We should rename always_inline -> clib_always_inline */
-#undef always_inline
-
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-
-#if CLIB_DEBUG > 0
-#define always_inline static inline
-#else
-#define always_inline static inline __attribute__ ((__always_inline__))
-#endif
-
-#include 
-
-#define NB_MBUF   (16<<10)
-
-extern vnet_device_class_t dpdk_device_class;
-extern vlib_node_registration_t dpdk_input_node;
-extern vlib_node_registration_t handoff_dispatch_node;
-
-#define foreach_dpdk_pmd          \
-  _ ("net_thunderx", THUNDERX)    \
-  _ ("net_e1000_em", E1000EM)     \
-  _ ("net_e1000_igb", IGB)        \
-  _ ("net_e1000_igb_vf", IGBVF)   \
-  _ ("net_ixgbe", IXGBE)          \
-  _ ("net_ixgbe_vf", IXGBEVF)     \
-  _ ("net_i40e", I40E)            \
-  _ ("net_i40e_vf", I40EVF)       \
-  _ ("net_virtio", VIRTIO)        \
-  _ ("net_enic", ENIC)            \
-  _ ("net_vmxnet3", VMXNET3)      \
-  _ ("AF_PACKET PMD", AF_PACKET)  \
-  _ ("rte_bond_pmd", BOND)        \
-  _ ("net_fm10k", FM10K)          \
-  _ ("net_cxgbe", CXGBE)          \
-  _ ("net_mlx5", MLX5)            \
-  _ ("net_dpaa2", DPAA2)
-
-typedef enum
-{
-  VNET_DPDK_PMD_NONE,
-#define _(s,f) VNET_DPDK_PMD_##f,
-  foreach_dpdk_pmd
-#undef _
-    VNET_DPDK_PMD_UNKNOWN,	/* must be last */
-} dpdk_pmd_t;
-
-typedef enum
-{
-  VNET_DPDK_PORT_TYPE_ETH_1G,
-  VNET_DPDK_PORT_TYPE_ETH_10G,
-  VNET_DPDK_PORT_TYPE_ETH_40G,
-  VNET_DPDK_PORT_TYPE_ETH_100G,
-  VNET_DPDK_PORT_TYPE_ETH_BOND,
-  VNET_DPDK_PORT_TYPE_ETH_SWITCH,
-  VNET_DPDK_PORT_TYPE_AF_PACKET,
-  VNET_DPDK_PORT_TYPE_UNKNOWN,
-} dpdk_port_type_t;
-
-/*
- * The header for the tx_vector in dpdk_device_t.
- * Head and tail are indexes into the tx_vector and are of type
- * u64 so they never overflow.
- */
-typedef struct
-{
-  u64 tx_head;
-  u64 tx_tail;
-} tx_ring_hdr_t;
-
-typedef struct
-{
-  struct rte_ring *swq;
-
-  u64 hqos_field0_slabmask;
-  u32 hqos_field0_slabpos;
-  u32 hqos_field0_slabshr;
-  u64 hqos_field1_slabmask;
-  u32 hqos_field1_slabpos;
-  u32 hqos_field1_slabshr;
-  u64 hqos_field2_slabmask;
-  u32 hqos_field2_slabpos;
-  u32 hqos_field2_slabshr;
-  u32 hqos_tc_table[64];
-} dpdk_device_hqos_per_worker_thread_t;
-
-typedef struct
-{
-  struct rte_ring **swq;
-  struct rte_mbuf **pkts_enq;
-  struct rte_mbuf **pkts_deq;
-  struct rte_sched_port *hqos;
-  u32 hqos_burst_enq;
-  u32 hqos_burst_deq;
-  u32 pkts_enq_len;
-  u32 swq_pos;
-  u32 flush_count;
-} dpdk_device_hqos_per_hqos_thread_t;
-
-typedef struct
-{
-  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-  volatile u32 **lockp;
-
-  /* Instance ID */
-  u32 device_index;
-
-  u32 vlib_hw_if_index;
-  u32 vlib_sw_if_index;
-
-  /* next node index if we decide to steal the rx graph arc */
-  u32 per_interface_next_index;
-
-  /* dpdk rte_mbuf rx and tx vectors, VLIB_FRAME_SIZE */
-  struct rte_mbuf ***tx_vectors;	/* one per worker thread */
-  struct rte_mbuf ***rx_vectors;
-
-  /* vector of traced contexts, per device */
-  u32 **d_trace_buffers;
-
-  dpdk_pmd_t pmd:8;
-  i8 cpu_socket;
-
-  u16 flags;
-#define DPDK_DEVICE_FLAG_ADMIN_UP           (1 << 0)
-#define DPDK_DEVICE_FLAG_PROMISC            (1 << 1)
-#define DPDK_DEVICE_FLAG_PMD                (1 << 2)
-#define DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE (1 << 3)
-#define DPDK_DEVICE_FLAG_MAYBE_MULTISEG     (1 << 4)
-#define DPDK_DEVICE_FLAG_HAVE_SUBIF         (1 << 5)
-#define DPDK_DEVICE_FLAG_HQOS               (1 << 6)
-
-  u16 nb_tx_desc;
-    CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
-
-  u8 *interface_name_suffix;
-
-  /* number of sub-interfaces */
-  u16 num_subifs;
-
-  /* PMD related */
-  u16 tx_q_used;
-  u16 rx_q_used;
-  u16 nb_rx_desc;
-  u16 *cpu_socket_id_by_queue;
-  struct rte_eth_conf port_conf;
-  struct rte_eth_txconf tx_conf;
-
-  /* HQoS related */
-  dpdk_device_hqos_per_worker_thread_t *hqos_wt;
-  dpdk_device_hqos_per_hqos_thread_t *hqos_ht;
-
-  /* af_packet */
-  u8 af_packet_port_id;
-
-  struct rte_eth_link link;
-  f64 time_last_link_update;
-
-  struct rte_eth_stats stats;
-  struct rte_eth_stats last_stats;
-  struct rte_eth_stats last_cleared_stats;
-  struct rte_eth_xstat *xstats;
-  struct rte_eth_xstat *last_cleared_xstats;
-  f64 time_last_stats_update;
-  dpdk_port_type_t port_type;
-
-  /* mac address */
-  u8 *default_mac_address;
-} dpdk_device_t;
-
-#define DPDK_STATS_POLL_INTERVAL      (10.0)
-#define DPDK_MIN_STATS_POLL_INTERVAL  (0.001)	/* 1msec */
-
-#define DPDK_LINK_POLL_INTERVAL       (3.0)
-#define DPDK_MIN_LINK_POLL_INTERVAL   (0.001)	/* 1msec */
-
-typedef struct
-{
-  u32 device;
-  u16 queue_id;
-} dpdk_device_and_queue_t;
-
-#ifndef DPDK_HQOS_DBG_BYPASS
-#define DPDK_HQOS_DBG_BYPASS 0
-#endif
-
-#ifndef HQOS_FLUSH_COUNT_THRESHOLD
-#define HQOS_FLUSH_COUNT_THRESHOLD              100000
-#endif
-
-typedef struct dpdk_device_config_hqos_t
-{
-  u32 hqos_thread;
-  u32 hqos_thread_valid;
-
-  u32 swq_size;
-  u32 burst_enq;
-  u32 burst_deq;
-
-  u32 pktfield0_slabpos;
-  u32 pktfield1_slabpos;
-  u32 pktfield2_slabpos;
-  u64 pktfield0_slabmask;
-  u64 pktfield1_slabmask;
-  u64 pktfield2_slabmask;
-  u32 tc_table[64];
-
-  struct rte_sched_port_params port;
-  struct rte_sched_subport_params *subport;
-  struct rte_sched_pipe_params *pipe;
-  uint32_t *pipe_map;
-} dpdk_device_config_hqos_t;
-
-int dpdk_hqos_validate_mask (u64 mask, u32 n);
-void dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
-						   hqos, u32 pipe_profile_id);
-void dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos);
-clib_error_t *dpdk_port_setup_hqos (dpdk_device_t * xd,
-				    dpdk_device_config_hqos_t * hqos);
-void dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
-			     struct rte_mbuf **pkts, u32 n_pkts);
-
-#define foreach_dpdk_device_config_item \
-  _ (num_rx_queues) \
-  _ (num_tx_queues) \
-  _ (num_rx_desc) \
-  _ (num_tx_desc) \
-  _ (rss_fn)
-
-typedef struct
-{
-  vlib_pci_addr_t pci_addr;
-  u8 is_blacklisted;
-  u8 vlan_strip_offload;
-#define DPDK_DEVICE_VLAN_STRIP_DEFAULT 0
-#define DPDK_DEVICE_VLAN_STRIP_OFF 1
-#define DPDK_DEVICE_VLAN_STRIP_ON  2
-
-#define _(x) uword x;
-    foreach_dpdk_device_config_item
-#undef _
-    clib_bitmap_t * workers;
-  u32 hqos_enabled;
-  dpdk_device_config_hqos_t hqos;
-} dpdk_device_config_t;
-
-typedef struct
-{
-
-  /* Config stuff */
-  u8 **eal_init_args;
-  u8 *eal_init_args_str;
-  u8 *uio_driver_name;
-  u8 no_multi_seg;
-  u8 enable_tcp_udp_checksum;
-  u8 cryptodev;
-
-  /* Required config parameters */
-  u8 coremask_set_manually;
-  u8 nchannels_set_manually;
-  u32 coremask;
-  u32 nchannels;
-  u32 num_mbufs;
-  u8 num_kni;			/* while kni_init allows u32, port_id in callback fn is only u8 */
-
-  /*
-   * format interface names ala xxxEthernet%d/%d/%d instead of
-   * xxxEthernet%x/%x/%x.
-   */
-  u8 interface_name_format_decimal;
-
-  /* per-device config */
-  dpdk_device_config_t default_devconf;
-  dpdk_device_config_t *dev_confs;
-  uword *device_config_index_by_pci_addr;
-
-} dpdk_config_main_t;
-
-dpdk_config_main_t dpdk_config_main;
-
-typedef struct
-{
-
-  /* Devices */
-  dpdk_device_t *devices;
-  dpdk_device_and_queue_t **devices_by_cpu;
-  dpdk_device_and_queue_t **devices_by_hqos_cpu;
-
-  /* per-thread recycle lists */
-  u32 **recycle;
-
-  /* buffer flags template, configurable to enable/disable tcp / udp cksum */
-  u32 buffer_flags_template;
-
-  /* vlib buffer free list, must be same size as an rte_mbuf */
-  u32 vlib_buffer_free_list_index;
-
-  /* Ethernet input node index */
-  u32 ethernet_input_node_index;
-
-  /* pcap tracing [only works if (CLIB_DEBUG > 0)] */
-  int tx_pcap_enable;
-  pcap_main_t pcap_main;
-  u8 *pcap_filename;
-  u32 pcap_sw_if_index;
-  u32 pcap_pkts_to_capture;
-
-  /* hashes */
-  uword *dpdk_device_by_kni_port_id;
-  uword *vu_sw_if_index_by_listener_fd;
-  uword *vu_sw_if_index_by_sock_fd;
-  u32 *vu_inactive_interfaces_device_index;
-
-  /*
-   * flag indicating that a posted admin up/down
-   * (via post_sw_interface_set_flags) is in progress
-   */
-  u8 admin_up_down_in_progress;
-
-  u8 use_rss;
-
-  /* which cpus are running dpdk-input */
-  int input_cpu_first_index;
-  int input_cpu_count;
-
-  /* which cpus are running I/O TX */
-  int hqos_cpu_first_index;
-  int hqos_cpu_count;
-
-  /* control interval of dpdk link state and stat polling */
-  f64 link_state_poll_interval;
-  f64 stat_poll_interval;
-
-  /* Sleep for this many MS after each device poll */
-  u32 poll_sleep;
-
-  /* convenience */
-  vlib_main_t *vlib_main;
-  vnet_main_t *vnet_main;
-  dpdk_config_main_t *conf;
-
-  /* mempool */
-  struct rte_mempool **pktmbuf_pools;
-} dpdk_main_t;
-
-dpdk_main_t dpdk_main;
-
-typedef struct
-{
-  u32 buffer_index;
-  u16 device_index;
-  u8 queue_index;
-  struct rte_mbuf mb;
-  /* Copy of VLIB buffer; packet data stored in pre_data. */
-  vlib_buffer_t buffer;
-} dpdk_tx_dma_trace_t;
-
-typedef struct
-{
-  u32 buffer_index;
-  u16 device_index;
-  u16 queue_index;
-  struct rte_mbuf mb;
-  vlib_buffer_t buffer;		/* Copy of VLIB buffer; pkt data stored in pre_data. */
-  u8 data[256];			/* First 256 data bytes, used for hexdump */
-} dpdk_rx_dma_trace_t;
-
-void vnet_buffer_needs_dpdk_mb (vlib_buffer_t * b);
-
-clib_error_t *dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address);
-
-clib_error_t *dpdk_set_mc_filter (vnet_hw_interface_t * hi,
-				  struct ether_addr mc_addr_vec[], int naddr);
-
-void dpdk_thread_input (dpdk_main_t * dm, dpdk_device_t * xd);
-
-clib_error_t *dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd);
-
-u32 dpdk_interface_tx_vector (vlib_main_t * vm, u32 dev_instance);
-
-struct rte_mbuf *dpdk_replicate_packet_mb (vlib_buffer_t * b);
-struct rte_mbuf *dpdk_zerocopy_replicate_packet_mb (vlib_buffer_t * b);
-
-#define foreach_dpdk_error						\
-  _(NONE, "no error")							\
-  _(RX_PACKET_ERROR, "Rx packet errors")				\
-  _(RX_BAD_FCS, "Rx bad fcs")						\
-  _(IP_CHECKSUM_ERROR, "Rx ip checksum errors")				\
-  _(RX_ALLOC_FAIL, "rx buf alloc from free list failed")		\
-  _(RX_ALLOC_NO_PHYSMEM, "rx buf alloc failed no physmem")		\
-  _(RX_ALLOC_DROP_PKTS, "rx packets dropped due to alloc error")
-
-typedef enum
-{
-#define _(f,s) DPDK_ERROR_##f,
-  foreach_dpdk_error
-#undef _
-    DPDK_N_ERROR,
-} dpdk_error_t;
-
-int dpdk_set_stat_poll_interval (f64 interval);
-int dpdk_set_link_state_poll_interval (f64 interval);
-void dpdk_update_link_state (dpdk_device_t * xd, f64 now);
-void dpdk_device_lock_init (dpdk_device_t * xd);
-void dpdk_device_lock_free (dpdk_device_t * xd);
-
-void dpdk_rx_trace (dpdk_main_t * dm,
-		    vlib_node_runtime_t * node,
-		    dpdk_device_t * xd,
-		    u16 queue_id, u32 * buffers, uword n_buffers);
-
-#define EFD_OPERATION_LESS_THAN          0
-#define EFD_OPERATION_GREATER_OR_EQUAL   1
-
-format_function_t format_dpdk_device_name;
-format_function_t format_dpdk_device;
-format_function_t format_dpdk_tx_dma_trace;
-format_function_t format_dpdk_rx_dma_trace;
-format_function_t format_dpdk_rte_mbuf;
-format_function_t format_dpdk_rx_rte_mbuf;
-unformat_function_t unformat_socket_mem;
-clib_error_t *unformat_rss_fn (unformat_input_t * input, uword * rss_fn);
-clib_error_t *unformat_hqos (unformat_input_t * input,
-			     dpdk_device_config_hqos_t * hqos);
-
-uword
-admin_up_down_process (vlib_main_t * vm,
-		       vlib_node_runtime_t * rt, vlib_frame_t * f);
-
-#endif /* __included_dpdk_h__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/dpdk_api.c b/src/vnet/devices/dpdk/dpdk_api.c
deleted file mode 100644
index 8faf5c2c..00000000
--- a/src/vnet/devices/dpdk/dpdk_api.c
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- *------------------------------------------------------------------
- * dpdk_api.c - dpdk interface api
- *
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *------------------------------------------------------------------
- */
-
-#include 
-#include 
-
-#if DPDK > 0
-#include 
-#endif
-
-#include 
-
-#define vl_typedefs		/* define message structures */
-#include 
-#undef vl_typedefs
-
-#define vl_endianfun		/* define message structures */
-#include 
-#undef vl_endianfun
-
-/* instantiate all the print functions we know about */
-#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
-#define vl_printfun
-#include 
-#undef vl_printfun
-
-#include 
-
-#define foreach_vpe_api_msg                                               \
-_(SW_INTERFACE_SET_DPDK_HQOS_PIPE, sw_interface_set_dpdk_hqos_pipe)       \
-_(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, sw_interface_set_dpdk_hqos_subport) \
-_(SW_INTERFACE_SET_DPDK_HQOS_TCTBL, sw_interface_set_dpdk_hqos_tctbl)
-
-static void
-  vl_api_sw_interface_set_dpdk_hqos_pipe_t_handler
-  (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp)
-{
-  vl_api_sw_interface_set_dpdk_hqos_pipe_reply_t *rmp;
-  int rv = 0;
-
-#if DPDK > 0
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd;
-
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  u32 subport = ntohl (mp->subport);
-  u32 pipe = ntohl (mp->pipe);
-  u32 profile = ntohl (mp->profile);
-  vnet_hw_interface_t *hw;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  /* hw_if & dpdk device */
-  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
-
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rv = rte_sched_pipe_config (xd->hqos_ht->hqos, subport, pipe, profile);
-
-  BAD_SW_IF_INDEX_LABEL;
-#else
-  clib_warning ("setting HQoS pipe parameters without DPDK not implemented");
-  rv = VNET_API_ERROR_UNIMPLEMENTED;
-#endif /* DPDK */
-
-  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY);
-}
-
-static void
-  vl_api_sw_interface_set_dpdk_hqos_subport_t_handler
-  (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp)
-{
-  vl_api_sw_interface_set_dpdk_hqos_subport_reply_t *rmp;
-  int rv = 0;
-
-#if DPDK > 0
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd;
-  struct rte_sched_subport_params p;
-
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  u32 subport = ntohl (mp->subport);
-  p.tb_rate = ntohl (mp->tb_rate);
-  p.tb_size = ntohl (mp->tb_size);
-  p.tc_rate[0] = ntohl (mp->tc_rate[0]);
-  p.tc_rate[1] = ntohl (mp->tc_rate[1]);
-  p.tc_rate[2] = ntohl (mp->tc_rate[2]);
-  p.tc_rate[3] = ntohl (mp->tc_rate[3]);
-  p.tc_period = ntohl (mp->tc_period);
-
-  vnet_hw_interface_t *hw;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  /* hw_if & dpdk device */
-  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
-
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport, &p);
-
-  BAD_SW_IF_INDEX_LABEL;
-#else
-  clib_warning
-    ("setting HQoS subport parameters without DPDK not implemented");
-  rv = VNET_API_ERROR_UNIMPLEMENTED;
-#endif /* DPDK */
-
-  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY);
-}
-
-static void
-  vl_api_sw_interface_set_dpdk_hqos_tctbl_t_handler
-  (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp)
-{
-  vl_api_sw_interface_set_dpdk_hqos_tctbl_reply_t *rmp;
-  int rv = 0;
-
-#if DPDK > 0
-  dpdk_main_t *dm = &dpdk_main;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_device_t *xd;
-
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  u32 entry = ntohl (mp->entry);
-  u32 tc = ntohl (mp->tc);
-  u32 queue = ntohl (mp->queue);
-  u32 val, i;
-
-  vnet_hw_interface_t *hw;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  /* hw_if & dpdk device */
-  hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index);
-
-  xd = vec_elt_at_index (dm->devices, hw->dev_instance);
-
-  if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
-    {
-      clib_warning ("invalid traffic class !!");
-      rv = VNET_API_ERROR_INVALID_VALUE;
-      goto done;
-    }
-  if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS)
-    {
-      clib_warning ("invalid queue !!");
-      rv = VNET_API_ERROR_INVALID_VALUE;
-      goto done;
-    }
-
-  /* Detect the set of worker threads */
-  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-
-  if (p == 0)
-    {
-      clib_warning ("worker thread registration AWOL !!");
-      rv = VNET_API_ERROR_INVALID_VALUE_2;
-      goto done;
-    }
-
-  vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
-  int worker_thread_first = tr->first_index;
-  int worker_thread_count = tr->count;
-
-  val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue;
-  for (i = 0; i < worker_thread_count; i++)
-    xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val;
-
-  BAD_SW_IF_INDEX_LABEL;
-done:
-#else
-  clib_warning ("setting HQoS DSCP table entry without DPDK not implemented");
-  rv = VNET_API_ERROR_UNIMPLEMENTED;
-#endif /* DPDK */
-
-  REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY);
-}
-
-/*
- * dpdk_api_hookup
- * Add vpe's API message handlers to the table.
- * vlib has alread mapped shared memory and
- * added the client registration handlers.
- * See .../vlib-api/vlibmemory/memclnt_vlib.c:memclnt_process()
- */
-#define vl_msg_name_crc_list
-#include 
-#undef vl_msg_name_crc_list
-
-static void
-setup_message_id_table (api_main_t * am)
-{
-#define _(id,n,crc) vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id);
-  foreach_vl_msg_name_crc_dpdk;
-#undef _
-}
-
-static clib_error_t *
-dpdk_api_hookup (vlib_main_t * vm)
-{
-  api_main_t *am = &api_main;
-
-#define _(N,n)                                                  \
-    vl_msg_api_set_handlers(VL_API_##N, #n,                     \
-                           vl_api_##n##_t_handler,              \
-                           vl_noop_handler,                     \
-                           vl_api_##n##_t_endian,               \
-                           vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
-  foreach_vpe_api_msg;
-#undef _
-
-  /*
-   * Set up the (msg_name, crc, message-id) table
-   */
-  setup_message_id_table (am);
-
-  return 0;
-}
-
-VLIB_API_INIT_FUNCTION (dpdk_api_hookup);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/dpdk_priv.h b/src/vnet/devices/dpdk/dpdk_priv.h
deleted file mode 100644
index dd40ff48..00000000
--- a/src/vnet/devices/dpdk/dpdk_priv.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1)
-#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1))
-
-#define DPDK_NB_RX_DESC_DEFAULT   1024
-#define DPDK_NB_TX_DESC_DEFAULT   1024
-#define DPDK_NB_RX_DESC_VIRTIO    256
-#define DPDK_NB_TX_DESC_VIRTIO    256
-
-#define I40E_DEV_ID_SFP_XL710           0x1572
-#define I40E_DEV_ID_QSFP_A              0x1583
-#define I40E_DEV_ID_QSFP_B              0x1584
-#define I40E_DEV_ID_QSFP_C              0x1585
-#define I40E_DEV_ID_10G_BASE_T          0x1586
-#define I40E_DEV_ID_VF                  0x154C
-
-/* These args appear by themselves */
-#define foreach_eal_double_hyphen_predicate_arg \
-_(no-shconf)                                    \
-_(no-hpet)                                      \
-_(no-huge)                                      \
-_(vmware-tsc-map)
-
-#define foreach_eal_single_hyphen_mandatory_arg \
-_(coremask, c)                                  \
-_(nchannels, n)                                 \
-
-#define foreach_eal_single_hyphen_arg           \
-_(blacklist, b)                                 \
-_(mem-alloc-request, m)                         \
-_(force-ranks, r)
-
-/* These args are preceeded by "--" and followed by a single string */
-#define foreach_eal_double_hyphen_arg           \
-_(huge-dir)                                     \
-_(proc-type)                                    \
-_(file-prefix)                                  \
-_(vdev)
-
-static inline void
-dpdk_get_xstats (dpdk_device_t * xd)
-{
-  int len;
-  if ((len = rte_eth_xstats_get (xd->device_index, NULL, 0)) > 0)
-    {
-      vec_validate (xd->xstats, len - 1);
-      vec_validate (xd->last_cleared_xstats, len - 1);
-
-      len =
-	rte_eth_xstats_get (xd->device_index, xd->xstats,
-			    vec_len (xd->xstats));
-
-      ASSERT (vec_len (xd->xstats) == len);
-      ASSERT (vec_len (xd->last_cleared_xstats) == len);
-
-      _vec_len (xd->xstats) = len;
-      _vec_len (xd->last_cleared_xstats) = len;
-
-    }
-}
-
-
-static inline void
-dpdk_update_counters (dpdk_device_t * xd, f64 now)
-{
-  vlib_simple_counter_main_t *cm;
-  vnet_main_t *vnm = vnet_get_main ();
-  u32 my_cpu = os_get_cpu_number ();
-  u64 rxerrors, last_rxerrors;
-
-  /* only update counters for PMD interfaces */
-  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
-    return;
-
-  xd->time_last_stats_update = now ? now : xd->time_last_stats_update;
-  clib_memcpy (&xd->last_stats, &xd->stats, sizeof (xd->last_stats));
-  rte_eth_stats_get (xd->device_index, &xd->stats);
-
-  /* maybe bump interface rx no buffer counter */
-  if (PREDICT_FALSE (xd->stats.rx_nombuf != xd->last_stats.rx_nombuf))
-    {
-      cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
-			     VNET_INTERFACE_COUNTER_RX_NO_BUF);
-
-      vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
-				     xd->stats.rx_nombuf -
-				     xd->last_stats.rx_nombuf);
-    }
-
-  /* missed pkt counter */
-  if (PREDICT_FALSE (xd->stats.imissed != xd->last_stats.imissed))
-    {
-      cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
-			     VNET_INTERFACE_COUNTER_RX_MISS);
-
-      vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
-				     xd->stats.imissed -
-				     xd->last_stats.imissed);
-    }
-  rxerrors = xd->stats.ierrors;
-  last_rxerrors = xd->last_stats.ierrors;
-
-  if (PREDICT_FALSE (rxerrors != last_rxerrors))
-    {
-      cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
-			     VNET_INTERFACE_COUNTER_RX_ERROR);
-
-      vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
-				     rxerrors - last_rxerrors);
-    }
-
-  dpdk_get_xstats (xd);
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/format.c b/src/vnet/devices/dpdk/format.c
deleted file mode 100644
index 1558630c..00000000
--- a/src/vnet/devices/dpdk/format.c
+++ /dev/null
@@ -1,754 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-
-#include "dpdk_priv.h"
-#include 
-
-#define foreach_dpdk_counter                    \
-  _ (tx_frames_ok, opackets)                    \
-  _ (tx_bytes_ok, obytes)                       \
-  _ (tx_errors, oerrors)                        \
-  _ (rx_frames_ok, ipackets)                    \
-  _ (rx_bytes_ok, ibytes)                       \
-  _ (rx_errors, ierrors)                        \
-  _ (rx_missed, imissed)                        \
-  _ (rx_no_bufs, rx_nombuf)
-
-#define foreach_dpdk_q_counter                  \
-  _ (rx_frames_ok, q_ipackets)                  \
-  _ (tx_frames_ok, q_opackets)                  \
-  _ (rx_bytes_ok, q_ibytes)                     \
-  _ (tx_bytes_ok, q_obytes)                     \
-  _ (rx_errors, q_errors)
-
-#define foreach_dpdk_rss_hf                    \
-  _(ETH_RSS_FRAG_IPV4,          "ipv4-frag")   \
-  _(ETH_RSS_NONFRAG_IPV4_TCP,   "ipv4-tcp")    \
-  _(ETH_RSS_NONFRAG_IPV4_UDP,   "ipv4-udp")    \
-  _(ETH_RSS_NONFRAG_IPV4_SCTP,  "ipv4-sctp")   \
-  _(ETH_RSS_NONFRAG_IPV4_OTHER, "ipv4-other")  \
-  _(ETH_RSS_IPV4,               "ipv4")        \
-  _(ETH_RSS_IPV6_TCP_EX,        "ipv6-tcp-ex") \
-  _(ETH_RSS_IPV6_UDP_EX,        "ipv6-udp-ex") \
-  _(ETH_RSS_FRAG_IPV6,          "ipv6-frag")   \
-  _(ETH_RSS_NONFRAG_IPV6_TCP,   "ipv6-tcp")    \
-  _(ETH_RSS_NONFRAG_IPV6_UDP,   "ipv6-udp")    \
-  _(ETH_RSS_NONFRAG_IPV6_SCTP,  "ipv6-sctp")   \
-  _(ETH_RSS_NONFRAG_IPV6_OTHER, "ipv6-other")  \
-  _(ETH_RSS_L2_PAYLOAD,         "l2-payload")  \
-  _(ETH_RSS_IPV6_EX,            "ipv6-ex")     \
-  _(ETH_RSS_IPV6,               "ipv6")
-
-
-#define foreach_dpdk_rx_offload_caps            \
-  _(DEV_RX_OFFLOAD_VLAN_STRIP, "vlan-strip")    \
-  _(DEV_RX_OFFLOAD_IPV4_CKSUM, "ipv4-cksum")    \
-  _(DEV_RX_OFFLOAD_UDP_CKSUM , "udp-cksum")     \
-  _(DEV_RX_OFFLOAD_TCP_CKSUM , "tcp-cksum")     \
-  _(DEV_RX_OFFLOAD_TCP_LRO   , "rcp-lro")       \
-  _(DEV_RX_OFFLOAD_QINQ_STRIP, "qinq-strip")
-
-#define foreach_dpdk_tx_offload_caps           \
-  _(DEV_TX_OFFLOAD_VLAN_INSERT, "vlan-insert") \
-  _(DEV_TX_OFFLOAD_IPV4_CKSUM,  "ipv4-cksum")  \
-  _(DEV_TX_OFFLOAD_UDP_CKSUM  , "udp-cksum")   \
-  _(DEV_TX_OFFLOAD_TCP_CKSUM  , "tcp-cksum")   \
-  _(DEV_TX_OFFLOAD_SCTP_CKSUM , "sctp-cksum")  \
-  _(DEV_TX_OFFLOAD_TCP_TSO    , "tcp-tso")     \
-  _(DEV_TX_OFFLOAD_UDP_TSO    , "udp-tso")     \
-  _(DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM, "outer-ipv4-cksum") \
-  _(DEV_TX_OFFLOAD_QINQ_INSERT, "qinq-insert")
-
-#define foreach_dpdk_pkt_rx_offload_flag                                \
-  _ (PKT_RX_VLAN_PKT, "RX packet is a 802.1q VLAN packet")              \
-  _ (PKT_RX_RSS_HASH, "RX packet with RSS hash result")                 \
-  _ (PKT_RX_FDIR, "RX packet with FDIR infos")                          \
-  _ (PKT_RX_L4_CKSUM_BAD, "L4 cksum of RX pkt. is not OK")              \
-  _ (PKT_RX_IP_CKSUM_BAD, "IP cksum of RX pkt. is not OK")              \
-  _ (PKT_RX_VLAN_STRIPPED, "RX packet VLAN tag stripped")               \
-  _ (PKT_RX_IP_CKSUM_GOOD, "IP cksum of RX pkt. is valid")              \
-  _ (PKT_RX_L4_CKSUM_GOOD, "L4 cksum of RX pkt. is valid")              \
-  _ (PKT_RX_IEEE1588_PTP, "RX IEEE1588 L2 Ethernet PT Packet")          \
-  _ (PKT_RX_IEEE1588_TMST, "RX IEEE1588 L2/L4 timestamped packet")      \
-  _ (PKT_RX_QINQ_STRIPPED, "RX packet QinQ tags stripped")
-
-#define foreach_dpdk_pkt_type                                           \
-  _ (L2, ETHER, "Ethernet packet")                                      \
-  _ (L2, ETHER_TIMESYNC, "Ethernet packet for time sync")               \
-  _ (L2, ETHER_ARP, "ARP packet")                                       \
-  _ (L2, ETHER_LLDP, "LLDP (Link Layer Discovery Protocol) packet")     \
-  _ (L2, ETHER_NSH, "NSH (Network Service Header) packet")              \
-  _ (L2, ETHER_VLAN, "VLAN packet")                                     \
-  _ (L2, ETHER_QINQ, "QinQ packet")                                     \
-  _ (L3, IPV4, "IPv4 packet without extension headers")                 \
-  _ (L3, IPV4_EXT, "IPv4 packet with extension headers")                \
-  _ (L3, IPV4_EXT_UNKNOWN, "IPv4 packet with or without extension headers") \
-  _ (L3, IPV6, "IPv6 packet without extension headers")                 \
-  _ (L3, IPV6_EXT, "IPv6 packet with extension headers")                \
-  _ (L3, IPV6_EXT_UNKNOWN, "IPv6 packet with or without extension headers") \
-  _ (L4, TCP, "TCP packet")                                             \
-  _ (L4, UDP, "UDP packet")                                             \
-  _ (L4, FRAG, "Fragmented IP packet")                                  \
-  _ (L4, SCTP, "SCTP (Stream Control Transmission Protocol) packet")    \
-  _ (L4, ICMP, "ICMP packet")                                           \
-  _ (L4, NONFRAG, "Non-fragmented IP packet")                           \
-  _ (TUNNEL, GRE, "GRE tunneling packet")                               \
-  _ (TUNNEL, VXLAN, "VXLAN tunneling packet")                           \
-  _ (TUNNEL, NVGRE, "NVGRE Tunneling packet")                           \
-  _ (TUNNEL, GENEVE, "GENEVE Tunneling packet")                         \
-  _ (TUNNEL, GRENAT, "Teredo, VXLAN or GRE Tunneling packet")           \
-  _ (INNER_L2, ETHER, "Inner Ethernet packet")                          \
-  _ (INNER_L2, ETHER_VLAN, "Inner Ethernet packet with VLAN")           \
-  _ (INNER_L3, IPV4, "Inner IPv4 packet without extension headers")     \
-  _ (INNER_L3, IPV4_EXT, "Inner IPv4 packet with extension headers")    \
-  _ (INNER_L3, IPV4_EXT_UNKNOWN, "Inner IPv4 packet with or without extension headers") \
-  _ (INNER_L3, IPV6, "Inner IPv6 packet without extension headers")     \
-  _ (INNER_L3, IPV6_EXT, "Inner IPv6 packet with extension headers")    \
-  _ (INNER_L3, IPV6_EXT_UNKNOWN, "Inner IPv6 packet with or without extension headers") \
-  _ (INNER_L4, TCP, "Inner TCP packet")                                 \
-  _ (INNER_L4, UDP, "Inner UDP packet")                                 \
-  _ (INNER_L4, FRAG, "Inner fagmented IP packet")                       \
-  _ (INNER_L4, SCTP, "Inner SCTP (Stream Control Transmission Protocol) packet") \
-  _ (INNER_L4, ICMP, "Inner ICMP packet")                               \
-  _ (INNER_L4, NONFRAG, "Inner non-fragmented IP packet")
-
-#define foreach_dpdk_pkt_tx_offload_flag                                \
-  _ (PKT_TX_VLAN_PKT, "TX packet is a 802.1q VLAN packet")              \
-  _ (PKT_TX_IP_CKSUM, "IP cksum of TX pkt. computed by NIC")            \
-  _ (PKT_TX_TCP_CKSUM, "TCP cksum of TX pkt. computed by NIC")          \
-  _ (PKT_TX_SCTP_CKSUM, "SCTP cksum of TX pkt. computed by NIC")        \
-  _ (PKT_TX_IEEE1588_TMST, "TX IEEE1588 packet to timestamp")
-
-#define foreach_dpdk_pkt_offload_flag           \
-  foreach_dpdk_pkt_rx_offload_flag              \
-  foreach_dpdk_pkt_tx_offload_flag
-
-u8 *
-format_dpdk_device_name (u8 * s, va_list * args)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  char *devname_format;
-  char *device_name;
-  u32 i = va_arg (*args, u32);
-  struct rte_eth_dev_info dev_info;
-  u8 *ret;
-
-  if (dm->conf->interface_name_format_decimal)
-    devname_format = "%s%d/%d/%d";
-  else
-    devname_format = "%s%x/%x/%x";
-
-  switch (dm->devices[i].port_type)
-    {
-    case VNET_DPDK_PORT_TYPE_ETH_1G:
-      device_name = "GigabitEthernet";
-      break;
-
-    case VNET_DPDK_PORT_TYPE_ETH_10G:
-      device_name = "TenGigabitEthernet";
-      break;
-
-    case VNET_DPDK_PORT_TYPE_ETH_40G:
-      device_name = "FortyGigabitEthernet";
-      break;
-
-    case VNET_DPDK_PORT_TYPE_ETH_100G:
-      device_name = "HundredGigabitEthernet";
-      break;
-
-    case VNET_DPDK_PORT_TYPE_ETH_BOND:
-      return format (s, "BondEthernet%d", dm->devices[i].device_index);
-
-    case VNET_DPDK_PORT_TYPE_ETH_SWITCH:
-      device_name = "EthernetSwitch";
-      break;
-
-    case VNET_DPDK_PORT_TYPE_AF_PACKET:
-      rte_eth_dev_info_get (i, &dev_info);
-      return format (s, "af_packet%d", dm->devices[i].af_packet_port_id);
-
-    default:
-    case VNET_DPDK_PORT_TYPE_UNKNOWN:
-      device_name = "UnknownEthernet";
-      break;
-    }
-
-  rte_eth_dev_info_get (i, &dev_info);
-
-  if (dev_info.pci_dev)
-    ret = format (s, devname_format, device_name, dev_info.pci_dev->addr.bus,
-		  dev_info.pci_dev->addr.devid,
-		  dev_info.pci_dev->addr.function);
-  else
-    ret = format (s, "%s%d", device_name, dm->devices[i].device_index);
-
-  if (dm->devices[i].interface_name_suffix)
-    return format (ret, "/%s", dm->devices[i].interface_name_suffix);
-  return ret;
-}
-
-static u8 *
-format_dpdk_device_type (u8 * s, va_list * args)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  char *dev_type;
-  u32 i = va_arg (*args, u32);
-
-  switch (dm->devices[i].pmd)
-    {
-    case VNET_DPDK_PMD_E1000EM:
-      dev_type = "Intel 82540EM (e1000)";
-      break;
-
-    case VNET_DPDK_PMD_IGB:
-      dev_type = "Intel e1000";
-      break;
-
-    case VNET_DPDK_PMD_I40E:
-      dev_type = "Intel X710/XL710 Family";
-      break;
-
-    case VNET_DPDK_PMD_I40EVF:
-      dev_type = "Intel X710/XL710 Family VF";
-      break;
-
-    case VNET_DPDK_PMD_FM10K:
-      dev_type = "Intel FM10000 Family Ethernet Switch";
-      break;
-
-    case VNET_DPDK_PMD_IGBVF:
-      dev_type = "Intel e1000 VF";
-      break;
-
-    case VNET_DPDK_PMD_VIRTIO:
-      dev_type = "Red Hat Virtio";
-      break;
-
-    case VNET_DPDK_PMD_IXGBEVF:
-      dev_type = "Intel 82599 VF";
-      break;
-
-    case VNET_DPDK_PMD_IXGBE:
-      dev_type = "Intel 82599";
-      break;
-
-    case VNET_DPDK_PMD_ENIC:
-      dev_type = "Cisco VIC";
-      break;
-
-    case VNET_DPDK_PMD_CXGBE:
-      dev_type = "Chelsio T4/T5";
-      break;
-
-    case VNET_DPDK_PMD_MLX5:
-      dev_type = "Mellanox ConnectX-4 Family";
-      break;
-
-    case VNET_DPDK_PMD_VMXNET3:
-      dev_type = "VMware VMXNET3";
-      break;
-
-    case VNET_DPDK_PMD_AF_PACKET:
-      dev_type = "af_packet";
-      break;
-
-    case VNET_DPDK_PMD_BOND:
-      dev_type = "Ethernet Bonding";
-      break;
-
-    case VNET_DPDK_PMD_DPAA2:
-      dev_type = "NXP DPAA2 Mac";
-      break;
-
-    default:
-    case VNET_DPDK_PMD_UNKNOWN:
-      dev_type = "### UNKNOWN ###";
-      break;
-    }
-
-  return format (s, dev_type);
-}
-
-static u8 *
-format_dpdk_link_status (u8 * s, va_list * args)
-{
-  dpdk_device_t *xd = va_arg (*args, dpdk_device_t *);
-  struct rte_eth_link *l = &xd->link;
-  vnet_main_t *vnm = vnet_get_main ();
-  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->vlib_hw_if_index);
-
-  s = format (s, "%s ", l->link_status ? "up" : "down");
-  if (l->link_status)
-    {
-      u32 promisc = rte_eth_promiscuous_get (xd->device_index);
-
-      s = format (s, "%s duplex ", (l->link_duplex == ETH_LINK_FULL_DUPLEX) ?
-		  "full" : "half");
-      s = format (s, "speed %u mtu %d %s\n", l->link_speed,
-		  hi->max_packet_bytes, promisc ? " promisc" : "");
-    }
-  else
-    s = format (s, "\n");
-
-  return s;
-}
-
-#define _line_len 72
-#define _(v, str)                                            \
-if (bitmap & v) {                                            \
-  if (format_get_indent (s) > next_split ) {                 \
-    next_split += _line_len;                                 \
-    s = format(s,"\n%U", format_white_space, indent);        \
-  }                                                          \
-  s = format(s, "%s ", str);                                 \
-}
-
-static u8 *
-format_dpdk_rss_hf_name (u8 * s, va_list * args)
-{
-  u64 bitmap = va_arg (*args, u64);
-  int next_split = _line_len;
-  int indent = format_get_indent (s);
-
-  if (!bitmap)
-    return format (s, "none");
-
-  foreach_dpdk_rss_hf return s;
-}
-
-static u8 *
-format_dpdk_rx_offload_caps (u8 * s, va_list * args)
-{
-  u32 bitmap = va_arg (*args, u32);
-  int next_split = _line_len;
-  int indent = format_get_indent (s);
-
-  if (!bitmap)
-    return format (s, "none");
-
-  foreach_dpdk_rx_offload_caps return s;
-}
-
-static u8 *
-format_dpdk_tx_offload_caps (u8 * s, va_list * args)
-{
-  u32 bitmap = va_arg (*args, u32);
-  int next_split = _line_len;
-  int indent = format_get_indent (s);
-  if (!bitmap)
-    return format (s, "none");
-
-  foreach_dpdk_tx_offload_caps return s;
-}
-
-#undef _line_len
-#undef _
-
-u8 *
-format_dpdk_device (u8 * s, va_list * args)
-{
-  u32 dev_instance = va_arg (*args, u32);
-  int verbose = va_arg (*args, int);
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance);
-  uword indent = format_get_indent (s);
-  f64 now = vlib_time_now (dm->vlib_main);
-  struct rte_eth_dev_info di;
-
-  dpdk_update_counters (xd, now);
-  dpdk_update_link_state (xd, now);
-
-  s = format (s, "%U\n%Ucarrier %U",
-	      format_dpdk_device_type, xd->device_index,
-	      format_white_space, indent + 2, format_dpdk_link_status, xd);
-
-  rte_eth_dev_info_get (xd->device_index, &di);
-
-  if (verbose > 1 && xd->flags & DPDK_DEVICE_FLAG_PMD)
-    {
-      struct rte_pci_device *pci;
-      struct rte_eth_rss_conf rss_conf;
-      int vlan_off;
-      int retval;
-
-      rss_conf.rss_key = 0;
-      retval = rte_eth_dev_rss_hash_conf_get (xd->device_index, &rss_conf);
-      if (retval < 0)
-	clib_warning ("rte_eth_dev_rss_hash_conf_get returned %d", retval);
-      pci = di.pci_dev;
-
-      if (pci)
-	s =
-	  format (s,
-		  "%Upci id:            device %04x:%04x subsystem %04x:%04x\n"
-		  "%Upci address:       %04x:%02x:%02x.%02x\n",
-		  format_white_space, indent + 2, pci->id.vendor_id,
-		  pci->id.device_id, pci->id.subsystem_vendor_id,
-		  pci->id.subsystem_device_id, format_white_space, indent + 2,
-		  pci->addr.domain, pci->addr.bus, pci->addr.devid,
-		  pci->addr.function);
-      s =
-	format (s, "%Umax rx packet len: %d\n", format_white_space,
-		indent + 2, di.max_rx_pktlen);
-      s =
-	format (s, "%Umax num of queues: rx %d tx %d\n", format_white_space,
-		indent + 2, di.max_rx_queues, di.max_tx_queues);
-      s =
-	format (s, "%Upromiscuous:       unicast %s all-multicast %s\n",
-		format_white_space, indent + 2,
-		rte_eth_promiscuous_get (xd->device_index) ? "on" : "off",
-		rte_eth_promiscuous_get (xd->device_index) ? "on" : "off");
-      vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
-      s = format (s, "%Uvlan offload:      strip %s filter %s qinq %s\n",
-		  format_white_space, indent + 2,
-		  vlan_off & ETH_VLAN_STRIP_OFFLOAD ? "on" : "off",
-		  vlan_off & ETH_VLAN_FILTER_OFFLOAD ? "on" : "off",
-		  vlan_off & ETH_VLAN_EXTEND_OFFLOAD ? "on" : "off");
-      s = format (s, "%Urx offload caps:   %U\n",
-		  format_white_space, indent + 2,
-		  format_dpdk_rx_offload_caps, di.rx_offload_capa);
-      s = format (s, "%Utx offload caps:   %U\n",
-		  format_white_space, indent + 2,
-		  format_dpdk_tx_offload_caps, di.tx_offload_capa);
-      s = format (s, "%Urss active:        %U\n"
-		  "%Urss supported:     %U\n",
-		  format_white_space, indent + 2,
-		  format_dpdk_rss_hf_name, rss_conf.rss_hf,
-		  format_white_space, indent + 2,
-		  format_dpdk_rss_hf_name, di.flow_type_rss_offloads);
-    }
-
-  s = format (s, "%Urx queues %d, rx desc %d, tx queues %d, tx desc %d\n",
-	      format_white_space, indent + 2,
-	      xd->rx_q_used, xd->nb_rx_desc, xd->tx_q_used, xd->nb_tx_desc);
-
-  if (xd->cpu_socket > -1)
-    s = format (s, "%Ucpu socket %d\n",
-		format_white_space, indent + 2, xd->cpu_socket);
-
-  /* $$$ MIB counters  */
-  {
-#define _(N, V)							\
-    if ((xd->stats.V - xd->last_cleared_stats.V) != 0) {       \
-      s = format (s, "\n%U%-40U%16Ld",                         \
-                  format_white_space, indent + 2,              \
-                  format_c_identifier, #N,                     \
-                  xd->stats.V - xd->last_cleared_stats.V);     \
-    }                                                          \
-
-    foreach_dpdk_counter
-#undef _
-  }
-
-  u8 *xs = 0;
-  u32 i = 0;
-  struct rte_eth_xstat *xstat, *last_xstat;
-  struct rte_eth_xstat_name *xstat_names = 0;
-  int len = rte_eth_xstats_get_names (xd->device_index, NULL, 0);
-  vec_validate (xstat_names, len - 1);
-  rte_eth_xstats_get_names (xd->device_index, xstat_names, len);
-
-  ASSERT (vec_len (xd->xstats) == vec_len (xd->last_cleared_xstats));
-
-  /* *INDENT-OFF* */
-  vec_foreach_index(i, xd->xstats)
-    {
-      u64 delta = 0;
-      xstat = vec_elt_at_index(xd->xstats, i);
-      last_xstat = vec_elt_at_index(xd->last_cleared_xstats, i);
-
-      delta = xstat->value - last_xstat->value;
-      if (verbose == 2 || (verbose && delta))
-        {
-          /* format_c_identifier doesn't like c strings inside vector */
-          u8 * name = format(0,"%s", xstat_names[i].name);
-          xs = format(xs, "\n%U%-38U%16Ld",
-                      format_white_space, indent + 4,
-                      format_c_identifier, name, delta);
-          vec_free(name);
-        }
-    }
-  /* *INDENT-ON* */
-
-  vec_free (xstat_names);
-
-  if (xs)
-    {
-      s = format (s, "\n%Uextended stats:%v",
-		  format_white_space, indent + 2, xs);
-      vec_free (xs);
-    }
-
-  return s;
-}
-
-u8 *
-format_dpdk_tx_dma_trace (u8 * s, va_list * va)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
-  CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
-  dpdk_tx_dma_trace_t *t = va_arg (*va, dpdk_tx_dma_trace_t *);
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index);
-  uword indent = format_get_indent (s);
-  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
-
-  s = format (s, "%U tx queue %d",
-	      format_vnet_sw_interface_name, vnm, sw, t->queue_index);
-
-  s = format (s, "\n%Ubuffer 0x%x: %U",
-	      format_white_space, indent,
-	      t->buffer_index, format_vlib_buffer, &t->buffer);
-
-  s = format (s, "\n%U%U", format_white_space, indent,
-	      format_ethernet_header_with_length, t->buffer.pre_data,
-	      sizeof (t->buffer.pre_data));
-
-  return s;
-}
-
-u8 *
-format_dpdk_rx_dma_trace (u8 * s, va_list * va)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
-  CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
-  dpdk_rx_dma_trace_t *t = va_arg (*va, dpdk_rx_dma_trace_t *);
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index);
-  format_function_t *f;
-  uword indent = format_get_indent (s);
-  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
-
-  s = format (s, "%U rx queue %d",
-	      format_vnet_sw_interface_name, vnm, sw, t->queue_index);
-
-  s = format (s, "\n%Ubuffer 0x%x: %U",
-	      format_white_space, indent,
-	      t->buffer_index, format_vlib_buffer, &t->buffer);
-
-  s = format (s, "\n%U%U",
-	      format_white_space, indent,
-	      format_dpdk_rte_mbuf, &t->mb, &t->data);
-
-  if (vm->trace_main.verbose)
-    {
-      s = format (s, "\n%UPacket Dump%s", format_white_space, indent + 2,
-		  t->mb.data_len > sizeof (t->data) ? " (truncated)" : "");
-      s = format (s, "\n%U%U", format_white_space, indent + 4,
-		  format_hexdump, &t->data,
-		  t->mb.data_len >
-		  sizeof (t->data) ? sizeof (t->data) : t->mb.data_len);
-    }
-  f = node->format_buffer;
-  if (!f)
-    f = format_hex_bytes;
-  s = format (s, "\n%U%U", format_white_space, indent,
-	      f, t->buffer.pre_data, sizeof (t->buffer.pre_data));
-
-  return s;
-}
-
-
-static inline u8 *
-format_dpdk_pkt_types (u8 * s, va_list * va)
-{
-  u32 *pkt_types = va_arg (*va, u32 *);
-  uword indent __attribute__ ((unused)) = format_get_indent (s) + 2;
-
-  if (!*pkt_types)
-    return s;
-
-  s = format (s, "Packet Types");
-
-#define _(L, F, S)             \
-  if ((*pkt_types & RTE_PTYPE_##L##_MASK) == RTE_PTYPE_##L##_##F)           \
-    {                                                                       \
-      s = format (s, "\n%U%s (0x%04x) %s", format_white_space, indent,      \
-                     "RTE_PTYPE_" #L "_" #F, RTE_PTYPE_##L##_##F, S);       \
-    }
-
-  foreach_dpdk_pkt_type
-#undef _
-    return s;
-}
-
-static inline u8 *
-format_dpdk_pkt_offload_flags (u8 * s, va_list * va)
-{
-  u64 *ol_flags = va_arg (*va, u64 *);
-  uword indent = format_get_indent (s) + 2;
-
-  if (!*ol_flags)
-    return s;
-
-  s = format (s, "Packet Offload Flags");
-
-#define _(F, S)             \
-  if (*ol_flags & F)            \
-    {               \
-      s = format (s, "\n%U%s (0x%04x) %s",      \
-      format_white_space, indent, #F, F, S);  \
-    }
-
-  foreach_dpdk_pkt_offload_flag
-#undef _
-    return s;
-}
-
-u8 *
-format_dpdk_rte_mbuf_vlan (u8 * s, va_list * va)
-{
-  ethernet_vlan_header_tv_t *vlan_hdr =
-    va_arg (*va, ethernet_vlan_header_tv_t *);
-
-  if (clib_net_to_host_u16 (vlan_hdr->type) == ETHERNET_TYPE_DOT1AD)
-    {
-      s = format (s, "%U 802.1q vlan ",
-		  format_ethernet_vlan_tci,
-		  clib_net_to_host_u16 (vlan_hdr->priority_cfi_and_id));
-      vlan_hdr++;
-    }
-
-  s = format (s, "%U",
-	      format_ethernet_vlan_tci,
-	      clib_net_to_host_u16 (vlan_hdr->priority_cfi_and_id));
-
-  return s;
-}
-
-u8 *
-format_dpdk_rte_mbuf (u8 * s, va_list * va)
-{
-  struct rte_mbuf *mb = va_arg (*va, struct rte_mbuf *);
-  ethernet_header_t *eth_hdr = va_arg (*va, ethernet_header_t *);
-  uword indent = format_get_indent (s) + 2;
-
-  s = format (s, "PKT MBUF: port %d, nb_segs %d, pkt_len %d"
-	      "\n%Ubuf_len %d, data_len %d, ol_flags 0x%x, data_off %d, phys_addr 0x%x"
-	      "\n%Upacket_type 0x%x",
-	      mb->port, mb->nb_segs, mb->pkt_len,
-	      format_white_space, indent,
-	      mb->buf_len, mb->data_len, mb->ol_flags, mb->data_off,
-	      mb->buf_physaddr, format_white_space, indent, mb->packet_type);
-
-  if (mb->ol_flags)
-    s = format (s, "\n%U%U", format_white_space, indent,
-		format_dpdk_pkt_offload_flags, &mb->ol_flags);
-
-  if ((mb->ol_flags & PKT_RX_VLAN_PKT) &&
-      ((mb->ol_flags & (PKT_RX_VLAN_STRIPPED | PKT_RX_QINQ_STRIPPED)) == 0))
-    {
-      ethernet_vlan_header_tv_t *vlan_hdr =
-	((ethernet_vlan_header_tv_t *) & (eth_hdr->type));
-      s = format (s, " %U", format_dpdk_rte_mbuf_vlan, vlan_hdr);
-    }
-
-  if (mb->packet_type)
-    s = format (s, "\n%U%U", format_white_space, indent,
-		format_dpdk_pkt_types, &mb->packet_type);
-
-  return s;
-}
-
-/* FIXME is this function used? */
-#if 0
-uword
-unformat_socket_mem (unformat_input_t * input, va_list * va)
-{
-  uword **r = va_arg (*va, uword **);
-  int i = 0;
-  u32 mem;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, ","))
-	hash_set (*r, i, 1024);
-      else if (unformat (input, "%u,", &mem))
-	hash_set (*r, i, mem);
-      else if (unformat (input, "%u", &mem))
-	hash_set (*r, i, mem);
-      else
-	{
-	  unformat_put_input (input);
-	  goto done;
-	}
-      i++;
-    }
-
-done:
-  return 1;
-}
-#endif
-
-clib_error_t *
-unformat_rss_fn (unformat_input_t * input, uword * rss_fn)
-{
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (0)
-	;
-#undef _
-#define _(f, s)                                 \
-      else if (unformat (input, s))             \
-        *rss_fn |= f;
-
-      foreach_dpdk_rss_hf
-#undef _
-	else
-	{
-	  return clib_error_return (0, "unknown input `%U'",
-				    format_unformat_error, input);
-	}
-    }
-  return 0;
-}
-
-clib_error_t *
-unformat_hqos (unformat_input_t * input, dpdk_device_config_hqos_t * hqos)
-{
-  clib_error_t *error = 0;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "hqos-thread %u", &hqos->hqos_thread))
-	hqos->hqos_thread_valid = 1;
-      else
-	{
-	  error = clib_error_return (0, "unknown input `%U'",
-				     format_unformat_error, input);
-	  break;
-	}
-    }
-
-  return error;
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/hqos.c b/src/vnet/devices/dpdk/hqos.c
deleted file mode 100644
index d68bc48f..00000000
--- a/src/vnet/devices/dpdk/hqos.c
+++ /dev/null
@@ -1,775 +0,0 @@
-/*
- * Copyright(c) 2016 Intel Corporation. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 	/* enumerate all vlib messages */
-
-#define vl_typedefs		/* define message structures */
-#include 
-#undef vl_typedefs
-
-/* instantiate all the print functions we know about */
-#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
-#define vl_printfun
-#include 
-#undef vl_printfun
-
-#include "dpdk_priv.h"
-
-dpdk_main_t dpdk_main;
-
-/***
- *
- * HQoS default configuration values
- *
- ***/
-
-static dpdk_device_config_hqos_t hqos_params_default = {
-  .hqos_thread_valid = 0,
-
-  .swq_size = 4096,
-  .burst_enq = 256,
-  .burst_deq = 220,
-
-  /*
-   * Packet field to identify the subport.
-   *
-   * Default value: Since only one subport is defined by default (see below:
-   *     n_subports_per_port = 1), the subport ID is hardcoded to 0.
-   */
-  .pktfield0_slabpos = 0,
-  .pktfield0_slabmask = 0,
-
-  /*
-   * Packet field to identify the pipe.
-   *
-   * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
-   */
-  .pktfield1_slabpos = 40,
-  .pktfield1_slabmask = 0x0000000FFF000000LLU,
-
-  /* Packet field used as index into TC translation table to identify the traffic
-   *     class and queue.
-   *
-   * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
-   */
-  .pktfield2_slabpos = 8,
-  .pktfield2_slabmask = 0x00000000000000FCLLU,
-  .tc_table = {
-	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-	       0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-	       },
-
-  /* port */
-  .port = {
-	   .name = NULL,	/* Set at init */
-	   .socket = 0,		/* Set at init */
-	   .rate = 1250000000,	/* Assuming 10GbE port */
-	   .mtu = 14 + 1500,	/* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
-	   .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
-	   .n_subports_per_port = 1,
-	   .n_pipes_per_subport = 4096,
-	   .qsize = {64, 64, 64, 64},
-	   .pipe_profiles = NULL,	/* Set at config */
-	   .n_pipe_profiles = 1,
-
-#ifdef RTE_SCHED_RED
-	   .red_params = {
-			  /* Traffic Class 0 Colors Green / Yellow / Red */
-			  [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-
-			  /* Traffic Class 1 - Colors Green / Yellow / Red */
-			  [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-
-			  /* Traffic Class 2 - Colors Green / Yellow / Red */
-			  [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-
-			  /* Traffic Class 3 - Colors Green / Yellow / Red */
-			  [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9},
-			  [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
-				    10,.wq_log2 = 9}
-			  },
-#endif /* RTE_SCHED_RED */
-	   },
-};
-
-static struct rte_sched_subport_params hqos_subport_params_default = {
-  .tb_rate = 1250000000,	/* 10GbE line rate (measured in bytes/second) */
-  .tb_size = 1000000,
-  .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
-  .tc_period = 10,
-};
-
-static struct rte_sched_pipe_params hqos_pipe_params_default = {
-  .tb_rate = 305175,		/* 10GbE line rate divided by 4K pipes */
-  .tb_size = 1000000,
-  .tc_rate = {305175, 305175, 305175, 305175},
-  .tc_period = 40,
-#ifdef RTE_SCHED_SUBPORT_TC_OV
-  .tc_ov_weight = 1,
-#endif
-  .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
-};
-
-/***
- *
- * HQoS configuration
- *
- ***/
-
-int
-dpdk_hqos_validate_mask (u64 mask, u32 n)
-{
-  int count = __builtin_popcountll (mask);
-  int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
-  int pos_trail = __builtin_ctzll (mask);
-  int count_expected = __builtin_popcount (n - 1);
-
-  /* Handle the exceptions */
-  if (n == 0)
-    return -1;			/* Error */
-
-  if ((mask == 0) && (n == 1))
-    return 0;			/* OK */
-
-  if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
-    return -2;			/* Error */
-
-  /* Check that mask is contiguous */
-  if ((pos_lead - pos_trail) != count)
-    return -3;			/* Error */
-
-  /* Check that mask contains the expected number of bits set */
-  if (count != count_expected)
-    return -4;			/* Error */
-
-  return 0;			/* OK */
-}
-
-void
-dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
-					      hqos, u32 pipe_profile_id)
-{
-  memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
-	  sizeof (hqos_pipe_params_default));
-}
-
-void
-dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos)
-{
-  struct rte_sched_subport_params *subport_params;
-  struct rte_sched_pipe_params *pipe_params;
-  u32 *pipe_map;
-  u32 i;
-
-  memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
-
-  /* pipe */
-  vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
-
-  for (i = 0; i < vec_len (hqos->pipe); i++)
-    memcpy (&pipe_params[i],
-	    &hqos_pipe_params_default, sizeof (hqos_pipe_params_default));
-
-  hqos->port.pipe_profiles = hqos->pipe;
-
-  /* subport */
-  vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
-
-  for (i = 0; i < vec_len (hqos->subport); i++)
-    memcpy (&subport_params[i],
-	    &hqos_subport_params_default,
-	    sizeof (hqos_subport_params_default));
-
-  /* pipe profile */
-  vec_add2 (hqos->pipe_map,
-	    pipe_map,
-	    hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
-
-  for (i = 0; i < vec_len (hqos->pipe_map); i++)
-    pipe_map[i] = 0;
-}
-
-/***
- *
- * HQoS init
- *
- ***/
-
-clib_error_t *
-dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos)
-{
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  char name[32];
-  u32 subport_id, i;
-  int rv;
-
-  /* Detect the set of worker threads */
-  int worker_thread_first = 0;
-  int worker_thread_count = 0;
-
-  uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-  vlib_thread_registration_t *tr =
-    p ? (vlib_thread_registration_t *) p[0] : 0;
-
-  if (tr && tr->count > 0)
-    {
-      worker_thread_first = tr->first_index;
-      worker_thread_count = tr->count;
-    }
-
-  /* Allocate the per-thread device data array */
-  vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1,
-			CLIB_CACHE_LINE_BYTES);
-  memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
-
-  vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES);
-  memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
-
-  /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
-  vec_validate (xd->hqos_ht->swq, worker_thread_count);
-
-  /* SWQ */
-  for (i = 0; i < worker_thread_count + 1; i++)
-    {
-      u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
-
-      snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
-		xd->device_index);
-      xd->hqos_ht->swq[i] =
-	rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
-      if (xd->hqos_ht->swq[i] == NULL)
-	return clib_error_return (0,
-				  "SWQ-worker%u-to-device%u: rte_ring_create err",
-				  i, xd->device_index);
-    }
-
-  /*
-   * HQoS
-   */
-
-  /* HQoS port */
-  snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
-  hqos->port.name = strdup (name);
-  if (hqos->port.name == NULL)
-    return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
-
-  hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
-  if (hqos->port.socket == SOCKET_ID_ANY)
-    hqos->port.socket = 0;
-
-  xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
-  if (xd->hqos_ht->hqos == NULL)
-    return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
-			      xd->device_index);
-
-  /* HQoS subport */
-  for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
-       subport_id++)
-    {
-      u32 pipe_id;
-
-      rv =
-	rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
-				  &hqos->subport[subport_id]);
-      if (rv)
-	return clib_error_return (0,
-				  "HQoS%u subport %u: rte_sched_subport_config err (%d)",
-				  xd->device_index, subport_id, rv);
-
-      /* HQoS pipe */
-      for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
-	{
-	  u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
-	  u32 profile_id = hqos->pipe_map[pos];
-
-	  rv =
-	    rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
-				   profile_id);
-	  if (rv)
-	    return clib_error_return (0,
-				      "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
-				      xd->device_index, subport_id, pipe_id,
-				      rv);
-	}
-    }
-
-  /* Set up per-thread device data for the I/O TX thread */
-  xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
-  xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
-  vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
-  vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
-  xd->hqos_ht->pkts_enq_len = 0;
-  xd->hqos_ht->swq_pos = 0;
-  xd->hqos_ht->flush_count = 0;
-
-  /* Set up per-thread device data for each worker thread */
-  for (i = 0; i < worker_thread_count + 1; i++)
-    {
-      u32 tid;
-      if (i)
-	tid = worker_thread_first + (i - 1);
-      else
-	tid = i;
-
-      xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
-      xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
-      xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
-      xd->hqos_wt[tid].hqos_field0_slabshr =
-	__builtin_ctzll (hqos->pktfield0_slabmask);
-      xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
-      xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
-      xd->hqos_wt[tid].hqos_field1_slabshr =
-	__builtin_ctzll (hqos->pktfield1_slabmask);
-      xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
-      xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
-      xd->hqos_wt[tid].hqos_field2_slabshr =
-	__builtin_ctzll (hqos->pktfield2_slabmask);
-      memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
-	      sizeof (hqos->tc_table));
-    }
-
-  return 0;
-}
-
-/***
- *
- * HQoS run-time
- *
- ***/
-/*
- * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
- *
- * w
- *     Information for the current thread
- */
-static_always_inline void
-dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  u32 cpu_index = vm->cpu_index;
-  u32 dev_pos;
-
-  dev_pos = 0;
-  while (1)
-    {
-      vlib_worker_thread_barrier_check ();
-
-      u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
-      if (dev_pos >= n_devs)
-	dev_pos = 0;
-
-      dpdk_device_and_queue_t *dq =
-	vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
-      dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
-
-      dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
-      u32 device_index = xd->device_index;
-      u16 queue_id = dq->queue_id;
-
-      struct rte_mbuf **pkts_enq = hqos->pkts_enq;
-      u32 pkts_enq_len = hqos->pkts_enq_len;
-      u32 swq_pos = hqos->swq_pos;
-      u32 n_swq = vec_len (hqos->swq), i;
-      u32 flush_count = hqos->flush_count;
-
-      for (i = 0; i < n_swq; i++)
-	{
-	  /* Get current SWQ for this device */
-	  struct rte_ring *swq = hqos->swq[swq_pos];
-
-	  /* Read SWQ burst to packet buffer of this device */
-	  pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
-						     (void **)
-						     &pkts_enq[pkts_enq_len],
-						     hqos->hqos_burst_enq);
-
-	  /* Get next SWQ for this device */
-	  swq_pos++;
-	  if (swq_pos >= n_swq)
-	    swq_pos = 0;
-	  hqos->swq_pos = swq_pos;
-
-	  /* HWQ TX enqueue when burst available */
-	  if (pkts_enq_len >= hqos->hqos_burst_enq)
-	    {
-	      u32 n_pkts = rte_eth_tx_burst (device_index,
-					     (uint16_t) queue_id,
-					     pkts_enq,
-					     (uint16_t) pkts_enq_len);
-
-	      for (; n_pkts < pkts_enq_len; n_pkts++)
-		rte_pktmbuf_free (pkts_enq[n_pkts]);
-
-	      pkts_enq_len = 0;
-	      flush_count = 0;
-	      break;
-	    }
-	}
-      if (pkts_enq_len)
-	{
-	  flush_count++;
-	  if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
-	    {
-	      rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
-
-	      pkts_enq_len = 0;
-	      flush_count = 0;
-	    }
-	}
-      hqos->pkts_enq_len = pkts_enq_len;
-      hqos->flush_count = flush_count;
-
-      /* Advance to next device */
-      dev_pos++;
-    }
-}
-
-static_always_inline void
-dpdk_hqos_thread_internal (vlib_main_t * vm)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  u32 cpu_index = vm->cpu_index;
-  u32 dev_pos;
-
-  dev_pos = 0;
-  while (1)
-    {
-      vlib_worker_thread_barrier_check ();
-
-      u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
-      if (PREDICT_FALSE (n_devs == 0))
-	{
-	  dev_pos = 0;
-	  continue;
-	}
-      if (dev_pos >= n_devs)
-	dev_pos = 0;
-
-      dpdk_device_and_queue_t *dq =
-	vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
-      dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
-
-      dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
-      u32 device_index = xd->device_index;
-      u16 queue_id = dq->queue_id;
-
-      struct rte_mbuf **pkts_enq = hqos->pkts_enq;
-      struct rte_mbuf **pkts_deq = hqos->pkts_deq;
-      u32 pkts_enq_len = hqos->pkts_enq_len;
-      u32 swq_pos = hqos->swq_pos;
-      u32 n_swq = vec_len (hqos->swq), i;
-      u32 flush_count = hqos->flush_count;
-
-      /*
-       * SWQ dequeue and HQoS enqueue for current device
-       */
-      for (i = 0; i < n_swq; i++)
-	{
-	  /* Get current SWQ for this device */
-	  struct rte_ring *swq = hqos->swq[swq_pos];
-
-	  /* Read SWQ burst to packet buffer of this device */
-	  pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
-						     (void **)
-						     &pkts_enq[pkts_enq_len],
-						     hqos->hqos_burst_enq);
-
-	  /* Get next SWQ for this device */
-	  swq_pos++;
-	  if (swq_pos >= n_swq)
-	    swq_pos = 0;
-	  hqos->swq_pos = swq_pos;
-
-	  /* HQoS enqueue when burst available */
-	  if (pkts_enq_len >= hqos->hqos_burst_enq)
-	    {
-	      rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
-
-	      pkts_enq_len = 0;
-	      flush_count = 0;
-	      break;
-	    }
-	}
-      if (pkts_enq_len)
-	{
-	  flush_count++;
-	  if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
-	    {
-	      rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
-
-	      pkts_enq_len = 0;
-	      flush_count = 0;
-	    }
-	}
-      hqos->pkts_enq_len = pkts_enq_len;
-      hqos->flush_count = flush_count;
-
-      /*
-       * HQoS dequeue and HWQ TX enqueue for current device
-       */
-      {
-	u32 pkts_deq_len, n_pkts;
-
-	pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
-					       pkts_deq,
-					       hqos->hqos_burst_deq);
-
-	for (n_pkts = 0; n_pkts < pkts_deq_len;)
-	  n_pkts += rte_eth_tx_burst (device_index,
-				      (uint16_t) queue_id,
-				      &pkts_deq[n_pkts],
-				      (uint16_t) (pkts_deq_len - n_pkts));
-      }
-
-      /* Advance to next device */
-      dev_pos++;
-    }
-}
-
-void
-dpdk_hqos_thread (vlib_worker_thread_t * w)
-{
-  vlib_main_t *vm;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_main_t *dm = &dpdk_main;
-
-  vm = vlib_get_main ();
-
-  ASSERT (vm->cpu_index == os_get_cpu_number ());
-
-  clib_time_init (&vm->clib_time);
-  clib_mem_set_heap (w->thread_mheap);
-
-  /* Wait until the dpdk init sequence is complete */
-  while (tm->worker_thread_release == 0)
-    vlib_worker_thread_barrier_check ();
-
-  if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0)
-    return
-      clib_error
-      ("current I/O TX thread does not have any devices assigned to it");
-
-  if (DPDK_HQOS_DBG_BYPASS)
-    dpdk_hqos_thread_internal_hqos_dbg_bypass (vm);
-  else
-    dpdk_hqos_thread_internal (vm);
-}
-
-void
-dpdk_hqos_thread_fn (void *arg)
-{
-  vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
-  vlib_worker_thread_init (w);
-  dpdk_hqos_thread (w);
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
-{
-  .name = "hqos-threads",
-  .short_name = "hqos-threads",
-  .function = dpdk_hqos_thread_fn,
-};
-/* *INDENT-ON* */
-
-/*
- * HQoS run-time code to be called by the worker threads
- */
-#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
-({                                                              \
-  u64 slab = *((u64 *) &byte_array[slab_pos]);                  \
-  u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;   \
-  val;                                                          \
-})
-
-#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
-  ((((u64) (queue)) & 0x3) |                               \
-  ((((u64) (traffic_class)) & 0x3) << 2) |                 \
-  ((((u64) (color)) & 0x3) << 4) |                         \
-  ((((u64) (subport)) & 0xFFFF) << 16) |                   \
-  ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
-
-void
-dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
-			struct rte_mbuf **pkts, u32 n_pkts)
-{
-  u32 i;
-
-  for (i = 0; i < (n_pkts & (~0x3)); i += 4)
-    {
-      struct rte_mbuf *pkt0 = pkts[i];
-      struct rte_mbuf *pkt1 = pkts[i + 1];
-      struct rte_mbuf *pkt2 = pkts[i + 2];
-      struct rte_mbuf *pkt3 = pkts[i + 3];
-
-      u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
-      u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
-      u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
-      u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
-
-      u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
-				   hqos->hqos_field0_slabmask,
-				   hqos->hqos_field0_slabshr);
-      u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
-				hqos->hqos_field1_slabmask,
-				hqos->hqos_field1_slabshr);
-      u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
-				hqos->hqos_field2_slabmask,
-				hqos->hqos_field2_slabshr);
-      u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
-      u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
-
-      u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
-				   hqos->hqos_field0_slabmask,
-				   hqos->hqos_field0_slabshr);
-      u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
-				hqos->hqos_field1_slabmask,
-				hqos->hqos_field1_slabshr);
-      u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
-				hqos->hqos_field2_slabmask,
-				hqos->hqos_field2_slabshr);
-      u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
-      u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
-
-      u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
-				   hqos->hqos_field0_slabmask,
-				   hqos->hqos_field0_slabshr);
-      u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
-				hqos->hqos_field1_slabmask,
-				hqos->hqos_field1_slabshr);
-      u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
-				hqos->hqos_field2_slabmask,
-				hqos->hqos_field2_slabshr);
-      u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
-      u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
-
-      u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
-				   hqos->hqos_field0_slabmask,
-				   hqos->hqos_field0_slabshr);
-      u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
-				hqos->hqos_field1_slabmask,
-				hqos->hqos_field1_slabshr);
-      u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
-				hqos->hqos_field2_slabmask,
-				hqos->hqos_field2_slabshr);
-      u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
-      u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
-
-      u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
-						 pkt0_pipe,
-						 pkt0_tc,
-						 pkt0_tc_q,
-						 0);
-      u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
-						 pkt1_pipe,
-						 pkt1_tc,
-						 pkt1_tc_q,
-						 0);
-      u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
-						 pkt2_pipe,
-						 pkt2_tc,
-						 pkt2_tc_q,
-						 0);
-      u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
-						 pkt3_pipe,
-						 pkt3_tc,
-						 pkt3_tc_q,
-						 0);
-
-      pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
-      pkt0->hash.sched.hi = pkt0_sched >> 32;
-      pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
-      pkt1->hash.sched.hi = pkt1_sched >> 32;
-      pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
-      pkt2->hash.sched.hi = pkt2_sched >> 32;
-      pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
-      pkt3->hash.sched.hi = pkt3_sched >> 32;
-    }
-
-  for (; i < n_pkts; i++)
-    {
-      struct rte_mbuf *pkt = pkts[i];
-
-      u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
-
-      u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
-				  hqos->hqos_field0_slabmask,
-				  hqos->hqos_field0_slabshr);
-      u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
-			       hqos->hqos_field1_slabmask,
-			       hqos->hqos_field1_slabshr);
-      u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
-			       hqos->hqos_field2_slabmask,
-			       hqos->hqos_field2_slabshr);
-      u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
-      u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
-
-      u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
-						pkt_pipe,
-						pkt_tc,
-						pkt_tc_q,
-						0);
-
-      pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
-      pkt->hash.sched.hi = pkt_sched >> 32;
-    }
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/init.c b/src/vnet/devices/dpdk/init.c
deleted file mode 100755
index 29423e15..00000000
--- a/src/vnet/devices/dpdk/init.c
+++ /dev/null
@@ -1,1801 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include "dpdk_priv.h"
-
-dpdk_main_t dpdk_main;
-
-/* force linker to link functions used by vlib and declared weak */
-void *vlib_weakly_linked_functions[] = {
-  &rte_pktmbuf_init,
-  &rte_pktmbuf_pool_init,
-};
-
-#define LINK_STATE_ELOGS	0
-
-#define DEFAULT_HUGE_DIR "/run/vpp/hugepages"
-#define VPP_RUN_DIR "/run/vpp"
-
-/* Port configuration, mildly modified Intel app values */
-
-static struct rte_eth_conf port_conf_template = {
-  .rxmode = {
-	     .split_hdr_size = 0,
-	     .header_split = 0,		/**< Header Split disabled */
-	     .hw_ip_checksum = 0,	/**< IP checksum offload disabled */
-	     .hw_vlan_filter = 0,	/**< VLAN filtering disabled */
-	     .hw_strip_crc = 0,		/**< CRC stripped by hardware */
-	     },
-  .txmode = {
-	     .mq_mode = ETH_MQ_TX_NONE,
-	     },
-};
-
-clib_error_t *
-dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd)
-{
-  int rv;
-  int j;
-
-  ASSERT (os_get_cpu_number () == 0);
-
-  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-    {
-      vnet_hw_interface_set_flags (dm->vnet_main, xd->vlib_hw_if_index, 0);
-      rte_eth_dev_stop (xd->device_index);
-    }
-
-  rv = rte_eth_dev_configure (xd->device_index, xd->rx_q_used,
-			      xd->tx_q_used, &xd->port_conf);
-
-  if (rv < 0)
-    return clib_error_return (0, "rte_eth_dev_configure[%d]: err %d",
-			      xd->device_index, rv);
-
-  /* Set up one TX-queue per worker thread */
-  for (j = 0; j < xd->tx_q_used; j++)
-    {
-      rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
-				   xd->cpu_socket, &xd->tx_conf);
-
-      /* retry with any other CPU socket */
-      if (rv < 0)
-	rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc,
-				     SOCKET_ID_ANY, &xd->tx_conf);
-      if (rv < 0)
-	break;
-    }
-
-  if (rv < 0)
-    return clib_error_return (0, "rte_eth_tx_queue_setup[%d]: err %d",
-			      xd->device_index, rv);
-
-  for (j = 0; j < xd->rx_q_used; j++)
-    {
-
-      rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
-				   xd->cpu_socket, 0,
-				   dm->
-				   pktmbuf_pools[xd->cpu_socket_id_by_queue
-						 [j]]);
-
-      /* retry with any other CPU socket */
-      if (rv < 0)
-	rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc,
-				     SOCKET_ID_ANY, 0,
-				     dm->
-				     pktmbuf_pools[xd->cpu_socket_id_by_queue
-						   [j]]);
-      if (rv < 0)
-	return clib_error_return (0, "rte_eth_rx_queue_setup[%d]: err %d",
-				  xd->device_index, rv);
-    }
-
-  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-    {
-      int rv;
-      rv = rte_eth_dev_start (xd->device_index);
-      if (!rv && xd->default_mac_address)
-	rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
-					       (struct ether_addr *)
-					       xd->default_mac_address);
-      if (rv < 0)
-	clib_warning ("rte_eth_dev_start %d returned %d",
-		      xd->device_index, rv);
-    }
-  return 0;
-}
-
-static u32
-dpdk_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
-  u32 old = 0;
-
-  if (ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC (flags))
-    {
-      old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0;
-
-      if (flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL)
-	xd->flags |= DPDK_DEVICE_FLAG_PROMISC;
-      else
-	xd->flags &= ~DPDK_DEVICE_FLAG_PROMISC;
-
-      if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-	{
-	  if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
-	    rte_eth_promiscuous_enable (xd->device_index);
-	  else
-	    rte_eth_promiscuous_disable (xd->device_index);
-	}
-    }
-  else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags))
-    {
-      /*
-       * DAW-FIXME: The Cisco VIC firmware does not provide an api for a
-       *            driver to dynamically change the mtu.  If/when the
-       *            VIC firmware gets fixed, then this should be removed.
-       */
-      if (xd->pmd == VNET_DPDK_PMD_ENIC)
-	{
-	  struct rte_eth_dev_info dev_info;
-
-	  /*
-	   * Restore mtu to what has been set by CIMC in the firmware cfg.
-	   */
-	  rte_eth_dev_info_get (xd->device_index, &dev_info);
-	  hi->max_packet_bytes = dev_info.max_rx_pktlen;
-
-	  vlib_cli_output (vlib_get_main (),
-			   "Cisco VIC mtu can only be changed "
-			   "using CIMC then rebooting the server!");
-	}
-      else
-	{
-	  int rv;
-
-	  xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes;
-
-	  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-	    rte_eth_dev_stop (xd->device_index);
-
-	  rv = rte_eth_dev_configure
-	    (xd->device_index, xd->rx_q_used, xd->tx_q_used, &xd->port_conf);
-
-	  if (rv < 0)
-	    vlib_cli_output (vlib_get_main (),
-			     "rte_eth_dev_configure[%d]: err %d",
-			     xd->device_index, rv);
-
-	  rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
-
-	  if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)
-	    {
-	      int rv = rte_eth_dev_start (xd->device_index);
-	      if (!rv && xd->default_mac_address)
-		rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
-						       (struct ether_addr *)
-						       xd->default_mac_address);
-	      if (rv < 0)
-		clib_warning ("rte_eth_dev_start %d returned %d",
-			      xd->device_index, rv);
-	    }
-	}
-    }
-  return old;
-}
-
-void
-dpdk_device_lock_init (dpdk_device_t * xd)
-{
-  int q;
-  vec_validate (xd->lockp, xd->tx_q_used - 1);
-  for (q = 0; q < xd->tx_q_used; q++)
-    {
-      xd->lockp[q] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
-					     CLIB_CACHE_LINE_BYTES);
-      memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES);
-    }
-}
-
-void
-dpdk_device_lock_free (dpdk_device_t * xd)
-{
-  int q;
-
-  for (q = 0; q < vec_len (xd->lockp); q++)
-    clib_mem_free ((void *) xd->lockp[q]);
-  vec_free (xd->lockp);
-  xd->lockp = 0;
-}
-
-static clib_error_t *
-dpdk_lib_init (dpdk_main_t * dm)
-{
-  u32 nports;
-  u32 nb_desc = 0;
-  int i;
-  clib_error_t *error;
-  vlib_main_t *vm = vlib_get_main ();
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  vnet_sw_interface_t *sw;
-  vnet_hw_interface_t *hi;
-  dpdk_device_t *xd;
-  vlib_pci_addr_t last_pci_addr;
-  u32 last_pci_addr_port = 0;
-  vlib_thread_registration_t *tr, *tr_hqos;
-  uword *p, *p_hqos;
-
-  u32 next_cpu = 0, next_hqos_cpu = 0;
-  u8 af_packet_port_id = 0;
-  last_pci_addr.as_u32 = ~0;
-
-  dm->input_cpu_first_index = 0;
-  dm->input_cpu_count = 1;
-
-  /* find out which cpus will be used for input */
-  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
-
-  if (tr && tr->count > 0)
-    {
-      dm->input_cpu_first_index = tr->first_index;
-      dm->input_cpu_count = tr->count;
-    }
-
-  vec_validate_aligned (dm->devices_by_cpu, tm->n_vlib_mains - 1,
-			CLIB_CACHE_LINE_BYTES);
-
-  dm->hqos_cpu_first_index = 0;
-  dm->hqos_cpu_count = 0;
-
-  /* find out which cpus will be used for I/O TX */
-  p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads");
-  tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0;
-
-  if (tr_hqos && tr_hqos->count > 0)
-    {
-      dm->hqos_cpu_first_index = tr_hqos->first_index;
-      dm->hqos_cpu_count = tr_hqos->count;
-    }
-
-  vec_validate_aligned (dm->devices_by_hqos_cpu, tm->n_vlib_mains - 1,
-			CLIB_CACHE_LINE_BYTES);
-
-  nports = rte_eth_dev_count ();
-  if (nports < 1)
-    {
-      clib_warning ("DPDK drivers found no ports...");
-    }
-
-  if (CLIB_DEBUG > 0)
-    clib_warning ("DPDK drivers found %d ports...", nports);
-
-  /*
-   * All buffers are all allocated from the same rte_mempool.
-   * Thus they all have the same number of data bytes.
-   */
-  dm->vlib_buffer_free_list_index =
-    vlib_buffer_get_or_create_free_list (vm,
-					 VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
-					 "dpdk rx");
-
-  if (dm->conf->enable_tcp_udp_checksum)
-    dm->buffer_flags_template &= ~(IP_BUFFER_L4_CHECKSUM_CORRECT
-				   | IP_BUFFER_L4_CHECKSUM_COMPUTED);
-
-  for (i = 0; i < nports; i++)
-    {
-      u8 addr[6];
-      u8 vlan_strip = 0;
-      int j;
-      struct rte_eth_dev_info dev_info;
-      clib_error_t *rv;
-      struct rte_eth_link l;
-      dpdk_device_config_t *devconf = 0;
-      vlib_pci_addr_t pci_addr;
-      uword *p = 0;
-
-      rte_eth_dev_info_get (i, &dev_info);
-      if (dev_info.pci_dev)	/* bonded interface has no pci info */
-	{
-	  pci_addr.domain = dev_info.pci_dev->addr.domain;
-	  pci_addr.bus = dev_info.pci_dev->addr.bus;
-	  pci_addr.slot = dev_info.pci_dev->addr.devid;
-	  pci_addr.function = dev_info.pci_dev->addr.function;
-	  p =
-	    hash_get (dm->conf->device_config_index_by_pci_addr,
-		      pci_addr.as_u32);
-	}
-
-      if (p)
-	devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]);
-      else
-	devconf = &dm->conf->default_devconf;
-
-      /* Create vnet interface */
-      vec_add2_aligned (dm->devices, xd, 1, CLIB_CACHE_LINE_BYTES);
-      xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT;
-      xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT;
-      xd->cpu_socket = (i8) rte_eth_dev_socket_id (i);
-
-      /* Handle interface naming for devices with multiple ports sharing same PCI ID */
-      if (dev_info.pci_dev)
-	{
-	  struct rte_eth_dev_info di = { 0 };
-	  rte_eth_dev_info_get (i + 1, &di);
-	  if (di.pci_dev && pci_addr.as_u32 != last_pci_addr.as_u32 &&
-	      memcmp (&dev_info.pci_dev->addr, &di.pci_dev->addr,
-		      sizeof (struct rte_pci_addr)) == 0)
-	    {
-	      xd->interface_name_suffix = format (0, "0");
-	      last_pci_addr.as_u32 = pci_addr.as_u32;
-	      last_pci_addr_port = i;
-	    }
-	  else if (pci_addr.as_u32 == last_pci_addr.as_u32)
-	    {
-	      xd->interface_name_suffix =
-		format (0, "%u", i - last_pci_addr_port);
-	    }
-	  else
-	    {
-	      last_pci_addr.as_u32 = ~0;
-	    }
-	}
-      else
-	last_pci_addr.as_u32 = ~0;
-
-      clib_memcpy (&xd->tx_conf, &dev_info.default_txconf,
-		   sizeof (struct rte_eth_txconf));
-      if (dm->conf->no_multi_seg)
-	{
-	  xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
-	  port_conf_template.rxmode.jumbo_frame = 0;
-	}
-      else
-	{
-	  xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS;
-	  port_conf_template.rxmode.jumbo_frame = 1;
-	  xd->flags |= DPDK_DEVICE_FLAG_MAYBE_MULTISEG;
-	}
-
-      clib_memcpy (&xd->port_conf, &port_conf_template,
-		   sizeof (struct rte_eth_conf));
-
-      xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains);
-
-      if (devconf->num_tx_queues > 0
-	  && devconf->num_tx_queues < xd->tx_q_used)
-	xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues);
-
-      if (devconf->num_rx_queues > 1 && dm->use_rss == 0)
-	{
-	  dm->use_rss = 1;
-	}
-
-      if (devconf->num_rx_queues > 1
-	  && dev_info.max_rx_queues >= devconf->num_rx_queues)
-	{
-	  xd->rx_q_used = devconf->num_rx_queues;
-	  xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
-	  if (devconf->rss_fn == 0)
-	    xd->port_conf.rx_adv_conf.rss_conf.rss_hf =
-	      ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
-	  else
-	    xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn;
-	}
-      else
-	xd->rx_q_used = 1;
-
-      xd->flags |= DPDK_DEVICE_FLAG_PMD;
-
-      /* workaround for drivers not setting driver_name */
-      if ((!dev_info.driver_name) && (dev_info.pci_dev))
-	dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
-
-      ASSERT (dev_info.driver_name);
-
-      if (!xd->pmd)
-	{
-
-
-#define _(s,f) else if (dev_info.driver_name &&                 \
-                        !strcmp(dev_info.driver_name, s))       \
-                 xd->pmd = VNET_DPDK_PMD_##f;
-	  if (0)
-	    ;
-	  foreach_dpdk_pmd
-#undef _
-	    else
-	    xd->pmd = VNET_DPDK_PMD_UNKNOWN;
-
-	  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
-	  xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT;
-	  xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT;
-
-	  switch (xd->pmd)
-	    {
-	      /* 1G adapters */
-	    case VNET_DPDK_PMD_E1000EM:
-	    case VNET_DPDK_PMD_IGB:
-	    case VNET_DPDK_PMD_IGBVF:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G;
-	      break;
-
-	      /* 10G adapters */
-	    case VNET_DPDK_PMD_IXGBE:
-	    case VNET_DPDK_PMD_IXGBEVF:
-	    case VNET_DPDK_PMD_THUNDERX:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-	      break;
-	    case VNET_DPDK_PMD_DPAA2:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-	      break;
-
-	      /* Cisco VIC */
-	    case VNET_DPDK_PMD_ENIC:
-	      rte_eth_link_get_nowait (i, &l);
-	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
-	      if (l.link_speed == 40000)
-		xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-	      else
-		xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-	      break;
-
-	      /* Intel Fortville */
-	    case VNET_DPDK_PMD_I40E:
-	    case VNET_DPDK_PMD_I40EVF:
-	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-
-	      switch (dev_info.pci_dev->id.device_id)
-		{
-		case I40E_DEV_ID_10G_BASE_T:
-		case I40E_DEV_ID_SFP_XL710:
-		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-		  break;
-		case I40E_DEV_ID_QSFP_A:
-		case I40E_DEV_ID_QSFP_B:
-		case I40E_DEV_ID_QSFP_C:
-		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-		  break;
-		case I40E_DEV_ID_VF:
-		  rte_eth_link_get_nowait (i, &l);
-		  xd->port_type = l.link_speed == 10000 ?
-		    VNET_DPDK_PORT_TYPE_ETH_10G : VNET_DPDK_PORT_TYPE_ETH_40G;
-		  break;
-		default:
-		  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
-		}
-	      break;
-
-	    case VNET_DPDK_PMD_CXGBE:
-	      switch (dev_info.pci_dev->id.device_id)
-		{
-		case 0x540d:	/* T580-CR */
-		case 0x5410:	/* T580-LP-cr */
-		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-		  break;
-		case 0x5403:	/* T540-CR */
-		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-		  break;
-		default:
-		  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
-		}
-	      break;
-
-	    case VNET_DPDK_PMD_MLX5:
-	      {
-		char *pn_100g[] = { "MCX415A-CCAT", "MCX416A-CCAT", 0 };
-		char *pn_40g[] = { "MCX413A-BCAT", "MCX414A-BCAT",
-		  "MCX415A-BCAT", "MCX416A-BCAT", "MCX4131A-BCAT", 0
-		};
-		char *pn_10g[] = { "MCX4111A-XCAT", "MCX4121A-XCAT", 0 };
-
-		vlib_pci_device_t *pd = vlib_get_pci_device (&pci_addr);
-		u8 *pn = 0;
-		char **c;
-		int found = 0;
-		pn = format (0, "%U%c",
-			     format_vlib_pci_vpd, pd->vpd_r, "PN", 0);
-
-		if (!pn)
-		  break;
-
-		c = pn_100g;
-		while (!found && c[0])
-		  {
-		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
-		      {
-			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_100G;
-			break;
-		      }
-		    c++;
-		  }
-
-		c = pn_40g;
-		while (!found && c[0])
-		  {
-		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
-		      {
-			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-			break;
-		      }
-		    c++;
-		  }
-
-		c = pn_10g;
-		while (!found && c[0])
-		  {
-		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
-		      {
-			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-			break;
-		      }
-		    c++;
-		  }
-
-		vec_free (pn);
-	      }
-
-	      break;
-	      /* Intel Red Rock Canyon */
-	    case VNET_DPDK_PMD_FM10K:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_SWITCH;
-	      break;
-
-	      /* virtio */
-	    case VNET_DPDK_PMD_VIRTIO:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G;
-	      xd->nb_rx_desc = DPDK_NB_RX_DESC_VIRTIO;
-	      xd->nb_tx_desc = DPDK_NB_TX_DESC_VIRTIO;
-	      break;
-
-	      /* vmxnet3 */
-	    case VNET_DPDK_PMD_VMXNET3:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G;
-	      xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
-	      break;
-
-	    case VNET_DPDK_PMD_AF_PACKET:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_AF_PACKET;
-	      xd->af_packet_port_id = af_packet_port_id++;
-	      break;
-
-	    case VNET_DPDK_PMD_BOND:
-	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
-	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_BOND;
-	      break;
-
-	    default:
-	      xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
-	    }
-
-	  if (devconf->num_rx_desc)
-	    xd->nb_rx_desc = devconf->num_rx_desc;
-
-	  if (devconf->num_tx_desc)
-	    xd->nb_tx_desc = devconf->num_tx_desc;
-	}
-
-      /*
-       * Ensure default mtu is not > the mtu read from the hardware.
-       * Otherwise rte_eth_dev_configure() will fail and the port will
-       * not be available.
-       */
-      if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen)
-	{
-	  /*
-	   * This device does not support the platforms's max frame
-	   * size. Use it's advertised mru instead.
-	   */
-	  xd->port_conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen;
-	}
-      else
-	{
-	  xd->port_conf.rxmode.max_rx_pkt_len = ETHERNET_MAX_PACKET_BYTES;
-
-	  /*
-	   * Some platforms do not account for Ethernet FCS (4 bytes) in
-	   * MTU calculations. To interop with them increase mru but only
-	   * if the device's settings can support it.
-	   */
-	  if ((dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)) &&
-	      xd->port_conf.rxmode.hw_strip_crc)
-	    {
-	      /*
-	       * Allow additional 4 bytes (for Ethernet FCS). These bytes are
-	       * stripped by h/w and so will not consume any buffer memory.
-	       */
-	      xd->port_conf.rxmode.max_rx_pkt_len += 4;
-	    }
-	}
-
-      if (xd->pmd == VNET_DPDK_PMD_AF_PACKET)
-	{
-	  f64 now = vlib_time_now (vm);
-	  u32 rnd;
-	  rnd = (u32) (now * 1e6);
-	  rnd = random_u32 (&rnd);
-	  clib_memcpy (addr + 2, &rnd, sizeof (rnd));
-	  addr[0] = 2;
-	  addr[1] = 0xfe;
-	}
-      else
-	rte_eth_macaddr_get (i, (struct ether_addr *) addr);
-
-      if (xd->tx_q_used < tm->n_vlib_mains)
-	dpdk_device_lock_init (xd);
-
-      xd->device_index = xd - dm->devices;
-      ASSERT (i == xd->device_index);
-      xd->per_interface_next_index = ~0;
-
-      /* assign interface to input thread */
-      dpdk_device_and_queue_t *dq;
-      int q;
-
-      if (devconf->workers)
-	{
-	  int i;
-	  q = 0;
-	  /* *INDENT-OFF* */
-	  clib_bitmap_foreach (i, devconf->workers, ({
-	    int cpu = dm->input_cpu_first_index + i;
-	    unsigned lcore = vlib_worker_threads[cpu].lcore_id;
-	    vec_validate(xd->cpu_socket_id_by_queue, q);
-	    xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id(lcore);
-	    vec_add2(dm->devices_by_cpu[cpu], dq, 1);
-	    dq->device = xd->device_index;
-	    dq->queue_id = q++;
-	  }));
-	  /* *INDENT-ON* */
-	}
-      else
-	for (q = 0; q < xd->rx_q_used; q++)
-	  {
-	    int cpu = dm->input_cpu_first_index + next_cpu;
-	    unsigned lcore = vlib_worker_threads[cpu].lcore_id;
-
-	    /*
-	     * numa node for worker thread handling this queue
-	     * needed for taking buffers from the right mempool
-	     */
-	    vec_validate (xd->cpu_socket_id_by_queue, q);
-	    xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id (lcore);
-
-	    /*
-	     * construct vector of (device,queue) pairs for each worker thread
-	     */
-	    vec_add2 (dm->devices_by_cpu[cpu], dq, 1);
-	    dq->device = xd->device_index;
-	    dq->queue_id = q;
-
-	    next_cpu++;
-	    if (next_cpu == dm->input_cpu_count)
-	      next_cpu = 0;
-	  }
-
-
-      if (devconf->hqos_enabled)
-	{
-	  xd->flags |= DPDK_DEVICE_FLAG_HQOS;
-
-	  if (devconf->hqos.hqos_thread_valid)
-	    {
-	      int cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread;
-
-	      if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count)
-		return clib_error_return (0, "invalid HQoS thread index");
-
-	      vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
-	      dq->device = xd->device_index;
-	      dq->queue_id = 0;
-	    }
-	  else
-	    {
-	      int cpu = dm->hqos_cpu_first_index + next_hqos_cpu;
-
-	      if (dm->hqos_cpu_count == 0)
-		return clib_error_return (0, "no HQoS threads available");
-
-	      vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1);
-	      dq->device = xd->device_index;
-	      dq->queue_id = 0;
-
-	      next_hqos_cpu++;
-	      if (next_hqos_cpu == dm->hqos_cpu_count)
-		next_hqos_cpu = 0;
-
-	      devconf->hqos.hqos_thread_valid = 1;
-	      devconf->hqos.hqos_thread = cpu;
-	    }
-	}
-
-      vec_validate_aligned (xd->tx_vectors, tm->n_vlib_mains,
-			    CLIB_CACHE_LINE_BYTES);
-      for (j = 0; j < tm->n_vlib_mains; j++)
-	{
-	  vec_validate_ha (xd->tx_vectors[j], xd->nb_tx_desc,
-			   sizeof (tx_ring_hdr_t), CLIB_CACHE_LINE_BYTES);
-	  vec_reset_length (xd->tx_vectors[j]);
-	}
-
-      vec_validate_aligned (xd->rx_vectors, xd->rx_q_used,
-			    CLIB_CACHE_LINE_BYTES);
-      for (j = 0; j < xd->rx_q_used; j++)
-	{
-	  vec_validate_aligned (xd->rx_vectors[j], VLIB_FRAME_SIZE - 1,
-				CLIB_CACHE_LINE_BYTES);
-	  vec_reset_length (xd->rx_vectors[j]);
-	}
-
-      vec_validate_aligned (xd->d_trace_buffers, tm->n_vlib_mains,
-			    CLIB_CACHE_LINE_BYTES);
-
-      rv = dpdk_port_setup (dm, xd);
-
-      if (rv)
-	return rv;
-
-      if (devconf->hqos_enabled)
-	{
-	  rv = dpdk_port_setup_hqos (xd, &devconf->hqos);
-	  if (rv)
-	    return rv;
-	}
-
-      /* count the number of descriptors used for this device */
-      nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
-
-      error = ethernet_register_interface
-	(dm->vnet_main, dpdk_device_class.index, xd->device_index,
-	 /* ethernet address */ addr,
-	 &xd->vlib_hw_if_index, dpdk_flag_change);
-      if (error)
-	return error;
-
-      sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->vlib_hw_if_index);
-      xd->vlib_sw_if_index = sw->sw_if_index;
-      hi = vnet_get_hw_interface (dm->vnet_main, xd->vlib_hw_if_index);
-
-      /*
-       * DAW-FIXME: The Cisco VIC firmware does not provide an api for a
-       *            driver to dynamically change the mtu.  If/when the
-       *            VIC firmware gets fixed, then this should be removed.
-       */
-      if (xd->pmd == VNET_DPDK_PMD_ENIC)
-	{
-	  /*
-	   * Initialize mtu to what has been set by CIMC in the firmware cfg.
-	   */
-	  hi->max_packet_bytes = dev_info.max_rx_pktlen;
-	  if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF)
-	    vlan_strip = 1;	/* remove vlan tag from VIC port by default */
-	  else
-	    clib_warning ("VLAN strip disabled for interface\n");
-	}
-      else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON)
-	vlan_strip = 1;
-
-      if (vlan_strip)
-	{
-	  int vlan_off;
-	  vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index);
-	  vlan_off |= ETH_VLAN_STRIP_OFFLOAD;
-	  xd->port_conf.rxmode.hw_vlan_strip = vlan_off;
-	  if (rte_eth_dev_set_vlan_offload (xd->device_index, vlan_off) == 0)
-	    clib_warning ("VLAN strip enabled for interface\n");
-	  else
-	    clib_warning ("VLAN strip cannot be supported by interface\n");
-	}
-
-      hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] =
-	xd->port_conf.rxmode.max_rx_pkt_len - sizeof (ethernet_header_t);
-
-      rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes);
-    }
-
-  if (nb_desc > dm->conf->num_mbufs)
-    clib_warning ("%d mbufs allocated but total rx/tx ring size is %d\n",
-		  dm->conf->num_mbufs, nb_desc);
-
-  return 0;
-}
-
-static void
-dpdk_bind_devices_to_uio (dpdk_config_main_t * conf)
-{
-  vlib_pci_main_t *pm = &pci_main;
-  clib_error_t *error;
-  vlib_pci_device_t *d;
-  u8 *pci_addr = 0;
-  int num_whitelisted = vec_len (conf->dev_confs);
-
-  /* *INDENT-OFF* */
-  pool_foreach (d, pm->pci_devs, ({
-    dpdk_device_config_t * devconf = 0;
-    vec_reset_length (pci_addr);
-    pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, &d->bus_address, 0);
-
-    if (d->device_class != PCI_CLASS_NETWORK_ETHERNET && d->device_class != PCI_CLASS_PROCESSOR_CO)
-      continue;
-
-    if (num_whitelisted)
-      {
-	uword * p = hash_get (conf->device_config_index_by_pci_addr, d->bus_address.as_u32);
-
-	if (!p)
-	  continue;
-
-	devconf = pool_elt_at_index (conf->dev_confs, p[0]);
-      }
-
-    /* virtio */
-    if (d->vendor_id == 0x1af4 && d->device_id == 0x1000)
-      ;
-    /* vmxnet3 */
-    else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0)
-      ;
-    /* all Intel devices */
-    else if (d->vendor_id == 0x8086)
-      ;
-    /* Cisco VIC */
-    else if (d->vendor_id == 0x1137 && d->device_id == 0x0043)
-      ;
-    /* Chelsio T4/T5 */
-    else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000)
-      ;
-    else
-      {
-        clib_warning ("Unsupported Ethernet PCI device 0x%04x:0x%04x found "
-		      "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id,
-		      pci_addr);
-        continue;
-      }
-
-    error = vlib_pci_bind_to_uio (d, (char *) conf->uio_driver_name);
-
-    if (error)
-      {
-	if (devconf == 0)
-	  {
-	    pool_get (conf->dev_confs, devconf);
-	    hash_set (conf->device_config_index_by_pci_addr, d->bus_address.as_u32,
-		      devconf - conf->dev_confs);
-	    devconf->pci_addr.as_u32 = d->bus_address.as_u32;
-	  }
-	devconf->is_blacklisted = 1;
-	clib_error_report (error);
-      }
-  }));
-  /* *INDENT-ON* */
-  vec_free (pci_addr);
-}
-
-static clib_error_t *
-dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr,
-		    unformat_input_t * input, u8 is_default)
-{
-  clib_error_t *error = 0;
-  uword *p;
-  dpdk_device_config_t *devconf;
-  unformat_input_t sub_input;
-
-  if (is_default)
-    {
-      devconf = &conf->default_devconf;
-    }
-  else
-    {
-      p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32);
-
-      if (!p)
-	{
-	  pool_get (conf->dev_confs, devconf);
-	  hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32,
-		    devconf - conf->dev_confs);
-	}
-      else
-	return clib_error_return (0,
-				  "duplicate configuration for PCI address %U",
-				  format_vlib_pci_addr, &pci_addr);
-    }
-
-  devconf->pci_addr.as_u32 = pci_addr.as_u32;
-  devconf->hqos_enabled = 0;
-  dpdk_device_config_hqos_default (&devconf->hqos);
-
-  if (!input)
-    return 0;
-
-  unformat_skip_white_space (input);
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues))
-	;
-      else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues))
-	;
-      else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc))
-	;
-      else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc))
-	;
-      else if (unformat (input, "workers %U", unformat_bitmap_list,
-			 &devconf->workers))
-	;
-      else
-	if (unformat
-	    (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input))
-	{
-	  error = unformat_rss_fn (&sub_input, &devconf->rss_fn);
-	  if (error)
-	    break;
-	}
-      else if (unformat (input, "vlan-strip-offload off"))
-	devconf->vlan_strip_offload = DPDK_DEVICE_VLAN_STRIP_OFF;
-      else if (unformat (input, "vlan-strip-offload on"))
-	devconf->vlan_strip_offload = DPDK_DEVICE_VLAN_STRIP_ON;
-      else
-	if (unformat
-	    (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input))
-	{
-	  devconf->hqos_enabled = 1;
-	  error = unformat_hqos (&sub_input, &devconf->hqos);
-	  if (error)
-	    break;
-	}
-      else if (unformat (input, "hqos"))
-	{
-	  devconf->hqos_enabled = 1;
-	}
-      else
-	{
-	  error = clib_error_return (0, "unknown input `%U'",
-				     format_unformat_error, input);
-	  break;
-	}
-    }
-
-  if (error)
-    return error;
-
-  if (devconf->workers && devconf->num_rx_queues == 0)
-    devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers);
-  else if (devconf->workers &&
-	   clib_bitmap_count_set_bits (devconf->workers) !=
-	   devconf->num_rx_queues)
-    error =
-      clib_error_return (0,
-			 "%U: number of worker threadds must be "
-			 "equal to number of rx queues", format_vlib_pci_addr,
-			 &pci_addr);
-
-  return error;
-}
-
-static clib_error_t *
-dpdk_config (vlib_main_t * vm, unformat_input_t * input)
-{
-  clib_error_t *error = 0;
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_config_main_t *conf = &dpdk_config_main;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  dpdk_device_config_t *devconf;
-  vlib_pci_addr_t pci_addr;
-  unformat_input_t sub_input;
-  u8 *s, *tmp = 0;
-  u8 *rte_cmd = 0, *ethname = 0;
-  u32 log_level;
-  int ret, i;
-  int num_whitelisted = 0;
-  u8 no_pci = 0;
-  u8 no_huge = 0;
-  u8 huge_dir = 0;
-  u8 file_prefix = 0;
-  u8 *socket_mem = 0;
-
-  conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      /* Prime the pump */
-      if (unformat (input, "no-hugetlb"))
-	{
-	  vec_add1 (conf->eal_init_args, (u8 *) "no-huge");
-	  no_huge = 1;
-	}
-
-      else if (unformat (input, "enable-tcp-udp-checksum"))
-	conf->enable_tcp_udp_checksum = 1;
-
-      else if (unformat (input, "decimal-interface-names"))
-	conf->interface_name_format_decimal = 1;
-
-      else if (unformat (input, "no-multi-seg"))
-	conf->no_multi_seg = 1;
-
-      else if (unformat (input, "enable-cryptodev"))
-	conf->cryptodev = 1;
-
-      else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
-			 &sub_input))
-	{
-	  error =
-	    dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input,
-				1);
-
-	  if (error)
-	    return error;
-	}
-      else
-	if (unformat
-	    (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr,
-	     unformat_vlib_cli_sub_input, &sub_input))
-	{
-	  error = dpdk_device_config (conf, pci_addr, &sub_input, 0);
-
-	  if (error)
-	    return error;
-
-	  num_whitelisted++;
-	}
-      else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr))
-	{
-	  error = dpdk_device_config (conf, pci_addr, 0, 0);
-
-	  if (error)
-	    return error;
-
-	  num_whitelisted++;
-	}
-      else if (unformat (input, "num-mbufs %d", &conf->num_mbufs))
-	;
-      else if (unformat (input, "kni %d", &conf->num_kni))
-	;
-      else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
-	;
-      else if (unformat (input, "socket-mem %s", &socket_mem))
-	;
-      else if (unformat (input, "no-pci"))
-	{
-	  no_pci = 1;
-	  tmp = format (0, "--no-pci%c", 0);
-	  vec_add1 (conf->eal_init_args, tmp);
-	}
-      else if (unformat (input, "poll-sleep %d", &dm->poll_sleep))
-	;
-
-#define _(a)                                    \
-      else if (unformat(input, #a))             \
-        {                                       \
-          tmp = format (0, "--%s%c", #a, 0);    \
-          vec_add1 (conf->eal_init_args, tmp);    \
-        }
-      foreach_eal_double_hyphen_predicate_arg
-#undef _
-#define _(a)                                          \
-	else if (unformat(input, #a " %s", &s))	      \
-	  {					      \
-            if (!strncmp(#a, "huge-dir", 8))          \
-              huge_dir = 1;                           \
-            else if (!strncmp(#a, "file-prefix", 11)) \
-              file_prefix = 1;                        \
-	    tmp = format (0, "--%s%c", #a, 0);	      \
-	    vec_add1 (conf->eal_init_args, tmp);      \
-	    vec_add1 (s, 0);			      \
-            if (!strncmp(#a, "vdev", 4))              \
-              if (strstr((char*)s, "af_packet"))      \
-                clib_warning ("af_packet obsoleted. Use CLI 'create host-interface'."); \
-	    vec_add1 (conf->eal_init_args, s);	      \
-	  }
-	foreach_eal_double_hyphen_arg
-#undef _
-#define _(a,b)						\
-	  else if (unformat(input, #a " %s", &s))	\
-	    {						\
-	      tmp = format (0, "-%s%c", #b, 0);		\
-	      vec_add1 (conf->eal_init_args, tmp);	\
-	      vec_add1 (s, 0);				\
-	      vec_add1 (conf->eal_init_args, s);	\
-	    }
-	foreach_eal_single_hyphen_arg
-#undef _
-#define _(a,b)						\
-	    else if (unformat(input, #a " %s", &s))	\
-	      {						\
-		tmp = format (0, "-%s%c", #b, 0);	\
-		vec_add1 (conf->eal_init_args, tmp);	\
-		vec_add1 (s, 0);			\
-		vec_add1 (conf->eal_init_args, s);	\
-		conf->a##_set_manually = 1;		\
-	      }
-	foreach_eal_single_hyphen_mandatory_arg
-#undef _
-	else if (unformat (input, "default"))
-	;
-
-      else if (unformat_skip_white_space (input))
-	;
-      else
-	{
-	  error = clib_error_return (0, "unknown input `%U'",
-				     format_unformat_error, input);
-	  goto done;
-	}
-    }
-
-  if (!conf->uio_driver_name)
-    conf->uio_driver_name = format (0, "uio_pci_generic%c", 0);
-
-  /*
-   * Use 1G huge pages if available.
-   */
-  if (!no_huge && !huge_dir)
-    {
-      u32 x, *mem_by_socket = 0;
-      uword c = 0;
-      u8 use_1g = 1;
-      u8 use_2m = 1;
-      u8 less_than_1g = 1;
-      int rv;
-
-      umount (DEFAULT_HUGE_DIR);
-
-      /* Process "socket-mem" parameter value */
-      if (vec_len (socket_mem))
-	{
-	  unformat_input_t in;
-	  unformat_init_vector (&in, socket_mem);
-	  while (unformat_check_input (&in) != UNFORMAT_END_OF_INPUT)
-	    {
-	      if (unformat (&in, "%u,", &x))
-		;
-	      else if (unformat (&in, "%u", &x))
-		;
-	      else if (unformat (&in, ","))
-		x = 0;
-	      else
-		break;
-
-	      vec_add1 (mem_by_socket, x);
-
-	      if (x > 1023)
-		less_than_1g = 0;
-	    }
-	  /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */
-	  unformat_free (&in);
-	  socket_mem = 0;
-	}
-      else
-	{
-	  /* *INDENT-OFF* */
-	  clib_bitmap_foreach (c, tm->cpu_socket_bitmap, (
-	    {
-	      vec_validate(mem_by_socket, c);
-	      mem_by_socket[c] = 256; /* default per-socket mem */
-	    }
-	  ));
-	  /* *INDENT-ON* */
-	}
-
-      /* check if available enough 1GB pages for each socket */
-      /* *INDENT-OFF* */
-      clib_bitmap_foreach (c, tm->cpu_socket_bitmap, (
-        {
-	  int pages_avail, page_size, mem;
-
-	  vec_validate(mem_by_socket, c);
-	  mem = mem_by_socket[c];
-
-	  page_size = 1024;
-	  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
-
-	  if (pages_avail < 0 || page_size * pages_avail < mem)
-	    use_1g = 0;
-
-	  page_size = 2;
-	  pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024);
-
-	  if (pages_avail < 0 || page_size * pages_avail < mem)
-	    use_2m = 0;
-      }));
-      /* *INDENT-ON* */
-
-      if (mem_by_socket == 0)
-	{
-	  error = clib_error_return (0, "mem_by_socket NULL");
-	  goto done;
-	}
-      _vec_len (mem_by_socket) = c + 1;
-
-      /* regenerate socket_mem string */
-      vec_foreach_index (x, mem_by_socket)
-	socket_mem = format (socket_mem, "%s%u",
-			     socket_mem ? "," : "", mem_by_socket[x]);
-      socket_mem = format (socket_mem, "%c", 0);
-
-      vec_free (mem_by_socket);
-
-      rv = mkdir (VPP_RUN_DIR, 0755);
-      if (rv && errno != EEXIST)
-	{
-	  error = clib_error_return (0, "mkdir '%s' failed errno %d",
-				     VPP_RUN_DIR, errno);
-	  goto done;
-	}
-
-      rv = mkdir (DEFAULT_HUGE_DIR, 0755);
-      if (rv && errno != EEXIST)
-	{
-	  error = clib_error_return (0, "mkdir '%s' failed errno %d",
-				     DEFAULT_HUGE_DIR, errno);
-	  goto done;
-	}
-
-      if (use_1g && !(less_than_1g && use_2m))
-	{
-	  rv =
-	    mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, "pagesize=1G");
-	}
-      else if (use_2m)
-	{
-	  rv = mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, NULL);
-	}
-      else
-	{
-	  return clib_error_return (0, "not enough free huge pages");
-	}
-
-      if (rv)
-	{
-	  error = clib_error_return (0, "mount failed %d", errno);
-	  goto done;
-	}
-
-      tmp = format (0, "--huge-dir%c", 0);
-      vec_add1 (conf->eal_init_args, tmp);
-      tmp = format (0, "%s%c", DEFAULT_HUGE_DIR, 0);
-      vec_add1 (conf->eal_init_args, tmp);
-      if (!file_prefix)
-	{
-	  tmp = format (0, "--file-prefix%c", 0);
-	  vec_add1 (conf->eal_init_args, tmp);
-	  tmp = format (0, "vpp%c", 0);
-	  vec_add1 (conf->eal_init_args, tmp);
-	}
-    }
-
-  vec_free (rte_cmd);
-  vec_free (ethname);
-
-  if (error)
-    return error;
-
-  /* I'll bet that -c and -n must be the first and second args... */
-  if (!conf->coremask_set_manually)
-    {
-      vlib_thread_registration_t *tr;
-      uword *coremask = 0;
-      int i;
-
-      /* main thread core */
-      coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
-
-      for (i = 0; i < vec_len (tm->registrations); i++)
-	{
-	  tr = tm->registrations[i];
-	  coremask = clib_bitmap_or (coremask, tr->coremask);
-	}
-
-      vec_insert (conf->eal_init_args, 2, 1);
-      conf->eal_init_args[1] = (u8 *) "-c";
-      tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
-      conf->eal_init_args[2] = tmp;
-      clib_bitmap_free (coremask);
-    }
-
-  if (!conf->nchannels_set_manually)
-    {
-      vec_insert (conf->eal_init_args, 2, 3);
-      conf->eal_init_args[3] = (u8 *) "-n";
-      tmp = format (0, "%d", conf->nchannels);
-      conf->eal_init_args[4] = tmp;
-    }
-
-  if (no_pci == 0 && geteuid () == 0)
-    dpdk_bind_devices_to_uio (conf);
-
-#define _(x) \
-    if (devconf->x == 0 && conf->default_devconf.x > 0) \
-      devconf->x = conf->default_devconf.x ;
-
-  /* *INDENT-OFF* */
-  pool_foreach (devconf, conf->dev_confs, ({
-
-    /* default per-device config items */
-    foreach_dpdk_device_config_item
-
-    /* add DPDK EAL whitelist/blacklist entry */
-    if (num_whitelisted > 0 && devconf->is_blacklisted == 0)
-      {
-	tmp = format (0, "-w%c", 0);
-	vec_add1 (conf->eal_init_args, tmp);
-	tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
-	vec_add1 (conf->eal_init_args, tmp);
-      }
-    else if (num_whitelisted == 0 && devconf->is_blacklisted != 0)
-      {
-	tmp = format (0, "-b%c", 0);
-	vec_add1 (conf->eal_init_args, tmp);
-	tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0);
-	vec_add1 (conf->eal_init_args, tmp);
-      }
-  }));
-  /* *INDENT-ON* */
-
-#undef _
-
-  /* set master-lcore */
-  tmp = format (0, "--master-lcore%c", 0);
-  vec_add1 (conf->eal_init_args, tmp);
-  tmp = format (0, "%u%c", tm->main_lcore, 0);
-  vec_add1 (conf->eal_init_args, tmp);
-
-  /* set socket-mem */
-  tmp = format (0, "--socket-mem%c", 0);
-  vec_add1 (conf->eal_init_args, tmp);
-  tmp = format (0, "%s%c", socket_mem, 0);
-  vec_add1 (conf->eal_init_args, tmp);
-
-  /* NULL terminate the "argv" vector, in case of stupidity */
-  vec_add1 (conf->eal_init_args, 0);
-  _vec_len (conf->eal_init_args) -= 1;
-
-  /* Set up DPDK eal and packet mbuf pool early. */
-
-  log_level = (CLIB_DEBUG > 0) ? RTE_LOG_DEBUG : RTE_LOG_NOTICE;
-
-  rte_set_log_level (log_level);
-
-  vm = vlib_get_main ();
-
-  /* make copy of args as rte_eal_init tends to mess up with arg array */
-  for (i = 1; i < vec_len (conf->eal_init_args); i++)
-    conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ",
-				      conf->eal_init_args[i]);
-
-  ret =
-    rte_eal_init (vec_len (conf->eal_init_args),
-		  (char **) conf->eal_init_args);
-
-  /* lazy umount hugepages */
-  umount2 (DEFAULT_HUGE_DIR, MNT_DETACH);
-
-  if (ret < 0)
-    return clib_error_return (0, "rte_eal_init returned %d", ret);
-
-  /* Dump the physical memory layout prior to creating the mbuf_pool */
-  fprintf (stdout, "DPDK physical memory layout:\n");
-  rte_dump_physmem_layout (stdout);
-
-  /* main thread 1st */
-  error = vlib_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ());
-  if (error)
-    return error;
-
-  for (i = 0; i < RTE_MAX_LCORE; i++)
-    {
-      error = vlib_buffer_pool_create (vm, conf->num_mbufs,
-				       rte_lcore_to_socket_id (i));
-      if (error)
-	return error;
-    }
-
-done:
-  return error;
-}
-
-VLIB_CONFIG_FUNCTION (dpdk_config, "dpdk");
-
-void
-dpdk_update_link_state (dpdk_device_t * xd, f64 now)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  struct rte_eth_link prev_link = xd->link;
-  u32 hw_flags = 0;
-  u8 hw_flags_chg = 0;
-
-  /* only update link state for PMD interfaces */
-  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
-    return;
-
-  xd->time_last_link_update = now ? now : xd->time_last_link_update;
-  memset (&xd->link, 0, sizeof (xd->link));
-  rte_eth_link_get_nowait (xd->device_index, &xd->link);
-
-  if (LINK_STATE_ELOGS)
-    {
-      vlib_main_t *vm = vlib_get_main ();
-      ELOG_TYPE_DECLARE (e) =
-      {
-      .format =
-	  "update-link-state: sw_if_index %d, admin_up %d,"
-	  "old link_state %d new link_state %d",.format_args = "i4i1i1i1",};
-
-      struct
-      {
-	u32 sw_if_index;
-	u8 admin_up;
-	u8 old_link_state;
-	u8 new_link_state;
-      } *ed;
-      ed = ELOG_DATA (&vm->elog_main, e);
-      ed->sw_if_index = xd->vlib_sw_if_index;
-      ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
-      ed->old_link_state = (u8)
-	vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index);
-      ed->new_link_state = (u8) xd->link.link_status;
-    }
-
-  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) &&
-      ((xd->link.link_status != 0) ^
-       vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index)))
-    {
-      hw_flags_chg = 1;
-      hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
-    }
-
-  if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex))
-    {
-      hw_flags_chg = 1;
-      switch (xd->link.link_duplex)
-	{
-	case ETH_LINK_HALF_DUPLEX:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_HALF_DUPLEX;
-	  break;
-	case ETH_LINK_FULL_DUPLEX:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_FULL_DUPLEX;
-	  break;
-	default:
-	  break;
-	}
-    }
-  if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed))
-    {
-      hw_flags_chg = 1;
-      switch (xd->link.link_speed)
-	{
-	case ETH_SPEED_NUM_10M:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_10M;
-	  break;
-	case ETH_SPEED_NUM_100M:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_100M;
-	  break;
-	case ETH_SPEED_NUM_1G:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G;
-	  break;
-	case ETH_SPEED_NUM_10G:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_10G;
-	  break;
-	case ETH_SPEED_NUM_40G:
-	  hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_40G;
-	  break;
-	case 0:
-	  break;
-	default:
-	  clib_warning ("unknown link speed %d", xd->link.link_speed);
-	  break;
-	}
-    }
-  if (hw_flags_chg)
-    {
-      if (LINK_STATE_ELOGS)
-	{
-	  vlib_main_t *vm = vlib_get_main ();
-
-	  ELOG_TYPE_DECLARE (e) =
-	  {
-	  .format =
-	      "update-link-state: sw_if_index %d, new flags %d",.format_args
-	      = "i4i4",};
-
-	  struct
-	  {
-	    u32 sw_if_index;
-	    u32 flags;
-	  } *ed;
-	  ed = ELOG_DATA (&vm->elog_main, e);
-	  ed->sw_if_index = xd->vlib_sw_if_index;
-	  ed->flags = hw_flags;
-	}
-      vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, hw_flags);
-    }
-}
-
-static uword
-dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
-{
-  clib_error_t *error;
-  vnet_main_t *vnm = vnet_get_main ();
-  dpdk_main_t *dm = &dpdk_main;
-  ethernet_main_t *em = ðernet_main;
-  dpdk_device_t *xd;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  int i;
-
-  error = dpdk_lib_init (dm);
-
-  /*
-   * Turn on the input node if we found some devices to drive
-   * and we're not running worker threads or i/o threads
-   */
-
-  if (error == 0 && vec_len (dm->devices) > 0)
-    {
-      if (tm->n_vlib_mains == 1)
-	vlib_node_set_state (vm, dpdk_input_node.index,
-			     VLIB_NODE_STATE_POLLING);
-      else
-	for (i = 0; i < tm->n_vlib_mains; i++)
-	  if (vec_len (dm->devices_by_cpu[i]) > 0)
-	    vlib_node_set_state (vlib_mains[i], dpdk_input_node.index,
-				 VLIB_NODE_STATE_POLLING);
-    }
-
-  if (error)
-    clib_error_report (error);
-
-  tm->worker_thread_release = 1;
-
-  f64 now = vlib_time_now (vm);
-  vec_foreach (xd, dm->devices)
-  {
-    dpdk_update_link_state (xd, now);
-  }
-
-  {
-    /*
-     * Extra set up for bond interfaces:
-     *  1. Setup MACs for bond interfaces and their slave links which was set
-     *     in dpdk_port_setup() but needs to be done again here to take effect.
-     *  2. Set up info for bond interface related CLI support.
-     */
-    int nports = rte_eth_dev_count ();
-    if (nports > 0)
-      {
-	for (i = 0; i < nports; i++)
-	  {
-	    struct rte_eth_dev_info dev_info;
-	    rte_eth_dev_info_get (i, &dev_info);
-	    if (!dev_info.driver_name)
-	      dev_info.driver_name = dev_info.pci_dev->driver->driver.name;
-
-	    ASSERT (dev_info.driver_name);
-	    if (strncmp (dev_info.driver_name, "rte_bond_pmd", 12) == 0)
-	      {
-		u8 addr[6];
-		u8 slink[16];
-		int nlink = rte_eth_bond_slaves_get (i, slink, 16);
-		if (nlink > 0)
-		  {
-		    vnet_hw_interface_t *bhi;
-		    ethernet_interface_t *bei;
-		    int rv;
-
-		    /* Get MAC of 1st slave link */
-		    rte_eth_macaddr_get (slink[0],
-					 (struct ether_addr *) addr);
-		    /* Set MAC of bounded interface to that of 1st slave link */
-		    rv =
-		      rte_eth_bond_mac_address_set (i,
-						    (struct ether_addr *)
-						    addr);
-		    if (rv < 0)
-		      clib_warning ("Failed to set MAC address");
-
-		    /* Populate MAC of bonded interface in VPP hw tables */
-		    bhi =
-		      vnet_get_hw_interface (vnm,
-					     dm->devices[i].vlib_hw_if_index);
-		    bei =
-		      pool_elt_at_index (em->interfaces, bhi->hw_instance);
-		    clib_memcpy (bhi->hw_address, addr, 6);
-		    clib_memcpy (bei->address, addr, 6);
-		    /* Init l3 packet size allowed on bonded interface */
-		    bhi->max_packet_bytes = ETHERNET_MAX_PACKET_BYTES;
-		    bhi->max_l3_packet_bytes[VLIB_RX] =
-		      bhi->max_l3_packet_bytes[VLIB_TX] =
-		      ETHERNET_MAX_PACKET_BYTES - sizeof (ethernet_header_t);
-		    while (nlink >= 1)
-		      {		/* for all slave links */
-			int slave = slink[--nlink];
-			dpdk_device_t *sdev = &dm->devices[slave];
-			vnet_hw_interface_t *shi;
-			vnet_sw_interface_t *ssi;
-			/* Add MAC to all slave links except the first one */
-			if (nlink)
-			  rte_eth_dev_mac_addr_add (slave,
-						    (struct ether_addr *)
-						    addr, 0);
-			/* Set slaves bitmap for bonded interface */
-			bhi->bond_info =
-			  clib_bitmap_set (bhi->bond_info,
-					   sdev->vlib_hw_if_index, 1);
-			/* Set slave link flags on slave interface */
-			shi =
-			  vnet_get_hw_interface (vnm, sdev->vlib_hw_if_index);
-			ssi =
-			  vnet_get_sw_interface (vnm, sdev->vlib_sw_if_index);
-			shi->bond_info = VNET_HW_INTERFACE_BOND_INFO_SLAVE;
-			ssi->flags |= VNET_SW_INTERFACE_FLAG_BOND_SLAVE;
-
-			/* Set l3 packet size allowed as the lowest of slave */
-			if (bhi->max_l3_packet_bytes[VLIB_RX] >
-			    shi->max_l3_packet_bytes[VLIB_RX])
-			  bhi->max_l3_packet_bytes[VLIB_RX] =
-			    bhi->max_l3_packet_bytes[VLIB_TX] =
-			    shi->max_l3_packet_bytes[VLIB_RX];
-
-			/* Set max packet size allowed as the lowest of slave */
-			if (bhi->max_packet_bytes > shi->max_packet_bytes)
-			  bhi->max_packet_bytes = shi->max_packet_bytes;
-		      }
-		  }
-	      }
-	  }
-      }
-  }
-
-  while (1)
-    {
-      /*
-       * check each time through the loop in case intervals are changed
-       */
-      f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ?
-	dm->link_state_poll_interval : dm->stat_poll_interval;
-
-      vlib_process_wait_for_event_or_clock (vm, min_wait);
-
-      if (dm->admin_up_down_in_progress)
-	/* skip the poll if an admin up down is in progress (on any interface) */
-	continue;
-
-      vec_foreach (xd, dm->devices)
-      {
-	f64 now = vlib_time_now (vm);
-	if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval)
-	  dpdk_update_counters (xd, now);
-	if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval)
-	  dpdk_update_link_state (xd, now);
-
-      }
-    }
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (dpdk_process_node,static) = {
-    .function = dpdk_process,
-    .type = VLIB_NODE_TYPE_PROCESS,
-    .name = "dpdk-process",
-    .process_log2_n_stack_bytes = 17,
-};
-/* *INDENT-ON* */
-
-int
-dpdk_set_stat_poll_interval (f64 interval)
-{
-  if (interval < DPDK_MIN_STATS_POLL_INTERVAL)
-    return (VNET_API_ERROR_INVALID_VALUE);
-
-  dpdk_main.stat_poll_interval = interval;
-
-  return 0;
-}
-
-int
-dpdk_set_link_state_poll_interval (f64 interval)
-{
-  if (interval < DPDK_MIN_LINK_POLL_INTERVAL)
-    return (VNET_API_ERROR_INVALID_VALUE);
-
-  dpdk_main.link_state_poll_interval = interval;
-
-  return 0;
-}
-
-clib_error_t *
-dpdk_init (vlib_main_t * vm)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  vlib_node_t *ei;
-  clib_error_t *error = 0;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-
-  /* verify that structs are cacheline aligned */
-  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0,
-		 "Cache line marker must be 1st element in dpdk_device_t");
-  STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) ==
-		 CLIB_CACHE_LINE_BYTES,
-		 "Data in cache line 0 is bigger than cache line size");
-  STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0,
-		 "Cache line marker must be 1st element in frame_queue_trace_t");
-
-  dm->vlib_main = vm;
-  dm->vnet_main = vnet_get_main ();
-  dm->conf = &dpdk_config_main;
-
-  ei = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
-  if (ei == 0)
-    return clib_error_return (0, "ethernet-input node AWOL");
-
-  dm->ethernet_input_node_index = ei->index;
-
-  dm->conf->nchannels = 4;
-  dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF;
-  vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
-
-  dm->dpdk_device_by_kni_port_id = hash_create (0, sizeof (uword));
-  dm->vu_sw_if_index_by_listener_fd = hash_create (0, sizeof (uword));
-  dm->vu_sw_if_index_by_sock_fd = hash_create (0, sizeof (uword));
-
-  /* $$$ use n_thread_stacks since it's known-good at this point */
-  vec_validate (dm->recycle, tm->n_thread_stacks - 1);
-
-  /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
-  dm->buffer_flags_template =
-    (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID
-     | IP_BUFFER_L4_CHECKSUM_COMPUTED | IP_BUFFER_L4_CHECKSUM_CORRECT);
-
-  dm->stat_poll_interval = DPDK_STATS_POLL_INTERVAL;
-  dm->link_state_poll_interval = DPDK_LINK_POLL_INTERVAL;
-
-  /* init CLI */
-  if ((error = vlib_call_init_function (vm, dpdk_cli_init)))
-    return error;
-
-  return error;
-}
-
-VLIB_INIT_FUNCTION (dpdk_init);
-
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/ipsec/cli.c b/src/vnet/devices/dpdk/ipsec/cli.c
deleted file mode 100644
index f9d3a5d0..00000000
--- a/src/vnet/devices/dpdk/ipsec/cli.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-
-static void
-dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display)
-{
-  dpdk_config_main_t *conf = &dpdk_config_main;
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  u32 i, skip_master;
-
-  if (!conf->cryptodev)
-    {
-      vlib_cli_output (vm, "DPDK Cryptodev support is disabled\n");
-      return;
-    }
-
-  if (detail_display)
-    vlib_cli_output (vm, "worker\t%10s\t%15s\tdir\tdev\tqp\n",
-		     "cipher", "auth");
-  else
-    vlib_cli_output (vm, "worker\tcrypto device id(type)\n");
-
-  skip_master = vlib_num_workers () > 0;
-
-  for (i = 0; i < tm->n_vlib_mains; i++)
-    {
-      uword key, data;
-      u32 cpu_index = vlib_mains[i]->cpu_index;
-      crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
-      u8 *s = 0;
-
-      if (skip_master)
-	{
-	  skip_master = 0;
-	  continue;
-	}
-
-      if (!detail_display)
-	{
-	  i32 last_cdev = -1;
-	  crypto_qp_data_t *qpd;
-
-	  s = format (s, "%u\t", cpu_index);
-
-	  /* *INDENT-OFF* */
-	  vec_foreach (qpd, cwm->qp_data)
-	    {
-	      u32 dev_id = qpd->dev_id;
-
-	      if ((u16) last_cdev != dev_id)
-		{
-		  struct rte_cryptodev_info cdev_info;
-
-		  rte_cryptodev_info_get (dev_id, &cdev_info);
-
-		  s = format(s, "%u(%s)\t", dev_id, cdev_info.feature_flags &
-			     RTE_CRYPTODEV_FF_HW_ACCELERATED ? "HW" : "SW");
-		}
-	      last_cdev = dev_id;
-	    }
-	  /* *INDENT-ON* */
-	  vlib_cli_output (vm, "%s", s);
-	}
-      else
-	{
-	  char cipher_str[15], auth_str[15];
-	  struct rte_cryptodev_capabilities cap;
-	  crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
-	  /* *INDENT-OFF* */
-	  hash_foreach (key, data, cwm->algo_qp_map,
-	  ({
-	    cap.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
-	    cap.sym.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER;
-	    cap.sym.cipher.algo = p_key->cipher_algo;
-	    check_algo_is_supported (&cap, cipher_str);
-	    cap.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
-	    cap.sym.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH;
-	    cap.sym.auth.algo = p_key->auth_algo;
-	    check_algo_is_supported (&cap, auth_str);
-	    vlib_cli_output (vm, "%u\t%10s\t%15s\t%3s\t%u\t%u\n",
-			     vlib_mains[i]->cpu_index, cipher_str, auth_str,
-			     p_key->is_outbound ? "out" : "in",
-			     cwm->qp_data[data].dev_id,
-			     cwm->qp_data[data].qp_id);
-	  }));
-	  /* *INDENT-ON* */
-	}
-    }
-}
-
-static clib_error_t *
-lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input,
-			vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  u16 detail = 0;
-  clib_error_t *error = NULL;
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "verbose"))
-	detail = 1;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  dpdk_ipsec_show_mapping (vm, detail);
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (lcore_cryptodev_map, static) = {
-    .path = "show crypto device mapping",
-    .short_help =
-    "show cryptodev device mapping ",
-    .function = lcore_cryptodev_map_fn,
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/ipsec/crypto_node.c b/src/vnet/devices/dpdk/ipsec/crypto_node.c
deleted file mode 100644
index e8fef235..00000000
--- a/src/vnet/devices/dpdk/ipsec/crypto_node.c
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- *------------------------------------------------------------------
- * crypto_node.c - DPDK Cryptodev input node
- *
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *------------------------------------------------------------------
- */
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-#define foreach_dpdk_crypto_input_next		\
-  _(DROP, "error-drop")				\
-  _(ENCRYPT_POST, "dpdk-esp-encrypt-post")	\
-  _(DECRYPT_POST, "dpdk-esp-decrypt-post")
-
-typedef enum
-{
-#define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f,
-  foreach_dpdk_crypto_input_next
-#undef _
-    DPDK_CRYPTO_INPUT_N_NEXT,
-} dpdk_crypto_input_next_t;
-
-#define foreach_dpdk_crypto_input_error		\
-  _(DQ_COPS, "Crypto ops dequeued")		\
-  _(COP_FAILED, "Crypto op failed")
-
-typedef enum
-{
-#define _(f,s) DPDK_CRYPTO_INPUT_ERROR_##f,
-  foreach_dpdk_crypto_input_error
-#undef _
-    DPDK_CRYPTO_INPUT_N_ERROR,
-} dpdk_crypto_input_error_t;
-
-static char *dpdk_crypto_input_error_strings[] = {
-#define _(n, s) s,
-  foreach_dpdk_crypto_input_error
-#undef _
-};
-
-vlib_node_registration_t dpdk_crypto_input_node;
-
-typedef struct
-{
-  u32 cdev;
-  u32 qp;
-  u32 status;
-  u32 sa_idx;
-  u32 next_index;
-} dpdk_crypto_input_trace_t;
-
-static u8 *
-format_dpdk_crypto_input_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  dpdk_crypto_input_trace_t *t = va_arg (*args, dpdk_crypto_input_trace_t *);
-
-  s = format (s, "dpdk_crypto: cryptodev-id %u queue-pair %u next-index %d",
-	      t->cdev, t->qp, t->next_index);
-
-  s = format (s, " status %u sa-idx %u\n", t->status, t->sa_idx);
-
-  return s;
-}
-
-static_always_inline u32
-dpdk_crypto_dequeue (vlib_main_t * vm, vlib_node_runtime_t * node,
-		     crypto_qp_data_t * qpd)
-{
-  u32 n_deq, *to_next = 0, next_index, n_cops, def_next_index;
-  struct rte_crypto_op **cops = qpd->cops;
-
-  if (qpd->inflights == 0)
-    return 0;
-
-  if (qpd->is_outbound)
-    def_next_index = DPDK_CRYPTO_INPUT_NEXT_ENCRYPT_POST;
-  else
-    def_next_index = DPDK_CRYPTO_INPUT_NEXT_DECRYPT_POST;
-
-  n_cops = rte_cryptodev_dequeue_burst (qpd->dev_id, qpd->qp_id,
-					cops, VLIB_FRAME_SIZE);
-  n_deq = n_cops;
-  next_index = def_next_index;
-
-  qpd->inflights -= n_cops;
-  ASSERT (qpd->inflights >= 0);
-
-  while (n_cops > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_cops > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0, next0;
-	  vlib_buffer_t *b0 = 0;
-	  struct rte_crypto_op *cop;
-	  struct rte_crypto_sym_op *sym_cop;
-
-	  cop = cops[0];
-	  cops += 1;
-	  n_cops -= 1;
-	  n_left_to_next -= 1;
-
-	  next0 = def_next_index;
-
-	  if (PREDICT_FALSE (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
-	    {
-	      next0 = DPDK_CRYPTO_INPUT_NEXT_DROP;
-	      vlib_node_increment_counter (vm, dpdk_crypto_input_node.index,
-					   DPDK_CRYPTO_INPUT_ERROR_COP_FAILED,
-					   1);
-	    }
-	  cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
-
-	  sym_cop = (struct rte_crypto_sym_op *) (cop + 1);
-	  b0 = vlib_buffer_from_rte_mbuf (sym_cop->m_src);
-	  bi0 = vlib_get_buffer_index (vm, b0);
-
-	  to_next[0] = bi0;
-	  to_next += 1;
-
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      vlib_trace_next_frame (vm, node, next0);
-	      dpdk_crypto_input_trace_t *tr =
-		vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      tr->cdev = qpd->dev_id;
-	      tr->qp = qpd->qp_id;
-	      tr->status = cop->status;
-	      tr->next_index = next0;
-	      tr->sa_idx = vnet_buffer (b0)->ipsec.sad_index;
-	    }
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-					   n_left_to_next, bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  crypto_free_cop (qpd, qpd->cops, n_deq);
-
-  vlib_node_increment_counter (vm, dpdk_crypto_input_node.index,
-			       DPDK_CRYPTO_INPUT_ERROR_DQ_COPS, n_deq);
-  return n_deq;
-}
-
-static uword
-dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
-		      vlib_frame_t * frame)
-{
-  u32 cpu_index = os_get_cpu_number ();
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
-  crypto_qp_data_t *qpd;
-  u32 n_deq = 0;
-
-  /* *INDENT-OFF* */
-  vec_foreach (qpd, cwm->qp_data)
-      n_deq += dpdk_crypto_dequeue(vm, node, qpd);
-  /* *INDENT-ON* */
-
-  return n_deq;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (dpdk_crypto_input_node) =
-{
-  .function = dpdk_crypto_input_fn,
-  .name = "dpdk-crypto-input",
-  .format_trace = format_dpdk_crypto_input_trace,
-  .type = VLIB_NODE_TYPE_INPUT,
-  .state = VLIB_NODE_STATE_DISABLED,
-  .n_errors = DPDK_CRYPTO_INPUT_N_ERROR,
-  .error_strings = dpdk_crypto_input_error_strings,
-  .n_next_nodes = DPDK_CRYPTO_INPUT_N_NEXT,
-  .next_nodes =
-  {
-#define _(s,n) [DPDK_CRYPTO_INPUT_NEXT_##s] = n,
-    foreach_dpdk_crypto_input_next
-#undef _
-  },
-};
-/* *INDENT-ON* */
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_crypto_input_node, dpdk_crypto_input_fn)
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/ipsec/dir.dox b/src/vnet/devices/dpdk/ipsec/dir.dox
deleted file mode 100644
index ffebfc4d..00000000
--- a/src/vnet/devices/dpdk/ipsec/dir.dox
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- @dir vnet/vnet/devices/dpdk/ipsec
- @brief IPSec ESP encrypt/decrypt using DPDK Cryptodev API
-*/
diff --git a/src/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md b/src/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md
deleted file mode 100644
index fed2fe0e..00000000
--- a/src/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# VPP IPSec implementation using DPDK Cryptodev API    {#dpdk_crypto_ipsec_doc}
-
-This document is meant to contain all related information about implementation and usability.
-
-
-## VPP IPsec with DPDK Cryptodev
-
-DPDK Cryptodev is an asynchronous crypto API that supports both Hardware and Software implementations (for more details refer to [DPDK Cryptography Device Library documentation](http://dpdk.org/doc/guides/prog_guide/cryptodev_lib.html)).
-
-When DPDK support is enabled and there are enough Cryptodev resources for all workers, the node graph is reconfigured by adding and changing default next nodes.
-
-The following nodes are added:
-* dpdk-crypto-input : polling input node, basically dequeuing from crypto devices.
-* dpdk-esp-encrypt : internal node.
-* dpdk-esp-decrypt : internal node.
-* dpdk-esp-encrypt-post : internal node.
-* dpdk-esp-decrypt-post : internal node.
-
-Set new default next nodes:
-* for esp encryption: esp-encrypt -> dpdk-esp-encrypt
-* for esp decryption: esp-decrypt -> dpdk-esp-decrypt
-
-
-### How to enable VPP IPSec with DPDK Cryptodev support
-
-DPDK Cryptodev is supported in DPDK enabled VPP.
-By default, only HW Cryptodev is supported but needs to be explicetly enabled with the following config option:
-
-```
-dpdk {
-    enable-cryptodev
-}
-```
-
-To enable SW Cryptodev support (AESNI-MB-PMD and GCM-PMD), we need the following env option:
-
-    vpp_uses_dpdk_cryptodev_sw=yes
-
-A couple of ways to achive this:
-* uncomment/add it in the platforms config (ie. build-data/platforms/vpp.mk)
-* set the option when building vpp (ie. make vpp_uses_dpdk_cryptodev_sw=yes build-release)
-
-When enabling SW Cryptodev support, it means that you need to pre-build the required crypto libraries needed by those SW Cryptodev PMDs.
-
-
-### Crypto Resources allocation
-
-VPP allocates crypto resources based on a best effort approach:
-* first allocate Hardware crypto resources, then Software.
-* if there are not enough crypto resources for all workers, the graph node is not modifed, therefore the default VPP IPsec implementation based in OpenSSL is used. The following message is displayed:
-
-      0: dpdk_ipsec_init: not enough cryptodevs for ipsec
-
-
-### Configuration example
-
-To enable DPDK Cryptodev the user just need to provide the startup.conf option
-as mentioned previously.
-
-Example startup.conf:
-
-```
-dpdk {
-    socket-mem 1024,1024
-    num-mbufs 131072
-    dev 0000:81:00.0
-    dev 0000:81:00.1
-    enable-cryptodev
-    dev 0000:85:01.0
-    dev 0000:85:01.1
-    vdev cryptodev_aesni_mb_pmd,socket_id=1
-    vdev cryptodev_aesni_mb_pmd,socket_id=1
-}
-```
-
-In the above configuration:
-* 0000:85:01.0 and 0000:85:01.1 are crypto BDFs and they require the same driver binding as DPDK Ethernet devices but they do not support any extra configuration options.
-* Two AESNI-MB Software Cryptodev PMDs are created in NUMA node 1.
-
-For further details refer to [DPDK Crypto Device Driver documentation](http://dpdk.org/doc/guides/cryptodevs/index.html)
-
-### Operational data
-
-The following CLI command displays the Cryptodev/Worker mapping:
-
-    show crypto device mapping [verbose]
diff --git a/src/vnet/devices/dpdk/ipsec/esp.h b/src/vnet/devices/dpdk/ipsec/esp.h
deleted file mode 100644
index d0b27618..00000000
--- a/src/vnet/devices/dpdk/ipsec/esp.h
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __DPDK_ESP_H__
-#define __DPDK_ESP_H__
-
-#include 
-#include 
-#include 
-
-typedef struct
-{
-  enum rte_crypto_cipher_algorithm algo;
-  u8 key_len;
-  u8 iv_len;
-} dpdk_esp_crypto_alg_t;
-
-typedef struct
-{
-  enum rte_crypto_auth_algorithm algo;
-  u8 trunc_size;
-} dpdk_esp_integ_alg_t;
-
-typedef struct
-{
-  dpdk_esp_crypto_alg_t *esp_crypto_algs;
-  dpdk_esp_integ_alg_t *esp_integ_algs;
-} dpdk_esp_main_t;
-
-dpdk_esp_main_t dpdk_esp_main;
-
-static_always_inline void
-dpdk_esp_init ()
-{
-  dpdk_esp_main_t *em = &dpdk_esp_main;
-  dpdk_esp_integ_alg_t *i;
-  dpdk_esp_crypto_alg_t *c;
-
-  vec_validate (em->esp_crypto_algs, IPSEC_CRYPTO_N_ALG - 1);
-
-  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
-  c->algo = RTE_CRYPTO_CIPHER_AES_CBC;
-  c->key_len = 16;
-  c->iv_len = 16;
-
-  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
-  c->algo = RTE_CRYPTO_CIPHER_AES_CBC;
-  c->key_len = 24;
-  c->iv_len = 16;
-
-  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
-  c->algo = RTE_CRYPTO_CIPHER_AES_CBC;
-  c->key_len = 32;
-  c->iv_len = 16;
-
-  c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
-  c->algo = RTE_CRYPTO_CIPHER_AES_GCM;
-  c->key_len = 16;
-  c->iv_len = 8;
-
-  vec_validate (em->esp_integ_algs, IPSEC_INTEG_N_ALG - 1);
-
-  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA1_96];
-  i->algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
-  i->trunc_size = 12;
-
-  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_256_96];
-  i->algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
-  i->trunc_size = 12;
-
-  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_256_128];
-  i->algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
-  i->trunc_size = 16;
-
-  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_384_192];
-  i->algo = RTE_CRYPTO_AUTH_SHA384_HMAC;
-  i->trunc_size = 24;
-
-  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_512_256];
-  i->algo = RTE_CRYPTO_AUTH_SHA512_HMAC;
-  i->trunc_size = 32;
-
-  i = &em->esp_integ_algs[IPSEC_INTEG_ALG_AES_GCM_128];
-  i->algo = RTE_CRYPTO_AUTH_AES_GCM;
-  i->trunc_size = 16;
-}
-
-static_always_inline int
-translate_crypto_algo (ipsec_crypto_alg_t crypto_algo,
-		       struct rte_crypto_sym_xform *cipher_xform)
-{
-  switch (crypto_algo)
-    {
-    case IPSEC_CRYPTO_ALG_NONE:
-      cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
-      break;
-    case IPSEC_CRYPTO_ALG_AES_CBC_128:
-    case IPSEC_CRYPTO_ALG_AES_CBC_192:
-    case IPSEC_CRYPTO_ALG_AES_CBC_256:
-      cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
-      break;
-    case IPSEC_CRYPTO_ALG_AES_GCM_128:
-      cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_GCM;
-      break;
-    default:
-      return -1;
-    }
-
-  cipher_xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
-
-  return 0;
-}
-
-static_always_inline int
-translate_integ_algo (ipsec_integ_alg_t integ_alg,
-		      struct rte_crypto_sym_xform *auth_xform, int use_esn)
-{
-  switch (integ_alg)
-    {
-    case IPSEC_INTEG_ALG_NONE:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_NULL;
-      auth_xform->auth.digest_length = 0;
-      break;
-    case IPSEC_INTEG_ALG_SHA1_96:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
-      auth_xform->auth.digest_length = 12;
-      break;
-    case IPSEC_INTEG_ALG_SHA_256_96:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
-      auth_xform->auth.digest_length = 12;
-      break;
-    case IPSEC_INTEG_ALG_SHA_256_128:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
-      auth_xform->auth.digest_length = 16;
-      break;
-    case IPSEC_INTEG_ALG_SHA_384_192:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA384_HMAC;
-      auth_xform->auth.digest_length = 24;
-      break;
-    case IPSEC_INTEG_ALG_SHA_512_256:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA512_HMAC;
-      auth_xform->auth.digest_length = 32;
-      break;
-    case IPSEC_INTEG_ALG_AES_GCM_128:
-      auth_xform->auth.algo = RTE_CRYPTO_AUTH_AES_GCM;
-      auth_xform->auth.digest_length = 16;
-      auth_xform->auth.add_auth_data_length = use_esn ? 12 : 8;
-      break;
-    default:
-      return -1;
-    }
-
-  auth_xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
-
-  return 0;
-}
-
-static_always_inline int
-create_sym_sess (ipsec_sa_t * sa, crypto_sa_session_t * sa_sess,
-		 u8 is_outbound)
-{
-  u32 cpu_index = os_get_cpu_number ();
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
-  struct rte_crypto_sym_xform cipher_xform = { 0 };
-  struct rte_crypto_sym_xform auth_xform = { 0 };
-  struct rte_crypto_sym_xform *xfs;
-  uword key = 0, *data;
-  crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
-
-  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
-    {
-      sa->crypto_key_len -= 4;
-      clib_memcpy (&sa->salt, &sa->crypto_key[sa->crypto_key_len], 4);
-    }
-  else
-    {
-      u32 seed = (u32) clib_cpu_time_now ();
-      sa->salt = random_u32 (&seed);
-    }
-
-  cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
-  cipher_xform.cipher.key.data = sa->crypto_key;
-  cipher_xform.cipher.key.length = sa->crypto_key_len;
-
-  auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
-  auth_xform.auth.key.data = sa->integ_key;
-  auth_xform.auth.key.length = sa->integ_key_len;
-
-  if (translate_crypto_algo (sa->crypto_alg, &cipher_xform) < 0)
-    return -1;
-  p_key->cipher_algo = cipher_xform.cipher.algo;
-
-  if (translate_integ_algo (sa->integ_alg, &auth_xform, sa->use_esn) < 0)
-    return -1;
-  p_key->auth_algo = auth_xform.auth.algo;
-
-  if (is_outbound)
-    {
-      cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
-      auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
-      cipher_xform.next = &auth_xform;
-      xfs = &cipher_xform;
-    }
-  else
-    {
-      cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
-      auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
-      auth_xform.next = &cipher_xform;
-      xfs = &auth_xform;
-    }
-
-  p_key->is_outbound = is_outbound;
-
-  data = hash_get (cwm->algo_qp_map, key);
-  if (!data)
-    return -1;
-
-  sa_sess->sess =
-    rte_cryptodev_sym_session_create (cwm->qp_data[*data].dev_id, xfs);
-
-  if (!sa_sess->sess)
-    return -1;
-
-  sa_sess->qp_index = (u8) * data;
-
-  return 0;
-}
-
-#endif /* __DPDK_ESP_H__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/ipsec/esp_decrypt.c b/src/vnet/devices/dpdk/ipsec/esp_decrypt.c
deleted file mode 100644
index 76007609..00000000
--- a/src/vnet/devices/dpdk/ipsec/esp_decrypt.c
+++ /dev/null
@@ -1,594 +0,0 @@
-/*
- * esp_decrypt.c : IPSec ESP Decrypt node using DPDK Cryptodev
- *
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define foreach_esp_decrypt_next	       \
-_(DROP, "error-drop")			       \
-_(IP4_INPUT, "ip4-input")		       \
-_(IP6_INPUT, "ip6-input")
-
-#define _(v, s) ESP_DECRYPT_NEXT_##v,
-typedef enum {
-  foreach_esp_decrypt_next
-#undef _
-  ESP_DECRYPT_N_NEXT,
-} esp_decrypt_next_t;
-
-#define foreach_esp_decrypt_error		 \
- _(RX_PKTS, "ESP pkts received")		 \
- _(DECRYPTION_FAILED, "ESP decryption failed")   \
- _(REPLAY, "SA replayed packet")	         \
- _(NOT_IP, "Not IP packet (dropped)")	         \
- _(ENQ_FAIL, "Enqueue failed (buffer full)")     \
- _(NO_CRYPTODEV, "Cryptodev not configured")     \
- _(BAD_LEN, "Invalid ciphertext length")         \
- _(UNSUPPORTED, "Cipher/Auth not supported")
-
-
-typedef enum {
-#define _(sym,str) ESP_DECRYPT_ERROR_##sym,
-  foreach_esp_decrypt_error
-#undef _
-  ESP_DECRYPT_N_ERROR,
-} esp_decrypt_error_t;
-
-static char * esp_decrypt_error_strings[] = {
-#define _(sym,string) string,
-  foreach_esp_decrypt_error
-#undef _
-};
-
-vlib_node_registration_t dpdk_esp_decrypt_node;
-
-typedef struct {
-  ipsec_crypto_alg_t crypto_alg;
-  ipsec_integ_alg_t integ_alg;
-} esp_decrypt_trace_t;
-
-/* packet trace format function */
-static u8 * format_esp_decrypt_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *);
-
-  s = format (s, "esp: crypto %U integrity %U",
-	      format_ipsec_crypto_alg, t->crypto_alg,
-	      format_ipsec_integ_alg, t->integ_alg);
-  return s;
-}
-
-static uword
-dpdk_esp_decrypt_node_fn (vlib_main_t * vm,
-	     vlib_node_runtime_t * node,
-	     vlib_frame_t * from_frame)
-{
-  u32 n_left_from, *from, *to_next, next_index;
-  ipsec_main_t *im = &ipsec_main;
-  u32 cpu_index = os_get_cpu_number();
-  dpdk_crypto_main_t * dcm = &dpdk_crypto_main;
-  dpdk_esp_main_t * em = &dpdk_esp_main;
-  u32 i;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  if (PREDICT_FALSE(!dcm->workers_main))
-    {
-      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-	      ESP_DECRYPT_ERROR_NO_CRYPTODEV, n_left_from);
-      vlib_buffer_free(vm, from, n_left_from);
-      return n_left_from;
-    }
-
-  crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, cpu_index);
-  u32 n_qps = vec_len(cwm->qp_data);
-  struct rte_crypto_op ** cops_to_enq[n_qps];
-  u32 n_cop_qp[n_qps], * bi_to_enq[n_qps];
-
-  for (i = 0; i < n_qps; i++)
-    {
-      bi_to_enq[i] = cwm->qp_data[i].bi;
-      cops_to_enq[i] = cwm->qp_data[i].cops;
-    }
-
-  memset(n_cop_qp, 0, n_qps * sizeof(u32));
-
-  crypto_alloc_cops();
-
-  next_index = ESP_DECRYPT_NEXT_DROP;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0, sa_index0 = ~0, seq, icv_size, iv_size;
-	  vlib_buffer_t * b0;
-	  esp_header_t * esp0;
-	  ipsec_sa_t * sa0;
-	  struct rte_mbuf * mb0 = 0;
-	  const int BLOCK_SIZE = 16;
-	  crypto_sa_session_t * sa_sess;
-	  void * sess;
-	  u16 qp_index;
-	  struct rte_crypto_op * cop = 0;
-
-	  bi0 = from[0];
-	  from += 1;
-	  n_left_from -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  esp0 = vlib_buffer_get_current (b0);
-
-	  sa_index0 = vnet_buffer(b0)->ipsec.sad_index;
-	  sa0 = pool_elt_at_index (im->sad, sa_index0);
-
-	  seq = clib_host_to_net_u32(esp0->seq);
-
-	  /* anti-replay check */
-	  if (sa0->use_anti_replay)
-	    {
-	      int rv = 0;
-
-	      if (PREDICT_TRUE(sa0->use_esn))
-		rv = esp_replay_check_esn(sa0, seq);
-	      else
-		rv = esp_replay_check(sa0, seq);
-
-	      if (PREDICT_FALSE(rv))
-		{
-		  clib_warning ("anti-replay SPI %u seq %u", sa0->spi, seq);
-		  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-					       ESP_DECRYPT_ERROR_REPLAY, 1);
-		  to_next[0] = bi0;
-		  to_next += 1;
-		  n_left_to_next -= 1;
-		  goto trace;
-		}
-	    }
-
-	  sa0->total_data_size += b0->current_length;
-
-	  if (PREDICT_FALSE(sa0->integ_alg == IPSEC_INTEG_ALG_NONE) ||
-		  PREDICT_FALSE(sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE))
-	    {
-	      clib_warning ("SPI %u : only cipher + auth supported", sa0->spi);
-	      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-					   ESP_DECRYPT_ERROR_UNSUPPORTED, 1);
-	      to_next[0] = bi0;
-	      to_next += 1;
-	      n_left_to_next -= 1;
-	      goto trace;
-	    }
-
-	  sa_sess = pool_elt_at_index(cwm->sa_sess_d[0], sa_index0);
-
-	  if (PREDICT_FALSE(!sa_sess->sess))
-	    {
-	      int ret = create_sym_sess(sa0, sa_sess, 0);
-
-	      if (PREDICT_FALSE (ret))
-		{
-		  to_next[0] = bi0;
-		  to_next += 1;
-		  n_left_to_next -= 1;
-		  goto trace;
-		}
-	    }
-
-	  sess = sa_sess->sess;
-	  qp_index = sa_sess->qp_index;
-
-	  ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0);
-	  cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops);
-	  ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
-
-	  cops_to_enq[qp_index][0] = cop;
-	  cops_to_enq[qp_index] += 1;
-	  n_cop_qp[qp_index] += 1;
-	  bi_to_enq[qp_index][0] = bi0;
-	  bi_to_enq[qp_index] += 1;
-
-	  rte_crypto_op_attach_sym_session(cop, sess);
-
-	  icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
-	  iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len;
-
-	  /* Convert vlib buffer to mbuf */
-	  mb0 = rte_mbuf_from_vlib_buffer(b0);
-	  mb0->data_len = b0->current_length;
-	  mb0->pkt_len = b0->current_length;
-	  mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data;
-
-	  /* Outer IP header has already been stripped */
-	  u16 payload_len = rte_pktmbuf_pkt_len(mb0) - sizeof (esp_header_t) -
-	      iv_size - icv_size;
-
-	  if ((payload_len & (BLOCK_SIZE - 1)) || (payload_len <= 0))
-	    {
-	      clib_warning ("payload %u not multiple of %d\n",
-			    payload_len, BLOCK_SIZE);
-	      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-					   ESP_DECRYPT_ERROR_BAD_LEN, 1);
-	      vec_add (vec_elt (cwm->qp_data, qp_index).free_cops, &cop, 1);
-	      bi_to_enq[qp_index] -= 1;
-	      cops_to_enq[qp_index] -= 1;
-	      n_cop_qp[qp_index] -= 1;
-	      to_next[0] = bi0;
-	      to_next += 1;
-	      n_left_to_next -= 1;
-	      goto trace;
-	    }
-
-	  struct rte_crypto_sym_op *sym_cop = (struct rte_crypto_sym_op *)(cop + 1);
-
-	  sym_cop->m_src = mb0;
-	  sym_cop->cipher.data.offset = sizeof (esp_header_t) + iv_size;
-	  sym_cop->cipher.data.length = payload_len;
-
-          u8 *iv = rte_pktmbuf_mtod_offset(mb0, void*, sizeof (esp_header_t));
-          dpdk_cop_priv_t * priv = (dpdk_cop_priv_t *)(sym_cop + 1);
-
-          if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
-            {
-              dpdk_gcm_cnt_blk *icb = &priv->cb;
-              icb->salt = sa0->salt;
-              clib_memcpy(icb->iv, iv, 8);
-              icb->cnt = clib_host_to_net_u32(1);
-              sym_cop->cipher.iv.data = (u8 *)icb;
-              sym_cop->cipher.iv.phys_addr = cop->phys_addr +
-		(uintptr_t)icb - (uintptr_t)cop;
-              sym_cop->cipher.iv.length = 16;
-
-              u8 *aad = priv->aad;
-              clib_memcpy(aad, iv - sizeof(esp_header_t), 8);
-              sym_cop->auth.aad.data = aad;
-              sym_cop->auth.aad.phys_addr = cop->phys_addr +
-                  (uintptr_t)aad - (uintptr_t)cop;
-              if (sa0->use_esn)
-                {
-                  *((u32*)&aad[8]) = sa0->seq_hi;
-                  sym_cop->auth.aad.length = 12;
-                }
-              else
-                {
-                  sym_cop->auth.aad.length = 8;
-                }
-
-              sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*,
-                       rte_pktmbuf_pkt_len(mb0) - icv_size);
-              sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
-                       rte_pktmbuf_pkt_len(mb0) - icv_size);
-              sym_cop->auth.digest.length = icv_size;
-
-            }
-          else
-            {
-              sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(mb0, void*,
-                       sizeof (esp_header_t));
-              sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
-                       sizeof (esp_header_t));
-              sym_cop->cipher.iv.length = iv_size;
-
-              if (sa0->use_esn)
-                {
-                  dpdk_cop_priv_t* priv = (dpdk_cop_priv_t*) (sym_cop + 1);
-                  u8* payload_end = rte_pktmbuf_mtod_offset(
-                      mb0, u8*, sizeof(esp_header_t) + iv_size + payload_len);
-
-                  clib_memcpy (priv->icv, payload_end, icv_size);
-                  *((u32*) payload_end) = sa0->seq_hi;
-                  sym_cop->auth.data.offset = 0;
-                  sym_cop->auth.data.length = sizeof(esp_header_t) + iv_size
-                      + payload_len + sizeof(sa0->seq_hi);
-                  sym_cop->auth.digest.data = priv->icv;
-                  sym_cop->auth.digest.phys_addr = cop->phys_addr
-                      + (uintptr_t) priv->icv - (uintptr_t) cop;
-                  sym_cop->auth.digest.length = icv_size;
-                }
-              else
-                {
-                  sym_cop->auth.data.offset = 0;
-                  sym_cop->auth.data.length = sizeof(esp_header_t) +
-                           iv_size + payload_len;
-
-                  sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*,
-                           rte_pktmbuf_pkt_len(mb0) - icv_size);
-                  sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
-                           rte_pktmbuf_pkt_len(mb0) - icv_size);
-                  sym_cop->auth.digest.length = icv_size;
-                }
-            }
-
-trace:
-	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      tr->crypto_alg = sa0->crypto_alg;
-	      tr->integ_alg = sa0->integ_alg;
-	    }
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-			       ESP_DECRYPT_ERROR_RX_PKTS,
-			       from_frame->n_vectors);
-  crypto_qp_data_t *qpd;
-  /* *INDENT-OFF* */
-  vec_foreach_index (i, cwm->qp_data)
-    {
-      u32 enq;
-
-      qpd = vec_elt_at_index(cwm->qp_data, i);
-      enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id,
-					qpd->cops, n_cop_qp[i]);
-      qpd->inflights += enq;
-
-      if (PREDICT_FALSE(enq < n_cop_qp[i]))
-	{
-	  crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq);
-	  vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq);
-
-	  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-				       ESP_DECRYPT_ERROR_ENQ_FAIL,
-				       n_cop_qp[i] - enq);
-	}
-    }
-  /* *INDENT-ON* */
-
-  return from_frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = {
-  .function = dpdk_esp_decrypt_node_fn,
-  .name = "dpdk-esp-decrypt",
-  .vector_size = sizeof (u32),
-  .format_trace = format_esp_decrypt_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(esp_decrypt_error_strings),
-  .error_strings = esp_decrypt_error_strings,
-
-  .n_next_nodes = ESP_DECRYPT_N_NEXT,
-  .next_nodes = {
-#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
-    foreach_esp_decrypt_next
-#undef _
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn)
-
-/*
- * Decrypt Post Node
- */
-
-#define foreach_esp_decrypt_post_error	      \
- _(PKTS, "ESP post pkts")
-
-typedef enum {
-#define _(sym,str) ESP_DECRYPT_POST_ERROR_##sym,
-  foreach_esp_decrypt_post_error
-#undef _
-  ESP_DECRYPT_POST_N_ERROR,
-} esp_decrypt_post_error_t;
-
-static char * esp_decrypt_post_error_strings[] = {
-#define _(sym,string) string,
-  foreach_esp_decrypt_post_error
-#undef _
-};
-
-vlib_node_registration_t dpdk_esp_decrypt_post_node;
-
-static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args)
-{
-  return s;
-}
-
-static uword
-dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm,
-	     vlib_node_runtime_t * node,
-	     vlib_frame_t * from_frame)
-{
-  u32 n_left_from, *from, *to_next = 0, next_index;
-  ipsec_sa_t * sa0;
-  u32 sa_index0 = ~0;
-  ipsec_main_t *im = &ipsec_main;
-  dpdk_esp_main_t *em = &dpdk_esp_main;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  esp_footer_t * f0;
-	  u32 bi0, next0, icv_size, iv_size;
-	  vlib_buffer_t * b0 = 0;
-	  ip4_header_t *ih4 = 0, *oh4 = 0;
-	  ip6_header_t *ih6 = 0, *oh6 = 0;
-	  u8 tunnel_mode = 1;
-	  u8 transport_ip6 = 0;
-
-	  next0 = ESP_DECRYPT_NEXT_DROP;
-
-	  bi0 = from[0];
-	  from += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  sa_index0 = vnet_buffer(b0)->ipsec.sad_index;
-	  sa0 = pool_elt_at_index (im->sad, sa_index0);
-
-	  to_next[0] = bi0;
-	  to_next += 1;
-
-	  icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
-	  iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len;
-
-	  if (sa0->use_anti_replay)
-	    {
-	      esp_header_t * esp0 = vlib_buffer_get_current (b0);
-	      u32 seq;
-	      seq = clib_host_to_net_u32(esp0->seq);
-	      if (PREDICT_TRUE(sa0->use_esn))
-		esp_replay_advance_esn(sa0, seq);
-	      else
-		esp_replay_advance(sa0, seq);
-	    }
-
-	  ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t));
-	  vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size);
-
-	  b0->current_length -= (icv_size + 2);
-	  b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
-	  f0 = (esp_footer_t *) ((u8 *) vlib_buffer_get_current (b0) +
-				 b0->current_length);
-	  b0->current_length -= f0->pad_length;
-
-	  /* transport mode */
-	  if (PREDICT_FALSE(!sa0->is_tunnel && !sa0->is_tunnel_ip6))
-	    {
-	      tunnel_mode = 0;
-
-	      if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) != 0x40))
-		{
-		  if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) == 0x60))
-		    transport_ip6 = 1;
-		  else
-		    {
-		      clib_warning("next header: 0x%x", f0->next_header);
-		      vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-						   ESP_DECRYPT_ERROR_NOT_IP, 1);
-		      goto trace;
-		    }
-		}
-	    }
-
-	  if (PREDICT_TRUE (tunnel_mode))
-	    {
-	      if (PREDICT_TRUE(f0->next_header == IP_PROTOCOL_IP_IN_IP))
-		next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
-	      else if (f0->next_header == IP_PROTOCOL_IPV6)
-		next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
-	      else
-		{
-		  clib_warning("next header: 0x%x", f0->next_header);
-		  vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
-					       ESP_DECRYPT_ERROR_DECRYPTION_FAILED,
-					       1);
-		  goto trace;
-		}
-	    }
-	  /* transport mode */
-	  else
-	    {
-	      if (PREDICT_FALSE(transport_ip6))
-		{
-		  ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t));
-		  vlib_buffer_advance (b0, -sizeof(ip6_header_t));
-		  oh6 = vlib_buffer_get_current (b0);
-		  memmove(oh6, ih6, sizeof(ip6_header_t));
-
-		  next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
-		  oh6->protocol = f0->next_header;
-		  oh6->payload_length =
-		      clib_host_to_net_u16 (
-			  vlib_buffer_length_in_chain(vm, b0) -
-			  sizeof (ip6_header_t));
-		}
-	      else
-		{
-		  vlib_buffer_advance (b0, -sizeof(ip4_header_t));
-		  oh4 = vlib_buffer_get_current (b0);
-		  memmove(oh4, ih4, sizeof(ip4_header_t));
-
-		  next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
-		  oh4->ip_version_and_header_length = 0x45;
-		  oh4->fragment_id = 0;
-		  oh4->flags_and_fragment_offset = 0;
-		  oh4->protocol = f0->next_header;
-		  oh4->length = clib_host_to_net_u16 (
-		      vlib_buffer_length_in_chain (vm, b0));
-		  oh4->checksum = ip4_header_checksum (oh4);
-		}
-	    }
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32)~0;
-
-trace:
-	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      tr->crypto_alg = sa0->crypto_alg;
-	      tr->integ_alg = sa0->integ_alg;
-	    }
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next, bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index,
-			       ESP_DECRYPT_POST_ERROR_PKTS,
-			       from_frame->n_vectors);
-
-  return from_frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = {
-  .function = dpdk_esp_decrypt_post_node_fn,
-  .name = "dpdk-esp-decrypt-post",
-  .vector_size = sizeof (u32),
-  .format_trace = format_esp_decrypt_post_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings),
-  .error_strings = esp_decrypt_post_error_strings,
-
-  .n_next_nodes = ESP_DECRYPT_N_NEXT,
-  .next_nodes = {
-#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
-    foreach_esp_decrypt_next
-#undef _
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn)
diff --git a/src/vnet/devices/dpdk/ipsec/esp_encrypt.c b/src/vnet/devices/dpdk/ipsec/esp_encrypt.c
deleted file mode 100644
index 6eb1afc9..00000000
--- a/src/vnet/devices/dpdk/ipsec/esp_encrypt.c
+++ /dev/null
@@ -1,609 +0,0 @@
-/*
- * esp_encrypt.c : IPSec ESP encrypt node using DPDK Cryptodev
- *
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define foreach_esp_encrypt_next                   \
-_(DROP, "error-drop")                              \
-_(IP4_LOOKUP, "ip4-lookup")                        \
-_(IP6_LOOKUP, "ip6-lookup")                        \
-_(INTERFACE_OUTPUT, "interface-output")
-
-#define _(v, s) ESP_ENCRYPT_NEXT_##v,
-typedef enum
-{
-  foreach_esp_encrypt_next
-#undef _
-    ESP_ENCRYPT_N_NEXT,
-} esp_encrypt_next_t;
-
-#define foreach_esp_encrypt_error                   \
- _(RX_PKTS, "ESP pkts received")                    \
- _(SEQ_CYCLED, "sequence number cycled")            \
- _(ENQ_FAIL, "Enqueue failed (buffer full)")        \
- _(NO_CRYPTODEV, "Cryptodev not configured")        \
- _(UNSUPPORTED, "Cipher/Auth not supported")
-
-
-typedef enum
-{
-#define _(sym,str) ESP_ENCRYPT_ERROR_##sym,
-  foreach_esp_encrypt_error
-#undef _
-    ESP_ENCRYPT_N_ERROR,
-} esp_encrypt_error_t;
-
-static char *esp_encrypt_error_strings[] = {
-#define _(sym,string) string,
-  foreach_esp_encrypt_error
-#undef _
-};
-
-vlib_node_registration_t dpdk_esp_encrypt_node;
-
-typedef struct
-{
-  u32 spi;
-  u32 seq;
-  ipsec_crypto_alg_t crypto_alg;
-  ipsec_integ_alg_t integ_alg;
-} esp_encrypt_trace_t;
-
-/* packet trace format function */
-static u8 *
-format_esp_encrypt_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *);
-
-  s = format (s, "esp: spi %u seq %u crypto %U integrity %U",
-	      t->spi, t->seq,
-	      format_ipsec_crypto_alg, t->crypto_alg,
-	      format_ipsec_integ_alg, t->integ_alg);
-  return s;
-}
-
-static uword
-dpdk_esp_encrypt_node_fn (vlib_main_t * vm,
-			  vlib_node_runtime_t * node,
-			  vlib_frame_t * from_frame)
-{
-  u32 n_left_from, *from, *to_next, next_index;
-  ipsec_main_t *im = &ipsec_main;
-  u32 cpu_index = os_get_cpu_number ();
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  dpdk_esp_main_t *em = &dpdk_esp_main;
-  u32 i;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  if (PREDICT_FALSE (!dcm->workers_main))
-    {
-      /* Likely there are not enough cryptodevs, so drop frame */
-      vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
-				   ESP_ENCRYPT_ERROR_NO_CRYPTODEV,
-				   n_left_from);
-      vlib_buffer_free (vm, from, n_left_from);
-      return n_left_from;
-    }
-
-  crypto_worker_main_t *cwm = vec_elt_at_index (dcm->workers_main, cpu_index);
-  u32 n_qps = vec_len (cwm->qp_data);
-  struct rte_crypto_op **cops_to_enq[n_qps];
-  u32 n_cop_qp[n_qps], *bi_to_enq[n_qps];
-
-  for (i = 0; i < n_qps; i++)
-    {
-      bi_to_enq[i] = cwm->qp_data[i].bi;
-      cops_to_enq[i] = cwm->qp_data[i].cops;
-    }
-
-  memset (n_cop_qp, 0, n_qps * sizeof (u32));
-
-  crypto_alloc_cops ();
-
-  next_index = ESP_ENCRYPT_NEXT_DROP;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0, next0;
-	  vlib_buffer_t *b0 = 0;
-	  u32 sa_index0;
-	  ipsec_sa_t *sa0;
-	  ip4_and_esp_header_t *ih0, *oh0 = 0;
-	  ip6_and_esp_header_t *ih6_0, *oh6_0 = 0;
-	  struct rte_mbuf *mb0 = 0;
-	  esp_footer_t *f0;
-	  u8 is_ipv6;
-	  u8 ip_hdr_size;
-	  u8 next_hdr_type;
-	  u8 transport_mode = 0;
-	  const int BLOCK_SIZE = 16;
-	  u32 iv_size;
-	  u16 orig_sz;
-	  crypto_sa_session_t *sa_sess;
-	  void *sess;
-	  struct rte_crypto_op *cop = 0;
-	  u16 qp_index;
-
-	  bi0 = from[0];
-	  from += 1;
-	  n_left_from -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  sa_index0 = vnet_buffer (b0)->ipsec.sad_index;
-	  sa0 = pool_elt_at_index (im->sad, sa_index0);
-
-	  if (PREDICT_FALSE (esp_seq_advance (sa0)))
-	    {
-	      clib_warning ("sequence number counter has cycled SPI %u",
-			    sa0->spi);
-	      vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
-					   ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1);
-	      //TODO: rekey SA
-	      to_next[0] = bi0;
-	      to_next += 1;
-	      n_left_to_next -= 1;
-	      goto trace;
-	    }
-
-	  sa0->total_data_size += b0->current_length;
-
-	  sa_sess = pool_elt_at_index (cwm->sa_sess_d[1], sa_index0);
-	  if (PREDICT_FALSE (!sa_sess->sess))
-	    {
-	      int ret = create_sym_sess (sa0, sa_sess, 1);
-
-	      if (PREDICT_FALSE (ret))
-		{
-		  to_next[0] = bi0;
-		  to_next += 1;
-		  n_left_to_next -= 1;
-		  goto trace;
-		}
-	    }
-
-	  qp_index = sa_sess->qp_index;
-	  sess = sa_sess->sess;
-
-	  ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0);
-	  cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops);
-	  ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
-
-	  cops_to_enq[qp_index][0] = cop;
-	  cops_to_enq[qp_index] += 1;
-	  n_cop_qp[qp_index] += 1;
-	  bi_to_enq[qp_index][0] = bi0;
-	  bi_to_enq[qp_index] += 1;
-
-	  ssize_t adv;
-	  iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len;
-	  ih0 = vlib_buffer_get_current (b0);
-	  orig_sz = b0->current_length;
-	  is_ipv6 = (ih0->ip4.ip_version_and_header_length & 0xF0) == 0x60;
-	  /* is ipv6 */
-	  if (PREDICT_TRUE (sa0->is_tunnel))
-	    {
-	      if (PREDICT_TRUE (!is_ipv6))
-		adv = -sizeof (ip4_and_esp_header_t);
-	      else
-		adv = -sizeof (ip6_and_esp_header_t);
-	    }
-	  else
-	    {
-	      adv = -sizeof (esp_header_t);
-	      if (PREDICT_TRUE (!is_ipv6))
-		orig_sz -= sizeof (ip4_header_t);
-	      else
-		orig_sz -= sizeof (ip6_header_t);
-	    }
-
-	  /*transport mode save the eth header before it is overwritten */
-	  if (PREDICT_FALSE (!sa0->is_tunnel))
-	    {
-	      ethernet_header_t *ieh0 = (ethernet_header_t *)
-		((u8 *) vlib_buffer_get_current (b0) -
-		 sizeof (ethernet_header_t));
-	      ethernet_header_t *oeh0 =
-		(ethernet_header_t *) ((u8 *) ieh0 + (adv - iv_size));
-	      clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t));
-	    }
-
-	  vlib_buffer_advance (b0, adv - iv_size);
-
-	  /* XXX IP6/ip4 and IP4/IP6 not supported, only IP4/IP4 and IP6/IP6 */
-
-	  /* is ipv6 */
-	  if (PREDICT_FALSE (is_ipv6))
-	    {
-	      ih6_0 = (ip6_and_esp_header_t *) ih0;
-	      ip_hdr_size = sizeof (ip6_header_t);
-	      oh6_0 = vlib_buffer_get_current (b0);
-
-	      if (PREDICT_TRUE (sa0->is_tunnel))
-		{
-		  next_hdr_type = IP_PROTOCOL_IPV6;
-		  oh6_0->ip6.ip_version_traffic_class_and_flow_label =
-		    ih6_0->ip6.ip_version_traffic_class_and_flow_label;
-		}
-	      else
-		{
-		  next_hdr_type = ih6_0->ip6.protocol;
-		  memmove (oh6_0, ih6_0, sizeof (ip6_header_t));
-		}
-
-	      oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP;
-	      oh6_0->ip6.hop_limit = 254;
-	      oh6_0->esp.spi = clib_net_to_host_u32 (sa0->spi);
-	      oh6_0->esp.seq = clib_net_to_host_u32 (sa0->seq);
-	    }
-	  else
-	    {
-	      ip_hdr_size = sizeof (ip4_header_t);
-	      oh0 = vlib_buffer_get_current (b0);
-
-	      if (PREDICT_TRUE (sa0->is_tunnel))
-		{
-		  next_hdr_type = IP_PROTOCOL_IP_IN_IP;
-		  oh0->ip4.tos = ih0->ip4.tos;
-		}
-	      else
-		{
-		  next_hdr_type = ih0->ip4.protocol;
-		  memmove (oh0, ih0, sizeof (ip4_header_t));
-		}
-
-	      oh0->ip4.ip_version_and_header_length = 0x45;
-	      oh0->ip4.fragment_id = 0;
-	      oh0->ip4.flags_and_fragment_offset = 0;
-	      oh0->ip4.ttl = 254;
-	      oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
-	      oh0->esp.spi = clib_net_to_host_u32 (sa0->spi);
-	      oh0->esp.seq = clib_net_to_host_u32 (sa0->seq);
-	    }
-
-	  if (PREDICT_TRUE (sa0->is_tunnel && !sa0->is_tunnel_ip6))
-	    {
-	      oh0->ip4.src_address.as_u32 = sa0->tunnel_src_addr.ip4.as_u32;
-	      oh0->ip4.dst_address.as_u32 = sa0->tunnel_dst_addr.ip4.as_u32;
-
-	      /* in tunnel mode send it back to FIB */
-	      next0 = ESP_ENCRYPT_NEXT_IP4_LOOKUP;
-	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	    }
-	  else if (sa0->is_tunnel && sa0->is_tunnel_ip6)
-	    {
-	      oh6_0->ip6.src_address.as_u64[0] =
-		sa0->tunnel_src_addr.ip6.as_u64[0];
-	      oh6_0->ip6.src_address.as_u64[1] =
-		sa0->tunnel_src_addr.ip6.as_u64[1];
-	      oh6_0->ip6.dst_address.as_u64[0] =
-		sa0->tunnel_dst_addr.ip6.as_u64[0];
-	      oh6_0->ip6.dst_address.as_u64[1] =
-		sa0->tunnel_dst_addr.ip6.as_u64[1];
-
-	      /* in tunnel mode send it back to FIB */
-	      next0 = ESP_ENCRYPT_NEXT_IP6_LOOKUP;
-	      vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	    }
-	  else
-	    {
-	      next0 = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT;
-	      transport_mode = 1;
-	    }
-
-	  ASSERT (sa0->crypto_alg < IPSEC_CRYPTO_N_ALG);
-	  ASSERT (sa0->crypto_alg != IPSEC_CRYPTO_ALG_NONE);
-
-	  int blocks = 1 + (orig_sz + 1) / BLOCK_SIZE;
-
-	  /* pad packet in input buffer */
-	  u8 pad_bytes = BLOCK_SIZE * blocks - 2 - orig_sz;
-	  u8 i;
-	  u8 *padding = vlib_buffer_get_current (b0) + b0->current_length;
-
-	  for (i = 0; i < pad_bytes; ++i)
-	    padding[i] = i + 1;
-
-	  f0 = vlib_buffer_get_current (b0) + b0->current_length + pad_bytes;
-	  f0->pad_length = pad_bytes;
-	  f0->next_header = next_hdr_type;
-	  b0->current_length += pad_bytes + 2 +
-	    em->esp_integ_algs[sa0->integ_alg].trunc_size;
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
-	    vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
-
-	  struct rte_crypto_sym_op *sym_cop;
-	  sym_cop = (struct rte_crypto_sym_op *) (cop + 1);
-
-	  dpdk_cop_priv_t *priv = (dpdk_cop_priv_t *) (sym_cop + 1);
-
-	  vnet_buffer (b0)->unused[0] = next0;
-
-	  mb0 = rte_mbuf_from_vlib_buffer (b0);
-	  mb0->data_len = b0->current_length;
-	  mb0->pkt_len = b0->current_length;
-	  mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data;
-
-	  rte_crypto_op_attach_sym_session (cop, sess);
-
-	  sym_cop->m_src = mb0;
-
-	  dpdk_gcm_cnt_blk *icb = &priv->cb;
-	  icb->salt = sa0->salt;
-	  icb->iv[0] = sa0->seq;
-	  icb->iv[1] = sa0->seq_hi;
-
-	  if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
-	    {
-	      icb->cnt = clib_host_to_net_u32 (1);
-	      clib_memcpy (vlib_buffer_get_current (b0) + ip_hdr_size +
-			   sizeof (esp_header_t), icb->iv, 8);
-	      sym_cop->cipher.data.offset =
-		ip_hdr_size + sizeof (esp_header_t) + iv_size;
-	      sym_cop->cipher.data.length = BLOCK_SIZE * blocks;
-	      sym_cop->cipher.iv.length = 16;
-	    }
-	  else
-	    {
-	      sym_cop->cipher.data.offset =
-		ip_hdr_size + sizeof (esp_header_t);
-	      sym_cop->cipher.data.length = BLOCK_SIZE * blocks + iv_size;
-	      sym_cop->cipher.iv.length = iv_size;
-	    }
-
-	  sym_cop->cipher.iv.data = (u8 *) icb;
-	  sym_cop->cipher.iv.phys_addr = cop->phys_addr + (uintptr_t) icb
-	    - (uintptr_t) cop;
-
-
-	  ASSERT (sa0->integ_alg < IPSEC_INTEG_N_ALG);
-	  ASSERT (sa0->integ_alg != IPSEC_INTEG_ALG_NONE);
-
-	  if (PREDICT_FALSE (sa0->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128))
-	    {
-	      u8 *aad = priv->aad;
-	      clib_memcpy (aad, vlib_buffer_get_current (b0) + ip_hdr_size,
-			   8);
-	      sym_cop->auth.aad.data = aad;
-	      sym_cop->auth.aad.phys_addr = cop->phys_addr +
-		(uintptr_t) aad - (uintptr_t) cop;
-
-	      if (PREDICT_FALSE (sa0->use_esn))
-		{
-		  *((u32 *) & aad[8]) = sa0->seq_hi;
-		  sym_cop->auth.aad.length = 12;
-		}
-	      else
-		{
-		  sym_cop->auth.aad.length = 8;
-		}
-	    }
-	  else
-	    {
-	      sym_cop->auth.data.offset = ip_hdr_size;
-	      sym_cop->auth.data.length = b0->current_length - ip_hdr_size
-		- em->esp_integ_algs[sa0->integ_alg].trunc_size;
-
-	      if (PREDICT_FALSE (sa0->use_esn))
-		{
-		  u8 *payload_end =
-		    vlib_buffer_get_current (b0) + b0->current_length;
-		  *((u32 *) payload_end) = sa0->seq_hi;
-		  sym_cop->auth.data.length += sizeof (sa0->seq_hi);
-		}
-	    }
-	  sym_cop->auth.digest.data = vlib_buffer_get_current (b0) +
-	    b0->current_length -
-	    em->esp_integ_algs[sa0->integ_alg].trunc_size;
-	  sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset (mb0,
-								       b0->current_length
-								       -
-								       em->esp_integ_algs
-								       [sa0->integ_alg].trunc_size);
-	  sym_cop->auth.digest.length =
-	    em->esp_integ_algs[sa0->integ_alg].trunc_size;
-
-
-	  if (PREDICT_FALSE (is_ipv6))
-	    {
-	      oh6_0->ip6.payload_length =
-		clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
-				      sizeof (ip6_header_t));
-	    }
-	  else
-	    {
-	      oh0->ip4.length =
-		clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
-	      oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4);
-	    }
-
-	  if (transport_mode)
-	    vlib_buffer_advance (b0, -sizeof (ethernet_header_t));
-
-	trace:
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      esp_encrypt_trace_t *tr =
-		vlib_add_trace (vm, node, b0, sizeof (*tr));
-	      tr->spi = sa0->spi;
-	      tr->seq = sa0->seq - 1;
-	      tr->crypto_alg = sa0->crypto_alg;
-	      tr->integ_alg = sa0->integ_alg;
-	    }
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
-			       ESP_ENCRYPT_ERROR_RX_PKTS,
-			       from_frame->n_vectors);
-  crypto_qp_data_t *qpd;
-  /* *INDENT-OFF* */
-  vec_foreach_index (i, cwm->qp_data)
-    {
-      u32 enq;
-
-      qpd = vec_elt_at_index(cwm->qp_data, i);
-      enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id,
-					qpd->cops, n_cop_qp[i]);
-      qpd->inflights += enq;
-
-      if (PREDICT_FALSE(enq < n_cop_qp[i]))
-	{
-	  crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq);
-	  vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq);
-
-          vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index,
-				       ESP_ENCRYPT_ERROR_ENQ_FAIL,
-				       n_cop_qp[i] - enq);
-        }
-    }
-  /* *INDENT-ON* */
-
-  return from_frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (dpdk_esp_encrypt_node) =
-{
-  .function = dpdk_esp_encrypt_node_fn,.name = "dpdk-esp-encrypt",.flags =
-    VLIB_NODE_FLAG_IS_OUTPUT,.vector_size = sizeof (u32),.format_trace =
-    format_esp_encrypt_trace,.n_errors =
-    ARRAY_LEN (esp_encrypt_error_strings),.error_strings =
-    esp_encrypt_error_strings,.n_next_nodes = 1,.next_nodes =
-  {
-  [ESP_ENCRYPT_NEXT_DROP] = "error-drop",}
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_encrypt_node, dpdk_esp_encrypt_node_fn)
-/*
- * ESP Encrypt Post Node
- */
-#define foreach_esp_encrypt_post_error              \
- _(PKTS, "ESP post pkts")
-     typedef enum
-     {
-#define _(sym,str) ESP_ENCRYPT_POST_ERROR_##sym,
-       foreach_esp_encrypt_post_error
-#undef _
-	 ESP_ENCRYPT_POST_N_ERROR,
-     } esp_encrypt_post_error_t;
-
-     static char *esp_encrypt_post_error_strings[] = {
-#define _(sym,string) string,
-       foreach_esp_encrypt_post_error
-#undef _
-     };
-
-vlib_node_registration_t dpdk_esp_encrypt_post_node;
-
-static u8 *
-format_esp_encrypt_post_trace (u8 * s, va_list * args)
-{
-  return s;
-}
-
-static uword
-dpdk_esp_encrypt_post_node_fn (vlib_main_t * vm,
-			       vlib_node_runtime_t * node,
-			       vlib_frame_t * from_frame)
-{
-  u32 n_left_from, *from, *to_next = 0, next_index;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0, next0;
-	  vlib_buffer_t *b0 = 0;
-
-	  bi0 = from[0];
-	  from += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  to_next[0] = bi0;
-	  to_next += 1;
-
-	  next0 = vnet_buffer (b0)->unused[0];
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next, bi0,
-					   next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, dpdk_esp_encrypt_post_node.index,
-			       ESP_ENCRYPT_POST_ERROR_PKTS,
-			       from_frame->n_vectors);
-
-  return from_frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (dpdk_esp_encrypt_post_node) =
-{
-  .function = dpdk_esp_encrypt_post_node_fn,.name =
-    "dpdk-esp-encrypt-post",.vector_size = sizeof (u32),.format_trace =
-    format_esp_encrypt_post_trace,.type = VLIB_NODE_TYPE_INTERNAL,.n_errors =
-    ARRAY_LEN (esp_encrypt_post_error_strings),.error_strings =
-    esp_encrypt_post_error_strings,.n_next_nodes =
-    ESP_ENCRYPT_N_NEXT,.next_nodes =
-  {
-#define _(s,n) [ESP_ENCRYPT_NEXT_##s] = n,
-    foreach_esp_encrypt_next
-#undef _
-  }
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_encrypt_post_node,
-			      dpdk_esp_encrypt_post_node_fn)
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/ipsec/ipsec.c b/src/vnet/devices/dpdk/ipsec/ipsec.c
deleted file mode 100644
index 05c17c99..00000000
--- a/src/vnet/devices/dpdk/ipsec/ipsec.c
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-#define DPDK_CRYPTO_NB_SESS_OBJS  20000
-#define DPDK_CRYPTO_CACHE_SIZE	  512
-#define DPDK_CRYPTO_PRIV_SIZE	  128
-#define DPDK_CRYPTO_N_QUEUE_DESC  1024
-#define DPDK_CRYPTO_NB_COPS	  (1024 * 4)
-
-static int
-add_del_sa_sess (u32 sa_index, u8 is_add)
-{
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  crypto_worker_main_t *cwm;
-  u8 skip_master = vlib_num_workers () > 0;
-
-  /* *INDENT-OFF* */
-  vec_foreach (cwm, dcm->workers_main)
-    {
-      crypto_sa_session_t *sa_sess;
-      u8 is_outbound;
-
-      if (skip_master)
-	{
-	  skip_master = 0;
-	  continue;
-	}
-
-      for (is_outbound = 0; is_outbound < 2; is_outbound++)
-	{
-	  if (is_add)
-	    {
-	      pool_get (cwm->sa_sess_d[is_outbound], sa_sess);
-	    }
-	  else
-	    {
-	      u8 dev_id;
-
-	      sa_sess = pool_elt_at_index (cwm->sa_sess_d[is_outbound], sa_index);
-	      dev_id = cwm->qp_data[sa_sess->qp_index].dev_id;
-
-	      if (!sa_sess->sess)
-		continue;
-
-	      if (rte_cryptodev_sym_session_free(dev_id, sa_sess->sess))
-		{
-		  clib_warning("failed to free session");
-		  return -1;
-		}
-	      memset(sa_sess, 0, sizeof(sa_sess[0]));
-	    }
-	}
-    }
-  /* *INDENT-OFF* */
-
-  return 0;
-}
-
-static void
-update_qp_data (crypto_worker_main_t * cwm,
-		u8 cdev_id, u16 qp_id, u8 is_outbound, u16 * idx)
-{
-  crypto_qp_data_t *qpd;
-
-  /* *INDENT-OFF* */
-  vec_foreach_index (*idx, cwm->qp_data)
-    {
-      qpd = vec_elt_at_index(cwm->qp_data, *idx);
-
-      if (qpd->dev_id == cdev_id && qpd->qp_id == qp_id &&
-	  qpd->is_outbound == is_outbound)
-	  return;
-    }
-  /* *INDENT-ON* */
-
-  vec_add2 (cwm->qp_data, qpd, 1);
-
-  qpd->dev_id = cdev_id;
-  qpd->qp_id = qp_id;
-  qpd->is_outbound = is_outbound;
-}
-
-/*
- * return:
- * 	0: already exist
- * 	1: mapped
- */
-static int
-add_mapping (crypto_worker_main_t * cwm,
-	     u8 cdev_id, u16 qp, u8 is_outbound,
-	     const struct rte_cryptodev_capabilities *cipher_cap,
-	     const struct rte_cryptodev_capabilities *auth_cap)
-{
-  u16 qp_index;
-  uword key = 0, data, *ret;
-  crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
-
-  p_key->cipher_algo = (u8) cipher_cap->sym.cipher.algo;
-  p_key->auth_algo = (u8) auth_cap->sym.auth.algo;
-  p_key->is_outbound = is_outbound;
-
-  ret = hash_get (cwm->algo_qp_map, key);
-  if (ret)
-    return 0;
-
-  update_qp_data (cwm, cdev_id, qp, is_outbound, &qp_index);
-
-  data = (uword) qp_index;
-  hash_set (cwm->algo_qp_map, key, data);
-
-  return 1;
-}
-
-/*
- * return:
- * 	0: already exist
- * 	1: mapped
- */
-static int
-add_cdev_mapping (crypto_worker_main_t * cwm,
-		  struct rte_cryptodev_info *dev_info, u8 cdev_id,
-		  u16 qp, u8 is_outbound)
-{
-  const struct rte_cryptodev_capabilities *i, *j;
-  u32 mapped = 0;
-
-  for (i = dev_info->capabilities; i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++)
-    {
-      if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
-	continue;
-
-      if (check_algo_is_supported (i, NULL) != 0)
-	continue;
-
-      for (j = dev_info->capabilities; j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED;
-	   j++)
-	{
-	  if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
-	    continue;
-
-	  if (check_algo_is_supported (j, NULL) != 0)
-	    continue;
-
-	  mapped |= add_mapping (cwm, cdev_id, qp, is_outbound, i, j);
-	}
-    }
-
-  return mapped;
-}
-
-static int
-check_cryptodev_queues ()
-{
-  u32 n_qs = 0;
-  u8 cdev_id;
-  u32 n_req_qs = 2;
-
-  if (vlib_num_workers () > 0)
-    n_req_qs = vlib_num_workers () * 2;
-
-  for (cdev_id = 0; cdev_id < rte_cryptodev_count (); cdev_id++)
-    {
-      struct rte_cryptodev_info cdev_info;
-
-      rte_cryptodev_info_get (cdev_id, &cdev_info);
-
-      if (!
-	  (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
-	continue;
-
-      n_qs += cdev_info.max_nb_queue_pairs;
-    }
-
-  if (n_qs >= n_req_qs)
-    return 0;
-  else
-    return -1;
-}
-
-static clib_error_t *
-dpdk_ipsec_check_support (ipsec_sa_t * sa)
-{
-  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
-    {
-      if (sa->integ_alg != IPSEC_INTEG_ALG_NONE)
-	return clib_error_return (0, "unsupported integ-alg %U with "
-				  "crypto-algo aes-gcm-128",
-				  format_ipsec_integ_alg, sa->integ_alg);
-      sa->integ_alg = IPSEC_INTEG_ALG_AES_GCM_128;
-    }
-  else
-    {
-      if (sa->integ_alg == IPSEC_INTEG_ALG_NONE ||
-	  sa->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128)
-	return clib_error_return (0, "unsupported integ-alg %U",
-				  format_ipsec_integ_alg, sa->integ_alg);
-    }
-
-  return 0;
-}
-
-static uword
-dpdk_ipsec_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
-		    vlib_frame_t * f)
-{
-  dpdk_config_main_t *conf = &dpdk_config_main;
-  ipsec_main_t *im = &ipsec_main;
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  struct rte_cryptodev_config dev_conf;
-  struct rte_cryptodev_qp_conf qp_conf;
-  struct rte_cryptodev_info cdev_info;
-  struct rte_mempool *rmp;
-  i32 dev_id, ret;
-  u32 i, skip_master;
-
-  if (!conf->cryptodev)
-    {
-      clib_warning ("DPDK Cryptodev support is disabled, "
-		    "default to OpenSSL IPsec");
-      return 0;
-    }
-
-  if (check_cryptodev_queues () < 0)
-    {
-      conf->cryptodev = 0;
-      clib_warning ("not enough Cryptodevs, default to OpenSSL IPsec");
-      return 0;
-    }
-
-  vec_alloc (dcm->workers_main, tm->n_vlib_mains);
-  _vec_len (dcm->workers_main) = tm->n_vlib_mains;
-
-  fprintf (stdout, "DPDK Cryptodevs info:\n");
-  fprintf (stdout, "dev_id\tn_qp\tnb_obj\tcache_size\n");
-  /* HW cryptodevs have higher dev_id, use HW first */
-  for (dev_id = rte_cryptodev_count () - 1; dev_id >= 0; dev_id--)
-    {
-      u16 max_nb_qp, qp = 0;
-      skip_master = vlib_num_workers () > 0;
-
-      rte_cryptodev_info_get (dev_id, &cdev_info);
-
-      if (!
-	  (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
-	continue;
-
-      max_nb_qp = cdev_info.max_nb_queue_pairs;
-
-      for (i = 0; i < tm->n_vlib_mains; i++)
-	{
-	  u8 is_outbound;
-	  crypto_worker_main_t *cwm;
-	  uword *map;
-
-	  if (skip_master)
-	    {
-	      skip_master = 0;
-	      continue;
-	    }
-
-	  cwm = vec_elt_at_index (dcm->workers_main, i);
-	  map = cwm->algo_qp_map;
-
-	  if (!map)
-	    {
-	      map = hash_create (0, sizeof (crypto_worker_qp_key_t));
-	      if (!map)
-		{
-		  clib_warning ("unable to create hash table for worker %u",
-				vlib_mains[i]->cpu_index);
-		  goto error;
-		}
-	      cwm->algo_qp_map = map;
-	    }
-
-	  for (is_outbound = 0; is_outbound < 2 && qp < max_nb_qp;
-	       is_outbound++)
-	    qp += add_cdev_mapping (cwm, &cdev_info, dev_id, qp, is_outbound);
-	}
-
-      if (qp == 0)
-	continue;
-
-      dev_conf.socket_id = rte_cryptodev_socket_id (dev_id);
-      dev_conf.nb_queue_pairs = cdev_info.max_nb_queue_pairs;
-      dev_conf.session_mp.nb_objs = DPDK_CRYPTO_NB_SESS_OBJS;
-      dev_conf.session_mp.cache_size = DPDK_CRYPTO_CACHE_SIZE;
-
-      ret = rte_cryptodev_configure (dev_id, &dev_conf);
-      if (ret < 0)
-	{
-	  clib_warning ("cryptodev %u config error", dev_id);
-	  goto error;
-	}
-
-      qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
-      for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++)
-	{
-	  ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf,
-						dev_conf.socket_id);
-	  if (ret < 0)
-	    {
-	      clib_warning ("cryptodev %u qp %u setup error", dev_id, qp);
-	      goto error;
-	    }
-	}
-      vec_validate_aligned (dcm->cop_pools, dev_conf.socket_id,
-			    CLIB_CACHE_LINE_BYTES);
-
-      if (!vec_elt (dcm->cop_pools, dev_conf.socket_id))
-	{
-	  u8 *pool_name = format (0, "crypto_op_pool_socket%u%c",
-				  dev_conf.socket_id, 0);
-
-	  rmp = rte_crypto_op_pool_create ((char *) pool_name,
-					   RTE_CRYPTO_OP_TYPE_SYMMETRIC,
-					   DPDK_CRYPTO_NB_COPS *
-					   (1 + vlib_num_workers ()),
-					   DPDK_CRYPTO_CACHE_SIZE,
-					   DPDK_CRYPTO_PRIV_SIZE,
-					   dev_conf.socket_id);
-	  vec_free (pool_name);
-
-	  if (!rmp)
-	    {
-	      clib_warning ("failed to allocate mempool on socket %u",
-			    dev_conf.socket_id);
-	      goto error;
-	    }
-	  vec_elt (dcm->cop_pools, dev_conf.socket_id) = rmp;
-	}
-
-      fprintf (stdout, "%u\t%u\t%u\t%u\n", dev_id, dev_conf.nb_queue_pairs,
-	       DPDK_CRYPTO_NB_SESS_OBJS, DPDK_CRYPTO_CACHE_SIZE);
-    }
-
-  dpdk_esp_init ();
-
-  /* Add new next node and set as default */
-  vlib_node_t *node, *next_node;
-
-  next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-encrypt");
-  ASSERT (next_node);
-  node = vlib_get_node_by_name (vm, (u8 *) "ipsec-output-ip4");
-  ASSERT (node);
-  im->esp_encrypt_node_index = next_node->index;
-  im->esp_encrypt_next_index =
-    vlib_node_add_next (vm, node->index, next_node->index);
-
-  next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-decrypt");
-  ASSERT (next_node);
-  node = vlib_get_node_by_name (vm, (u8 *) "ipsec-input-ip4");
-  ASSERT (node);
-  im->esp_decrypt_node_index = next_node->index;
-  im->esp_decrypt_next_index =
-    vlib_node_add_next (vm, node->index, next_node->index);
-
-  im->cb.check_support_cb = dpdk_ipsec_check_support;
-  im->cb.add_del_sa_sess_cb = add_del_sa_sess;
-
-  if (vec_len (vlib_mains) == 0)
-    vlib_node_set_state (&vlib_global_main, dpdk_crypto_input_node.index,
-			 VLIB_NODE_STATE_POLLING);
-  else
-    for (i = 1; i < tm->n_vlib_mains; i++)
-      vlib_node_set_state (vlib_mains[i], dpdk_crypto_input_node.index,
-			   VLIB_NODE_STATE_POLLING);
-
-  /* TODO cryptodev counters */
-
-  return 0;
-
-error:
-  ;
-  crypto_worker_main_t *cwm;
-  struct rte_mempool **mp;
-  /* *INDENT-OFF* */
-  vec_foreach (cwm, dcm->workers_main)
-    hash_free (cwm->algo_qp_map);
-
-  vec_foreach (mp, dcm->cop_pools)
-    {
-      if (mp)
-	rte_mempool_free (mp[0]);
-    }
-  /* *INDENT-ON* */
-  vec_free (dcm->workers_main);
-  vec_free (dcm->cop_pools);
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
-    .function = dpdk_ipsec_process,
-    .type = VLIB_NODE_TYPE_PROCESS,
-    .name = "dpdk-ipsec-process",
-    .process_log2_n_stack_bytes = 17,
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/ipsec/ipsec.h b/src/vnet/devices/dpdk/ipsec/ipsec.h
deleted file mode 100644
index 3465b361..00000000
--- a/src/vnet/devices/dpdk/ipsec/ipsec.h
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Copyright (c) 2016 Intel and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __DPDK_IPSEC_H__
-#define __DPDK_IPSEC_H__
-
-#include 
-
-#undef always_inline
-#include 
-#include 
-
-#if CLIB_DEBUG > 0
-#define always_inline static inline
-#else
-#define always_inline static inline __attribute__ ((__always_inline__))
-#endif
-
-
-#define MAX_QP_PER_LCORE 16
-
-typedef struct
-{
-  u32 salt;
-  u32 iv[2];
-  u32 cnt;
-} dpdk_gcm_cnt_blk;
-
-typedef struct
-{
-  dpdk_gcm_cnt_blk cb;
-  union
-  {
-    u8 aad[12];
-    u8 icv[64];
-  };
-} dpdk_cop_priv_t;
-
-typedef struct
-{
-  u8 cipher_algo;
-  u8 auth_algo;
-  u8 is_outbound;
-} crypto_worker_qp_key_t;
-
-typedef struct
-{
-  u16 dev_id;
-  u16 qp_id;
-  u16 is_outbound;
-  i16 inflights;
-  u32 bi[VLIB_FRAME_SIZE];
-  struct rte_crypto_op *cops[VLIB_FRAME_SIZE];
-  struct rte_crypto_op **free_cops;
-} crypto_qp_data_t;
-
-typedef struct
-{
-  u8 qp_index;
-  void *sess;
-} crypto_sa_session_t;
-
-typedef struct
-{
-  crypto_sa_session_t *sa_sess_d[2];
-  crypto_qp_data_t *qp_data;
-  uword *algo_qp_map;
-} crypto_worker_main_t;
-
-typedef struct
-{
-  struct rte_mempool **cop_pools;
-  crypto_worker_main_t *workers_main;
-} dpdk_crypto_main_t;
-
-dpdk_crypto_main_t dpdk_crypto_main;
-
-extern vlib_node_registration_t dpdk_crypto_input_node;
-
-#define CRYPTO_N_FREE_COPS (VLIB_FRAME_SIZE * 3)
-
-static_always_inline void
-crypto_alloc_cops ()
-{
-  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  u32 cpu_index = os_get_cpu_number ();
-  crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index];
-  unsigned socket_id = rte_socket_id ();
-  crypto_qp_data_t *qpd;
-
-  /* *INDENT-OFF* */
-  vec_foreach (qpd, cwm->qp_data)
-    {
-      u32 l = vec_len (qpd->free_cops);
-
-      if (PREDICT_FALSE (l < VLIB_FRAME_SIZE))
-	{
-	  u32 n_alloc;
-
-	  if (PREDICT_FALSE (!qpd->free_cops))
-	    vec_alloc (qpd->free_cops, CRYPTO_N_FREE_COPS);
-
-	  n_alloc = rte_crypto_op_bulk_alloc (dcm->cop_pools[socket_id],
-					      RTE_CRYPTO_OP_TYPE_SYMMETRIC,
-					      &qpd->free_cops[l],
-					      CRYPTO_N_FREE_COPS - l - 1);
-
-	  _vec_len (qpd->free_cops) = l + n_alloc;
-	}
-    }
-  /* *INDENT-ON* */
-}
-
-static_always_inline void
-crypto_free_cop (crypto_qp_data_t * qpd, struct rte_crypto_op **cops, u32 n)
-{
-  u32 l = vec_len (qpd->free_cops);
-
-  if (l + n >= CRYPTO_N_FREE_COPS)
-    {
-      l -= VLIB_FRAME_SIZE;
-      rte_mempool_put_bulk (cops[0]->mempool,
-			    (void **) &qpd->free_cops[l], VLIB_FRAME_SIZE);
-    }
-  clib_memcpy (&qpd->free_cops[l], cops, sizeof (*cops) * n);
-
-  _vec_len (qpd->free_cops) = l + n;
-}
-
-static_always_inline int
-check_algo_is_supported (const struct rte_cryptodev_capabilities *cap,
-			 char *name)
-{
-  struct
-  {
-    uint8_t cipher_algo;
-    enum rte_crypto_sym_xform_type type;
-    union
-    {
-      enum rte_crypto_auth_algorithm auth;
-      enum rte_crypto_cipher_algorithm cipher;
-    };
-    char *name;
-  } supported_algo[] =
-  {
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_NULL,.name = "NULL"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_AES_CBC,.name = "AES_CBC"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_AES_CTR,.name = "AES_CTR"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_3DES_CBC,.name = "3DES-CBC"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_AES_GCM,.name = "AES-GCM"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA1_HMAC,.name = "HMAC-SHA1"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA256_HMAC,.name = "HMAC-SHA256"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA384_HMAC,.name = "HMAC-SHA384"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA512_HMAC,.name = "HMAC-SHA512"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_AES_XCBC_MAC,.name = "AES-XCBC-MAC"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_AES_GCM,.name = "AES-GCM"},
-    {
-      /* tail */
-  .type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED},};
-  uint32_t i = 0;
-
-  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
-    return -1;
-
-  while (supported_algo[i].type != RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED)
-    {
-      if (cap->sym.xform_type == supported_algo[i].type)
-	{
-	  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
-	       cap->sym.cipher.algo == supported_algo[i].cipher) ||
-	      (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH &&
-	       cap->sym.auth.algo == supported_algo[i].auth))
-	    {
-	      if (name)
-		strcpy (name, supported_algo[i].name);
-	      return 0;
-	    }
-	}
-
-      i++;
-    }
-
-  return -1;
-}
-
-#endif /* __DPDK_IPSEC_H__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/main.c b/src/vnet/devices/dpdk/main.c
deleted file mode 100644
index 9ea3aa04..00000000
--- a/src/vnet/devices/dpdk/main.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-
-
-/*
- * Called by the dpdk driver's rte_delay_us() function.
- * Return 0 to have the dpdk do a regular delay loop.
- * Return 1 if to skip the delay loop because we are suspending
- * the calling vlib process instead.
- */
-int
-rte_delay_us_override (unsigned us)
-{
-  vlib_main_t *vm;
-
-  /* Don't bother intercepting for short delays */
-  if (us < 10)
-    return 0;
-
-  /*
-   * Only intercept if we are in a vlib process.
-   * If we are called from a vlib worker thread or the vlib main
-   * thread then do not intercept. (Must not be called from an
-   * independent pthread).
-   */
-  if (os_get_cpu_number () == 0)
-    {
-      /*
-       * We're in the vlib main thread or a vlib process. Make sure
-       * the process is running and we're not still initializing.
-       */
-      vm = vlib_get_main ();
-      if (vlib_in_process_context (vm))
-	{
-	  /* Only suspend for the admin_down_process */
-	  vlib_process_t *proc = vlib_get_current_process (vm);
-	  if (!(proc->flags & VLIB_PROCESS_IS_RUNNING) ||
-	      (proc->node_runtime.function != admin_up_down_process))
-	    return 0;
-
-	  f64 delay = 1e-6 * us;
-	  vlib_process_suspend (vm, delay);
-	  return 1;
-	}
-    }
-  return 0;			// no override
-}
-
-static void
-rte_delay_us_override_cb (unsigned us)
-{
-  if (rte_delay_us_override (us) == 0)
-    rte_delay_us_block (us);
-}
-
-static clib_error_t * dpdk_main_init (vlib_main_t * vm)
-{
-  clib_error_t * error = 0;
-
-  if ((error = vlib_call_init_function (vm, dpdk_init)))
-    return error;
-
-  /* register custom delay function */
-  rte_delay_us_callback_register (rte_delay_us_override_cb);
-
-  return error;
-}
-
-VLIB_INIT_FUNCTION (dpdk_main_init);
-
diff --git a/src/vnet/devices/dpdk/node.c b/src/vnet/devices/dpdk/node.c
deleted file mode 100644
index 0d64ae08..00000000
--- a/src/vnet/devices/dpdk/node.c
+++ /dev/null
@@ -1,674 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include "dpdk_priv.h"
-
-static char *dpdk_error_strings[] = {
-#define _(n,s) s,
-  foreach_dpdk_error
-#undef _
-};
-
-always_inline int
-vlib_buffer_is_ip4 (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) b->data;
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4));
-}
-
-always_inline int
-vlib_buffer_is_ip6 (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) b->data;
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6));
-}
-
-always_inline int
-vlib_buffer_is_mpls (vlib_buffer_t * b)
-{
-  ethernet_header_t *h = (ethernet_header_t *) b->data;
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST));
-}
-
-always_inline u32
-dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0)
-{
-  if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0)))
-    if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0))
-      return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
-    else
-      return VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0)))
-    return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
-    return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
-  else
-    return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
-}
-
-always_inline int
-dpdk_mbuf_is_vlan (struct rte_mbuf *mb)
-{
-  return (mb->packet_type & RTE_PTYPE_L2_ETHER_VLAN) ==
-    RTE_PTYPE_L2_ETHER_VLAN;
-}
-
-always_inline int
-dpdk_mbuf_is_ip4 (struct rte_mbuf *mb)
-{
-  return RTE_ETH_IS_IPV4_HDR (mb->packet_type) != 0;
-}
-
-always_inline int
-dpdk_mbuf_is_ip6 (struct rte_mbuf *mb)
-{
-  return RTE_ETH_IS_IPV6_HDR (mb->packet_type) != 0;
-}
-
-always_inline u32
-dpdk_rx_next_from_mb (struct rte_mbuf * mb, vlib_buffer_t * b0)
-{
-  if (PREDICT_FALSE (dpdk_mbuf_is_vlan (mb)))
-    return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
-  else if (PREDICT_TRUE (dpdk_mbuf_is_ip4 (mb)))
-    return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT;
-  else if (PREDICT_TRUE (dpdk_mbuf_is_ip6 (mb)))
-    return VNET_DEVICE_INPUT_NEXT_IP6_INPUT;
-  else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0)))
-    return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT;
-  else
-    return dpdk_rx_next_from_etype (mb, b0);
-}
-
-always_inline void
-dpdk_rx_error_from_mb (struct rte_mbuf *mb, u32 * next, u8 * error)
-{
-  if (mb->ol_flags & PKT_RX_IP_CKSUM_BAD)
-    {
-      *error = DPDK_ERROR_IP_CHECKSUM_ERROR;
-      *next = VNET_DEVICE_INPUT_NEXT_DROP;
-    }
-  else
-    *error = DPDK_ERROR_NONE;
-}
-
-void
-dpdk_rx_trace (dpdk_main_t * dm,
-	       vlib_node_runtime_t * node,
-	       dpdk_device_t * xd,
-	       u16 queue_id, u32 * buffers, uword n_buffers)
-{
-  vlib_main_t *vm = vlib_get_main ();
-  u32 *b, n_left;
-  u32 next0;
-
-  n_left = n_buffers;
-  b = buffers;
-
-  while (n_left >= 1)
-    {
-      u32 bi0;
-      vlib_buffer_t *b0;
-      dpdk_rx_dma_trace_t *t0;
-      struct rte_mbuf *mb;
-      u8 error0;
-
-      bi0 = b[0];
-      n_left -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      mb = rte_mbuf_from_vlib_buffer (b0);
-
-      if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
-	next0 = xd->per_interface_next_index;
-      else if (PREDICT_TRUE
-	       ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0))
-	next0 = dpdk_rx_next_from_mb (mb, b0);
-      else
-	next0 = dpdk_rx_next_from_etype (mb, b0);
-
-      dpdk_rx_error_from_mb (mb, &next0, &error0);
-
-      vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
-      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
-      t0->queue_index = queue_id;
-      t0->device_index = xd->device_index;
-      t0->buffer_index = bi0;
-
-      clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
-      clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
-      clib_memcpy (t0->buffer.pre_data, b0->data,
-		   sizeof (t0->buffer.pre_data));
-      clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data));
-
-      b += 1;
-    }
-}
-
-static inline u32
-dpdk_rx_burst (dpdk_main_t * dm, dpdk_device_t * xd, u16 queue_id)
-{
-  u32 n_buffers;
-  u32 n_left;
-  u32 n_this_chunk;
-
-  n_left = VLIB_FRAME_SIZE;
-  n_buffers = 0;
-
-  if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
-    {
-      while (n_left)
-	{
-	  n_this_chunk = rte_eth_rx_burst (xd->device_index, queue_id,
-					   xd->rx_vectors[queue_id] +
-					   n_buffers, n_left);
-	  n_buffers += n_this_chunk;
-	  n_left -= n_this_chunk;
-
-	  /* Empirically, DPDK r1.8 produces vectors w/ 32 or fewer elts */
-	  if (n_this_chunk < 32)
-	    break;
-	}
-    }
-  else
-    {
-      ASSERT (0);
-    }
-
-  return n_buffers;
-}
-
-
-static_always_inline void
-dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b,
-			  struct rte_mbuf *mb, vlib_buffer_free_list_t * fl)
-{
-  u8 nb_seg = 1;
-  struct rte_mbuf *mb_seg = 0;
-  vlib_buffer_t *b_seg, *b_chain = 0;
-  mb_seg = mb->next;
-  b_chain = b;
-
-  while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
-    {
-      ASSERT (mb_seg != 0);
-
-      b_seg = vlib_buffer_from_rte_mbuf (mb_seg);
-      vlib_buffer_init_for_free_list (b_seg, fl);
-
-      ASSERT ((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
-      ASSERT (b_seg->current_data == 0);
-
-      /*
-       * The driver (e.g. virtio) may not put the packet data at the start
-       * of the segment, so don't assume b_seg->current_data == 0 is correct.
-       */
-      b_seg->current_data =
-	(mb_seg->buf_addr + mb_seg->data_off) - (void *) b_seg->data;
-
-      b_seg->current_length = mb_seg->data_len;
-      b->total_length_not_including_first_buffer += mb_seg->data_len;
-
-      b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
-      b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
-
-      b_chain = b_seg;
-      mb_seg = mb_seg->next;
-      nb_seg++;
-    }
-}
-
-static_always_inline void
-dpdk_prefetch_buffer (struct rte_mbuf *mb)
-{
-  vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
-  CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, LOAD);
-  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, STORE);
-}
-
-/*
- * This function is used when there are no worker threads.
- * The main thread performs IO and forwards the packets.
- */
-static_always_inline u32
-dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd,
-		   vlib_node_runtime_t * node, u32 cpu_index, u16 queue_id)
-{
-  u32 n_buffers;
-  u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
-  u32 n_left_to_next, *to_next;
-  u32 mb_index;
-  vlib_main_t *vm = vlib_get_main ();
-  uword n_rx_bytes = 0;
-  u32 n_trace, trace_cnt __attribute__ ((unused));
-  vlib_buffer_free_list_t *fl;
-  u32 buffer_flags_template;
-
-  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
-    return 0;
-
-  n_buffers = dpdk_rx_burst (dm, xd, queue_id);
-
-  if (n_buffers == 0)
-    {
-      return 0;
-    }
-
-  buffer_flags_template = dm->buffer_flags_template;
-
-  vec_reset_length (xd->d_trace_buffers[cpu_index]);
-  trace_cnt = n_trace = vlib_get_trace_count (vm, node);
-
-  if (n_trace > 0)
-    {
-      u32 n = clib_min (n_trace, n_buffers);
-      mb_index = 0;
-
-      while (n--)
-	{
-	  struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index++];
-	  vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb);
-	  vec_add1 (xd->d_trace_buffers[cpu_index],
-		    vlib_get_buffer_index (vm, b));
-	}
-    }
-
-  fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-
-  mb_index = 0;
-
-  while (n_buffers > 0)
-    {
-      vlib_buffer_t *b0, *b1, *b2, *b3;
-      u32 bi0, next0, l3_offset0;
-      u32 bi1, next1, l3_offset1;
-      u32 bi2, next2, l3_offset2;
-      u32 bi3, next3, l3_offset3;
-      u8 error0, error1, error2, error3;
-      u64 or_ol_flags;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_buffers > 8 && n_left_to_next > 4)
-	{
-	  struct rte_mbuf *mb0 = xd->rx_vectors[queue_id][mb_index];
-	  struct rte_mbuf *mb1 = xd->rx_vectors[queue_id][mb_index + 1];
-	  struct rte_mbuf *mb2 = xd->rx_vectors[queue_id][mb_index + 2];
-	  struct rte_mbuf *mb3 = xd->rx_vectors[queue_id][mb_index + 3];
-
-	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 4]);
-	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 5]);
-	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 6]);
-	  dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 7]);
-
-	  if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG)
-	    {
-	      if (PREDICT_FALSE (mb0->nb_segs > 1))
-		dpdk_prefetch_buffer (mb0->next);
-	      if (PREDICT_FALSE (mb1->nb_segs > 1))
-		dpdk_prefetch_buffer (mb1->next);
-	      if (PREDICT_FALSE (mb2->nb_segs > 1))
-		dpdk_prefetch_buffer (mb2->next);
-	      if (PREDICT_FALSE (mb3->nb_segs > 1))
-		dpdk_prefetch_buffer (mb3->next);
-	    }
-
-	  ASSERT (mb0);
-	  ASSERT (mb1);
-	  ASSERT (mb2);
-	  ASSERT (mb3);
-
-	  or_ol_flags = (mb0->ol_flags | mb1->ol_flags |
-			 mb2->ol_flags | mb3->ol_flags);
-	  b0 = vlib_buffer_from_rte_mbuf (mb0);
-	  b1 = vlib_buffer_from_rte_mbuf (mb1);
-	  b2 = vlib_buffer_from_rte_mbuf (mb2);
-	  b3 = vlib_buffer_from_rte_mbuf (mb3);
-
-	  vlib_buffer_init_for_free_list (b0, fl);
-	  vlib_buffer_init_for_free_list (b1, fl);
-	  vlib_buffer_init_for_free_list (b2, fl);
-	  vlib_buffer_init_for_free_list (b3, fl);
-
-	  bi0 = vlib_get_buffer_index (vm, b0);
-	  bi1 = vlib_get_buffer_index (vm, b1);
-	  bi2 = vlib_get_buffer_index (vm, b2);
-	  bi3 = vlib_get_buffer_index (vm, b3);
-
-	  to_next[0] = bi0;
-	  to_next[1] = bi1;
-	  to_next[2] = bi2;
-	  to_next[3] = bi3;
-	  to_next += 4;
-	  n_left_to_next -= 4;
-
-	  if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
-	    {
-	      next0 = next1 = next2 = next3 = xd->per_interface_next_index;
-	    }
-	  else if (PREDICT_TRUE
-		   ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0))
-	    {
-	      next0 = dpdk_rx_next_from_mb (mb0, b0);
-	      next1 = dpdk_rx_next_from_mb (mb1, b1);
-	      next2 = dpdk_rx_next_from_mb (mb2, b2);
-	      next3 = dpdk_rx_next_from_mb (mb3, b3);
-	    }
-	  else
-	    {
-	      next0 = dpdk_rx_next_from_etype (mb0, b0);
-	      next1 = dpdk_rx_next_from_etype (mb1, b1);
-	      next2 = dpdk_rx_next_from_etype (mb2, b2);
-	      next3 = dpdk_rx_next_from_etype (mb3, b3);
-	    }
-
-	  if (PREDICT_FALSE (or_ol_flags & PKT_RX_IP_CKSUM_BAD))
-	    {
-	      dpdk_rx_error_from_mb (mb0, &next0, &error0);
-	      dpdk_rx_error_from_mb (mb1, &next1, &error1);
-	      dpdk_rx_error_from_mb (mb2, &next2, &error2);
-	      dpdk_rx_error_from_mb (mb3, &next3, &error3);
-	      b0->error = node->errors[error0];
-	      b1->error = node->errors[error1];
-	      b2->error = node->errors[error2];
-	      b3->error = node->errors[error3];
-	    }
-	  else
-	    {
-	      b0->error = b1->error = node->errors[DPDK_ERROR_NONE];
-	      b2->error = b3->error = node->errors[DPDK_ERROR_NONE];
-	    }
-
-	  l3_offset0 = device_input_next_node_advance[next0];
-	  l3_offset1 = device_input_next_node_advance[next1];
-	  l3_offset2 = device_input_next_node_advance[next2];
-	  l3_offset3 = device_input_next_node_advance[next3];
-
-	  b0->current_data = l3_offset0 + mb0->data_off;
-	  b1->current_data = l3_offset1 + mb1->data_off;
-	  b2->current_data = l3_offset2 + mb2->data_off;
-	  b3->current_data = l3_offset3 + mb3->data_off;
-
-	  b0->current_data -= RTE_PKTMBUF_HEADROOM;
-	  b1->current_data -= RTE_PKTMBUF_HEADROOM;
-	  b2->current_data -= RTE_PKTMBUF_HEADROOM;
-	  b3->current_data -= RTE_PKTMBUF_HEADROOM;
-
-	  b0->current_length = mb0->data_len - l3_offset0;
-	  b1->current_length = mb1->data_len - l3_offset1;
-	  b2->current_length = mb2->data_len - l3_offset2;
-	  b3->current_length = mb3->data_len - l3_offset3;
-
-	  b0->flags = buffer_flags_template;
-	  b1->flags = buffer_flags_template;
-	  b2->flags = buffer_flags_template;
-	  b3->flags = buffer_flags_template;
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b1)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b2)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b3)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	  vnet_buffer (b2)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	  vnet_buffer (b3)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-
-	  n_rx_bytes += mb0->pkt_len;
-	  n_rx_bytes += mb1->pkt_len;
-	  n_rx_bytes += mb2->pkt_len;
-	  n_rx_bytes += mb3->pkt_len;
-
-	  /* Process subsequent segments of multi-segment packets */
-	  if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG)
-	    {
-	      dpdk_process_subseq_segs (vm, b0, mb0, fl);
-	      dpdk_process_subseq_segs (vm, b1, mb1, fl);
-	      dpdk_process_subseq_segs (vm, b2, mb2, fl);
-	      dpdk_process_subseq_segs (vm, b3, mb3, fl);
-	    }
-
-	  /*
-	   * Turn this on if you run into
-	   * "bad monkey" contexts, and you want to know exactly
-	   * which nodes they've visited... See main.c...
-	   */
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
-
-	  /* Do we have any driver RX features configured on the interface? */
-	  vnet_feature_start_device_input_x4 (xd->vlib_sw_if_index,
-					      &next0, &next1, &next2, &next3,
-					      b0, b1, b2, b3,
-					      l3_offset0, l3_offset1,
-					      l3_offset2, l3_offset3);
-
-	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, bi1, bi2, bi3,
-					   next0, next1, next2, next3);
-	  n_buffers -= 4;
-	  mb_index += 4;
-	}
-      while (n_buffers > 0 && n_left_to_next > 0)
-	{
-	  struct rte_mbuf *mb0 = xd->rx_vectors[queue_id][mb_index];
-
-	  ASSERT (mb0);
-
-	  b0 = vlib_buffer_from_rte_mbuf (mb0);
-
-	  /* Prefetch one next segment if it exists. */
-	  if (PREDICT_FALSE (mb0->nb_segs > 1))
-	    dpdk_prefetch_buffer (mb0->next);
-
-	  vlib_buffer_init_for_free_list (b0, fl);
-
-	  bi0 = vlib_get_buffer_index (vm, b0);
-
-	  to_next[0] = bi0;
-	  to_next++;
-	  n_left_to_next--;
-
-	  if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
-	    next0 = xd->per_interface_next_index;
-	  else if (PREDICT_TRUE
-		   ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0))
-	    next0 = dpdk_rx_next_from_mb (mb0, b0);
-	  else
-	    next0 = dpdk_rx_next_from_etype (mb0, b0);
-
-	  dpdk_rx_error_from_mb (mb0, &next0, &error0);
-	  b0->error = node->errors[error0];
-
-	  l3_offset0 = device_input_next_node_advance[next0];
-
-	  b0->current_data = l3_offset0;
-	  b0->current_data += mb0->data_off - RTE_PKTMBUF_HEADROOM;
-	  b0->current_length = mb0->data_len - l3_offset0;
-
-	  b0->flags = buffer_flags_template;
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	  n_rx_bytes += mb0->pkt_len;
-
-	  /* Process subsequent segments of multi-segment packets */
-	  dpdk_process_subseq_segs (vm, b0, mb0, fl);
-
-	  /*
-	   * Turn this on if you run into
-	   * "bad monkey" contexts, and you want to know exactly
-	   * which nodes they've visited... See main.c...
-	   */
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-
-	  /* Do we have any driver RX features configured on the interface? */
-	  vnet_feature_start_device_input_x1 (xd->vlib_sw_if_index, &next0,
-					      b0, l3_offset0);
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	  n_buffers--;
-	  mb_index++;
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[cpu_index]) > 0))
-    {
-      dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers[cpu_index],
-		     vec_len (xd->d_trace_buffers[cpu_index]));
-      vlib_set_trace_count (vm, node, n_trace -
-			    vec_len (xd->d_trace_buffers[cpu_index]));
-    }
-
-  vlib_increment_combined_counter
-    (vnet_get_main ()->interface_main.combined_sw_if_counters
-     + VNET_INTERFACE_COUNTER_RX,
-     cpu_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes);
-
-  vnet_device_increment_rx_packets (cpu_index, mb_index);
-
-  return mb_index;
-}
-
-static inline void
-poll_rate_limit (dpdk_main_t * dm)
-{
-  /* Limit the poll rate by sleeping for N msec between polls */
-  if (PREDICT_FALSE (dm->poll_sleep != 0))
-    {
-      struct timespec ts, tsrem;
-
-      ts.tv_sec = 0;
-      ts.tv_nsec = 1000 * 1000 * dm->poll_sleep;	/* 1ms */
-
-      while (nanosleep (&ts, &tsrem) < 0)
-	{
-	  ts = tsrem;
-	}
-    }
-}
-
-/** \brief Main DPDK input node
-    @node dpdk-input
-
-    This is the main DPDK input node: across each assigned interface,
-    call rte_eth_rx_burst(...) or similar to obtain a vector of
-    packets to process. Handle early packet discard. Derive @c
-    vlib_buffer_t metadata from struct rte_mbuf metadata,
-    Depending on the resulting metadata: adjust b->current_data,
-    b->current_length  and dispatch directly to
-    ip4-input-no-checksum, or ip6-input. Trace the packet if required.
-
-    @param vm   vlib_main_t corresponding to the current thread
-    @param node vlib_node_runtime_t
-    @param f    vlib_frame_t input-node, not used.
-
-    @par Graph mechanics: buffer metadata, next index usage
-
-    @em Uses:
-    - struct rte_mbuf mb->ol_flags
-        - PKT_RX_IP_CKSUM_BAD
-    -  RTE_ETH_IS_xxx_HDR(mb->packet_type) 
-        - packet classification result
-
-    @em Sets:
-    - b->error if the packet is to be dropped immediately
-    - b->current_data, b->current_length
-        - adjusted as needed to skip the L2 header in  direct-dispatch cases
-    - vnet_buffer(b)->sw_if_index[VLIB_RX]
-        - rx interface sw_if_index
-    - vnet_buffer(b)->sw_if_index[VLIB_TX] = ~0
-        - required by ipX-lookup
-    - b->flags
-        - to indicate multi-segment pkts (VLIB_BUFFER_NEXT_PRESENT), etc.
-
-    Next Nodes:
-    - Static arcs to: error-drop, ethernet-input,
-      ip4-input-no-checksum, ip6-input, mpls-input
-    - per-interface redirection, controlled by
-      xd->per_interface_next_index
-*/
-
-static uword
-dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f)
-{
-  dpdk_main_t *dm = &dpdk_main;
-  dpdk_device_t *xd;
-  uword n_rx_packets = 0;
-  dpdk_device_and_queue_t *dq;
-  u32 cpu_index = os_get_cpu_number ();
-
-  /*
-   * Poll all devices on this cpu for input/interrupts.
-   */
-  /* *INDENT-OFF* */
-  vec_foreach (dq, dm->devices_by_cpu[cpu_index])
-    {
-      xd = vec_elt_at_index(dm->devices, dq->device);
-      n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id);
-    }
-  /* *INDENT-ON* */
-
-  poll_rate_limit (dm);
-
-  return n_rx_packets;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (dpdk_input_node) = {
-  .function = dpdk_input,
-  .type = VLIB_NODE_TYPE_INPUT,
-  .name = "dpdk-input",
-  .sibling_of = "device-input",
-
-  /* Will be enabled if/when hardware is detected. */
-  .state = VLIB_NODE_STATE_DISABLED,
-
-  .format_buffer = format_ethernet_header_with_length,
-  .format_trace = format_dpdk_rx_dma_trace,
-
-  .n_errors = DPDK_N_ERROR,
-  .error_strings = dpdk_error_strings,
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (dpdk_input_node, dpdk_input);
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/dpdk/qos_doc.md b/src/vnet/devices/dpdk/qos_doc.md
deleted file mode 100644
index 7c064246..00000000
--- a/src/vnet/devices/dpdk/qos_doc.md
+++ /dev/null
@@ -1,411 +0,0 @@
-# QoS Hierarchical Scheduler    {#qos_doc}
-
-The Quality-of-Service (QoS) scheduler performs egress-traffic management by
-prioritizing the transmission of the packets of different type services and
-subcribers based on the Service Level Agreements (SLAs). The QoS scheduler can
-be enabled on one or more NIC output interfaces depending upon the
-requirement.
-
-
-## Overview
-
-The QoS schdeuler supports a number of scheduling and shaping levels which
-construct hierarchical-tree. The first level in the hierarchy is port (i.e.
-the physical interface) that constitutes the root node of the tree. The
-subsequent level is subport which represents the group of the
-users/subscribers. The individual user/subscriber is represented by the pipe
-at the next level. Each user can have different traffic type based on the
-criteria of specific loss rate, jitter, and latency. These traffic types are
-represented at the traffic-class level in the form of different traffic-
-classes. The last level contains number of queues which are grouped together
-to host the packets of the specific class type traffic.
-
-The QoS scheduler implementation requires flow classification, enqueue  and
-dequeue operations. The flow classification is mandatory stage for HQoS where
-incoming packets are classified by mapping the packet fields information to
-5-tuple (HQoS subport, pipe, traffic class, queue within traffic class, and
-color) and storing that information in mbuf sched field. The enqueue operation
-uses this information to determine the queue for storing the packet, and at
-this stage, if the specific queue is full, QoS drops the packet. The dequeue
-operation consists of scheduling the packet based on its length and available
-credits, and handing over the scheduled packet to the output interface.
-
-For more information on QoS Scheduler, please refer DPDK Programmer's Guide-
-http://dpdk.org/doc/guides/prog_guide/qos_framework.html
-
-
-### QoS Schdeuler Parameters
-
-Following illustrates the default HQoS configuration for each 10GbE output
-port:
-
-Single subport (subport 0):
-  - Subport rate set to 100% of port rate
-  - Each of the 4 traffic classes has rate set to 100% of port rate
-
-4K pipes per subport 0 (pipes 0 .. 4095) with identical configuration:
-  - Pipe rate set to 1/4K of port rate
-  - Each of the 4 traffic classes has rate set to 100% of pipe rate
-  - Within each traffic class, the byte-level WRR weights for the 4 queues are set to 1:1:1:1
-
-
-#### Port configuration
-
-```
-port {
-  rate 1250000000           /* Assuming 10GbE port */
-  frame_overhead 24         /* Overhead fields per Ethernet frame:
-                             * 7B (Preamble) +
-                             * 1B (Start of Frame Delimiter (SFD)) + 
-                             * 4B (Frame Check Sequence (FCS)) +
-                             * 12B (Inter Frame Gap (IFG))
-                             */
-  mtu 1522                  /* Assuming Ethernet/IPv4 pkt (FCS not included) */
-  n_subports_per_port 1     /* Number of subports per output interface */
-  n_pipes_per_subport 4096  /* Number of pipes (users/subscribers) */
-  queue_sizes 64 64 64 64   /* Packet queue size for each traffic class.
-                             * All queues within the same pipe traffic class
-                             * have the same size. Queues from different
-                             * pipes serving the same traffic class have
-                             * the same size. */
-}
-```
-
-
-#### Subport configuration
-
-```
-subport 0 {
-  tb_rate 1250000000        /* Subport level token bucket rate (bytes per second) */
-  tb_size 1000000           /* Subport level token bucket size (bytes) */
-  tc0_rate 1250000000       /* Subport level token bucket rate for traffic class 0 (bytes per second) */
-  tc1_rate 1250000000       /* Subport level token bucket rate for traffic class 1 (bytes per second) */
-  tc2_rate 1250000000       /* Subport level token bucket rate for traffic class 2 (bytes per second) */
-  tc3_rate 1250000000       /* Subport level token bucket rate for traffic class 3 (bytes per second) */
-  tc_period 10              /* Time interval for refilling the token bucket associated with traffic class (Milliseconds) */
-  pipe 0 4095 profile 0     /* pipes (users/subscribers) configured with pipe profile 0 */
-}
-```
-
-
-#### Pipe configuration
-
-```
-pipe_profile 0 {
-  tb_rate 305175          /* Pipe level token bucket rate (bytes per second) */
-  tb_size 1000000         /* Pipe level token bucket size (bytes) */
-  tc0_rate 305175         /* Pipe level token bucket rate for traffic class 0 (bytes per second) */
-  tc1_rate 305175         /* Pipe level token bucket rate for traffic class 1 (bytes per second) */
-  tc2_rate 305175         /* Pipe level token bucket rate for traffic class 2 (bytes per second) */
-  tc3_rate 305175         /* Pipe level token bucket rate for traffic class 3 (bytes per second) */
-  tc_period 40            /* Time interval for refilling the token bucket associated with traffic class at pipe level (Milliseconds) */
-  tc3_oversubscription_weight 1 /* Weight traffic class 3 oversubscription */
-  tc0_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 0 */
-  tc1_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 1 */
-  tc2_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 2 */
-  tc3_wrr_weights 1 1 1 1     /* Pipe queues WRR weights for traffic class 3 */
-}
-```
-
-
-#### Random Early Detection (RED) parameters per traffic class and color (Green / Yellow / Red)
-
-```
-red {
-  tc0_wred_min 48 40 32     /* Minimum threshold for traffic class 0 queue (min_th) in number of packets */
-  tc0_wred_max 64 64 64     /* Maximum threshold for traffic class 0 queue (max_th) in number of packets */
-  tc0_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 0 queue (maxp = 1 / maxp_inv) */
-  tc0_wred_weight 9 9 9     /* Traffic Class 0 queue weight */
-  tc1_wred_min 48 40 32     /* Minimum threshold for traffic class 1 queue (min_th) in number of packets */
-  tc1_wred_max 64 64 64     /* Maximum threshold for traffic class 1 queue (max_th) in number of packets */
-  tc1_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 1 queue (maxp = 1 / maxp_inv) */
-  tc1_wred_weight 9 9 9     /* Traffic Class 1 queue weight */
-  tc2_wred_min 48 40 32     /* Minimum threshold for traffic class 2 queue (min_th) in number of packets */
-  tc2_wred_max 64 64 64     /* Maximum threshold for traffic class 2 queue (max_th) in number of packets */
-  tc2_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 2 queue (maxp = 1 / maxp_inv) */
-  tc2_wred_weight 9 9 9     /* Traffic Class 2 queue weight */
-  tc3_wred_min 48 40 32     /* Minimum threshold for traffic class 3 queue (min_th) in number of packets */
-  tc3_wred_max 64 64 64     /* Maximum threshold for traffic class 3 queue (max_th) in number of packets */
-  tc3_wred_inv_prob 10 10 10    /* Inverse of packet marking probability for traffic class 3 queue (maxp = 1 / maxp_inv) */
-  tc3_wred_weight 9 9 9     /* Traffic Class 3 queue weight */
-}
-```
-
-
-### DPDK QoS Scheduler Integration in VPP
-
-The Hierarchical Quaity-of-Service (HQoS) scheduler object could be seen as
-part of the logical NIC output interface. To enable HQoS on specific output
-interface, vpp startup.conf file has to be configured accordingly. The output
-interface that requires HQoS, should have "hqos" parameter specified in dpdk
-section. Another optional parameter "hqos-thread"  has been defined which can
-be used to associate the output interface with specific hqos thread. In cpu
-section of the config file, "corelist-hqos-threads" is introduced to assign
-logical cpu cores to run the HQoS threads. A HQoS thread can run multiple HQoS
-objects each associated with different output interfaces. All worker threads
-instead of writing packets to NIC TX queue directly, write the packets to a
-software queues. The hqos_threads read the software queues, and enqueue the
-packets to HQoS objects, as well as dequeue packets from HQOS objects and
-write them to NIC output interfaces. The worker threads need to be able to
-send the packets to any output interface, therefore, each HQoS object
-associated with NIC output interface should have software queues equal to
-worker threads count.
-
-Following illustrates the sample startup configuration file with 4x worker
-threads feeding 2x hqos threads that handle each QoS scheduler for 1x output
-interface.
-
-```
-dpdk {
-  socket-mem 16384,16384
-
-  dev 0000:02:00.0 {
-    num-rx-queues 2
-    hqos
-  }
-  dev 0000:06:00.0 {
-    num-rx-queues 2
-    hqos
-  }
-
-  num-mbufs 1000000
-}
-
-cpu {
-  main-core 0
-  corelist-workers  1, 2, 3, 4
-  corelist-hqos-threads  5, 6
-}
-```
-
-
-### QoS scheduler CLI Commands
-
-Each QoS scheduler instance is initialised with default parameters required to
-configure hqos port, subport, pipe and queues. Some of the parameters can be
-re-configured in run-time through CLI commands.
-
-
-#### Configuration
-
-Following commands can be used to configure QoS scheduler parameters.
-
-The command below can be used to set the subport level parameters such as
-token bucket rate (bytes per seconds), token bucket size (bytes), traffic
-class rates (bytes per seconds) and token update period (Milliseconds).
-
-```
-set dpdk interface hqos subport  subport  [rate ]
-    [bktsize ] [tc0 ] [tc1 ] [tc2 ] [tc3 ] [period ]
-```
-
-For setting the pipe profile, following command can be used.    
-
-```
-set dpdk interface hqos pipe  subport  pipe 
-    profile 
-```
-
-To assign QoS scheduler instance to the specific thread, following command can
-be used.
-
-```
-set dpdk interface hqos placement  thread 
-```
-
-The command below is used to set the packet fields required for classifiying
-the incoming packet. As a result of classification process,     packet field
-information will be mapped to 5 tuples (subport, pipe, traffic class, pipe,
-color) and stored in packet mbuf.
-
-```
-set dpdk interface hqos pktfield  id subport|pipe|tc offset 
-    mask 
-```
-
-The DSCP table entries used for idenfiying the traffic class and queue can be set using the command below;   
-
-```  
-set dpdk interface hqos tctbl  entry  tc  queue 
-```
-
-
-#### Show Command
-
-The QoS Scheduler configuration can displayed using the command below.
-
-```
-   vpp# show dpdk interface hqos TenGigabitEthernet2/0/0
-   Thread:
-     Input SWQ size = 4096 packets
-     Enqueue burst size = 256 packets
-     Dequeue burst size = 220 packets
-     Packet field 0: slab position =    0, slab bitmask = 0x0000000000000000   (subport)
-     Packet field 1: slab position =   40, slab bitmask = 0x0000000fff000000   (pipe)
-     Packet field 2: slab position =    8, slab bitmask = 0x00000000000000fc   (tc)
-     Packet field 2  tc translation table: ([Mapped Value Range]: tc/queue tc/queue ...)
-     [ 0 .. 15]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
-     [16 .. 31]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
-     [32 .. 47]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
-     [48 .. 63]: 0/0 0/1 0/2 0/3 1/0 1/1 1/2 1/3 2/0 2/1 2/2 2/3 3/0 3/1 3/2 3/3
-   Port:
-     Rate = 1250000000 bytes/second
-     MTU = 1514 bytes
-     Frame overhead = 24 bytes
-     Number of subports = 1
-     Number of pipes per subport = 4096
-     Packet queue size: TC0 = 64, TC1 = 64, TC2 = 64, TC3 = 64 packets
-     Number of pipe profiles = 1
-   Subport 0:
-     Rate = 120000000 bytes/second
-     Token bucket size = 1000000 bytes
-     Traffic class rate: TC0 = 120000000, TC1 = 120000000, TC2 = 120000000, TC3 = 120000000 bytes/second
-     TC period = 10 milliseconds
-   Pipe profile 0:
-     Rate = 305175 bytes/second
-     Token bucket size = 1000000 bytes
-     Traffic class rate: TC0 = 305175, TC1 = 305175, TC2 = 305175, TC3 = 305175 bytes/second
-     TC period = 40 milliseconds
-     TC0 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
-     TC1 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
-     TC2 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
-     TC3 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1
-```
-
-The QoS Scheduler placement over the logical cpu cores can be displayed using
-below command.
-
-```
-    vpp# show dpdk interface hqos placement
-    Thread 5 (vpp_hqos-threads_0 at lcore 5):
-      TenGigabitEthernet2/0/0 queue 0
-    Thread 6 (vpp_hqos-threads_1 at lcore 6):
-      TenGigabitEthernet4/0/1 queue 0
-```
-
-
-### QoS Scheduler Binary APIs
-
-This section explans the available binary APIs for configuring QoS scheduler
-parameters in run-time.
-
-The following API can be used to set the pipe profile of a pipe that belongs
-to a given subport:
-
-```
-sw_interface_set_dpdk_hqos_pipe rx  | sw_if_index 
-    subport  pipe  profile 
-```
-
-The data structures used for set the pipe profile parameter are as follows;
-
-```
-  /** \\brief DPDK interface HQoS pipe profile set request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - the interface
-    @param subport - subport ID
-    @param pipe - pipe ID within its subport
-    @param profile - pipe profile ID
-  */
-  define sw_interface_set_dpdk_hqos_pipe {
-    u32 client_index;
-    u32 context;
-    u32 sw_if_index;
-    u32 subport;
-    u32 pipe;
-    u32 profile;
-  };
-
-  /** \\brief DPDK interface HQoS pipe profile set reply
-    @param context - sender context, to match reply w/ request
-    @param retval - request return code
-  */
-  define sw_interface_set_dpdk_hqos_pipe_reply {
-    u32 context;
-    i32 retval;
-  };
-```
-
-The following API can be used to set the subport level parameters, for
-example- token bucket rate (bytes per seconds), token bucket size (bytes),
-traffic class rate (bytes per seconds) and tokens update period.
-
-```
-sw_interface_set_dpdk_hqos_subport rx  | sw_if_index 
-    subport  [rate ] [bktsize ]
-    [tc0 ] [tc1 ] [tc2 ] [tc3 ] [period ]
-```
-
-The data structures used for set the subport level parameter are as follows;
-
-```
-  /** \\brief DPDK interface HQoS subport parameters set request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - the interface
-    @param subport - subport ID
-    @param tb_rate - subport token bucket rate (measured in bytes/second)
-    @param tb_size - subport token bucket size (measured in credits)
-    @param tc_rate - subport traffic class 0 .. 3 rates (measured in bytes/second)
-    @param tc_period - enforcement period for rates (measured in milliseconds)
-  */
-  define sw_interface_set_dpdk_hqos_subport {
-    u32 client_index;
-    u32 context;
-    u32 sw_if_index;
-    u32 subport;
-    u32 tb_rate;
-    u32 tb_size;
-    u32 tc_rate[4];
-    u32 tc_period;
-  };
-
-  /** \\brief DPDK interface HQoS subport parameters set reply
-    @param context - sender context, to match reply w/ request
-    @param retval - request return code
-  */
-  define sw_interface_set_dpdk_hqos_subport_reply {
-    u32 context;
-    i32 retval;
-  };
-```
-
-The following API can be used set the DSCP table entry. The DSCP table have
-64 entries to map the packet DSCP field onto traffic class and hqos input
-queue.
-
-```
-sw_interface_set_dpdk_hqos_tctbl rx  | sw_if_index  
-    entry  tc  queue 
-```
-
-The data structures used for setting DSCP table entries are given below.
-
-```
-  /** \\brief DPDK interface HQoS tctbl entry set request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - the interface
-    @param entry - entry index ID
-    @param tc - traffic class (0 .. 3)
-    @param queue - traffic class queue (0 .. 3)
-  */
-  define sw_interface_set_dpdk_hqos_tctbl {
-    u32 client_index;
-    u32 context;
-    u32 sw_if_index;
-    u32 entry;
-    u32 tc;
-    u32 queue;
-  };
-
-  /** \\brief DPDK interface HQoS tctbl entry set reply
-    @param context - sender context, to match reply w/ request
-    @param retval - request return code
-  */
-  define sw_interface_set_dpdk_hqos_tctbl_reply {
-    u32 context;
-    i32 retval;
-  };
-```
diff --git a/src/vnet/devices/dpdk/thread.c b/src/vnet/devices/dpdk/thread.c
deleted file mode 100644
index 475dd142..00000000
--- a/src/vnet/devices/dpdk/thread.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-
-static clib_error_t *
-dpdk_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id)
-{
-  int r;
-  r = rte_eal_remote_launch (fp, (void *) w, lcore_id);
-  if (r)
-    return clib_error_return (0, "Failed to launch thread %u", lcore_id);
-  return 0;
-}
-
-static clib_error_t *
-dpdk_thread_set_lcore (u32 thread, u16 lcore)
-{
-  return 0;
-}
-
-static vlib_thread_callbacks_t callbacks = {
-  .vlib_launch_thread_cb = &dpdk_launch_thread,
-  .vlib_thread_set_lcore_cb = &dpdk_thread_set_lcore,
-};
-
-static clib_error_t *
-dpdk_thread_init (vlib_main_t * vm)
-{
-  vlib_thread_cb_register (vm, &callbacks);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_thread_init);
-
-/** @endcond */
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/virtio/vhost-user.h b/src/vnet/devices/virtio/vhost-user.h
index 3083b614..dd23a909 100644
--- a/src/vnet/devices/virtio/vhost-user.h
+++ b/src/vnet/devices/virtio/vhost-user.h
@@ -328,17 +328,6 @@ typedef struct
 int vhost_user_dump_ifs (vnet_main_t * vnm, vlib_main_t * vm,
 			 vhost_user_intf_details_t ** out_vuids);
 
-// CLI commands to be used from dpdk
-clib_error_t *vhost_user_connect_command_fn (vlib_main_t * vm,
-					     unformat_input_t * input,
-					     vlib_cli_command_t * cmd);
-clib_error_t *vhost_user_delete_command_fn (vlib_main_t * vm,
-					    unformat_input_t * input,
-					    vlib_cli_command_t * cmd);
-clib_error_t *show_vhost_user_command_fn (vlib_main_t * vm,
-					  unformat_input_t * input,
-					  vlib_cli_command_t * cmd);
-
 #endif
 
 /*
diff --git a/src/vnet/ipsec/ipsec_api.c b/src/vnet/ipsec/ipsec_api.c
index 49b475cf..e37bccee 100644
--- a/src/vnet/ipsec/ipsec_api.c
+++ b/src/vnet/ipsec/ipsec_api.c
@@ -79,11 +79,7 @@ static void vl_api_ipsec_spd_add_del_t_handler
   vl_api_ipsec_spd_add_del_reply_t *rmp;
   int rv;
 
-#if DPDK > 0
   rv = ipsec_add_del_spd (vm, ntohl (mp->spd_id), mp->is_add);
-#else
-  rv = VNET_API_ERROR_UNIMPLEMENTED;
-#endif
 
   REPLY_MACRO (VL_API_IPSEC_SPD_ADD_DEL_REPLY);
 #endif
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index e15faeb8..4a65b024 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1212,10 +1212,10 @@ pg_stream_fill_helper (pg_main_t * pg,
 
   /*
    * Historically, the pg maintained its own free lists and
-   * device drivers tx paths would return pkts. With the DPDK,
-   * that doesn't happen.
+   * device drivers tx paths would return pkts.
    */
-  if (DPDK == 0 && !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
+  if (vm->buffer_main->extern_buffer_mgmt == 0 &&
+      !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
     f->buffer_init_function = pg_buffer_init;
   f->buffer_init_function_opaque =
     (s - pg->streams) | ((bi - s->buffer_indices) << 24);
@@ -1238,7 +1238,7 @@ pg_stream_fill_helper (pg_main_t * pg,
   n_alloc = n_allocated;
 
   /* Reinitialize buffers */
-  if (DPDK == 0 || CLIB_DEBUG > 0
+  if (vm->buffer_main->extern_buffer_mgmt == 0 || CLIB_DEBUG > 0
       || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
     init_buffers_inline
       (vm, s,
@@ -1246,7 +1246,8 @@ pg_stream_fill_helper (pg_main_t * pg,
        n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
        s->buffer_bytes,
        /* set_data */
-       DPDK == 1 || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0);
+       vm->buffer_main->extern_buffer_mgmt != 0
+       || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0);
 
   if (next_buffers)
     pg_set_next_buffer_pointers (pg, s, buffers, next_buffers, n_alloc);
diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c
index c46875e1..560c4b07 100644
--- a/src/vnet/pg/stream.c
+++ b/src/vnet/pg/stream.c
@@ -442,9 +442,8 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
     pg_buffer_index_t *bi;
     int n;
 
-#if DPDK > 0
-    s->buffer_bytes = VLIB_BUFFER_DATA_SIZE;
-#endif
+    if (vm->buffer_main->extern_buffer_mgmt)
+      s->buffer_bytes = VLIB_BUFFER_DATA_SIZE;
 
     if (!s->buffer_bytes)
       s->buffer_bytes = s->max_packet_bytes;
diff --git a/src/vnet/replication.c b/src/vnet/replication.c
index 02755195..86d922b5 100644
--- a/src/vnet/replication.c
+++ b/src/vnet/replication.c
@@ -214,9 +214,9 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl)
 	  b0->flags |= VLIB_BUFFER_IS_RECYCLED;
 
 #if (CLIB_DEBUG > 0)
-#if DPDK == 0
-	  vlib_buffer_set_known_state (vm, bi0, VLIB_BUFFER_KNOWN_ALLOCATED);
-#endif
+	  if (vm->buffer_main->extern_buffer_mgmt == 0)
+	    vlib_buffer_set_known_state (vm, bi0,
+					 VLIB_BUFFER_KNOWN_ALLOCATED);
 #endif
 
 	  /* If buffer is traced, mark frame as traced */
diff --git a/src/vnet/vnet_all_api_h.h b/src/vnet/vnet_all_api_h.h
index c4075db6..9d3abae5 100644
--- a/src/vnet/vnet_all_api_h.h
+++ b/src/vnet/vnet_all_api_h.h
@@ -30,9 +30,6 @@
 #endif /* included_from_layer_3 */
 
 #include 
-#if DPDK > 0
-#include 
-#endif
 #include 
 #include 
 #include 
diff --git a/src/vpp/api/custom_dump.c b/src/vpp/api/custom_dump.c
index 4cc6aa73..3871601b 100644
--- a/src/vpp/api/custom_dump.c
+++ b/src/vpp/api/custom_dump.c
@@ -237,58 +237,6 @@ static void *vl_api_sw_interface_set_l2_bridge_t_print
   FINISH;
 }
 
-#if DPDK > 0
-static void *vl_api_sw_interface_set_dpdk_hqos_pipe_t_print
-  (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_pipe ");
-
-  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
-
-  s = format (s, "subport %u  pipe %u  profile %u ",
-	      ntohl (mp->subport), ntohl (mp->pipe), ntohl (mp->profile));
-
-  FINISH;
-}
-
-static void *vl_api_sw_interface_set_dpdk_hqos_subport_t_print
-  (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_subport ");
-
-  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
-
-  s =
-    format (s,
-	    "subport %u  rate %u  bkt_size %u  tc0 %u tc1 %u tc2 %u tc3 %u period %u",
-	    ntohl (mp->subport), ntohl (mp->tb_rate), ntohl (mp->tb_size),
-	    ntohl (mp->tc_rate[0]), ntohl (mp->tc_rate[1]),
-	    ntohl (mp->tc_rate[2]), ntohl (mp->tc_rate[3]),
-	    ntohl (mp->tc_period));
-
-  FINISH;
-}
-
-static void *vl_api_sw_interface_set_dpdk_hqos_tctbl_t_print
-  (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_tctbl ");
-
-  s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index));
-
-  s = format (s, "entry %u  tc %u  queue %u",
-	      ntohl (mp->entry), ntohl (mp->tc), ntohl (mp->queue));
-
-  FINISH;
-}
-#endif
-
 static void *vl_api_bridge_domain_add_del_t_print
   (vl_api_bridge_domain_add_del_t * mp, void *handle)
 {
@@ -3036,18 +2984,6 @@ vl_msg_api_custom_dump_configure (api_main_t * am)
     = (void *) vl_api_##f##_t_print;
   foreach_custom_print_function;
 #undef _
-
-#if DPDK > 0
-  /*
-   * manually add DPDK hqos print handlers
-   */
-  am->msg_print_handlers[VL_API_SW_INTERFACE_SET_DPDK_HQOS_PIPE] =
-    (void *) vl_api_sw_interface_set_dpdk_hqos_pipe_t_print;
-  am->msg_print_handlers[VL_API_SW_INTERFACE_SET_DPDK_HQOS_SUBPORT] =
-    (void *) vl_api_sw_interface_set_dpdk_hqos_subport_t_print;
-  am->msg_print_handlers[VL_API_SW_INTERFACE_SET_DPDK_HQOS_TCTBL] =
-    (void *) vl_api_sw_interface_set_dpdk_hqos_tctbl_t_print;
-#endif
 }
 
 /*
diff --git a/src/vpp/api/gmon.c b/src/vpp/api/gmon.c
index b28608f0..610f40ed 100644
--- a/src/vpp/api/gmon.c
+++ b/src/vpp/api/gmon.c
@@ -137,7 +137,8 @@ gmon_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
       *gm->vector_rate_ptr = vector_rate;
       now = vlib_time_now (vm);
       dt = now - last_runtime;
-      input_packets = vnet_get_aggregate_rx_packets ();
+      // TODO
+      //input_packets = vnet_get_aggregate_rx_packets ();
       *gm->input_rate_ptr = (f64) (input_packets - last_input_packets) / dt;
       last_runtime = now;
       last_input_packets = input_packets;
diff --git a/src/vpp/api/vpe.api b/src/vpp/api/vpe.api
index 2d6e4f37..7f9c2038 100644
--- a/src/vpp/api/vpe.api
+++ b/src/vpp/api/vpe.api
@@ -41,7 +41,6 @@
  * SESSION APIs: .../vnet/session/{session.api session_api.c}
  * MPLS APIs: see .../src/vnet/mpls/{mpls.api, mpls_api.c}
  * SR APIs: see .../src/vnet/sr/{sr.api, sr_api.c}
- * DPDK APIs: see ... /src/vnet/devices/dpdk/{dpdk.api, dpdk_api.c}
  * CLASSIFY APIs: see ... /src/vnet/classify/{classify.api, classify_api.c}
  * FLOW APIs: see ... /src/vnet/flow/{flow.api, flow_api.c}
  * DHCP APIs: see ... /src/vnet/dhcp/{dhcpk.api, dhcp_api.c}
diff --git a/src/vpp/app/l2t.c b/src/vpp/app/l2t.c
deleted file mode 100644
index e1eda155..00000000
--- a/src/vpp/app/l2t.c
+++ /dev/null
@@ -1,562 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-
-#if DPDK == 0
-#include 
-#else
-#include 
-#endif
-
-#include 
-#include 
-#include 
-
-l2t_main_t l2t_main;
-
-/* $$$$ unused?
- * get_interface_ethernet_address
- * paints the ethernet address for a given interface
- * into the supplied destination
- */
-void
-get_interface_ethernet_address (l2t_main_t * lm, u8 * dst, u32 sw_if_index)
-{
-  ethernet_main_t *em = ethernet_get_main (lm->vlib_main);
-  ethernet_interface_t *ei;
-  vnet_hw_interface_t *hi;
-
-  hi = vnet_get_sup_hw_interface (lm->vnet_main, sw_if_index);
-  ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
-  clib_memcpy (dst, ei->address, sizeof (ei->address));
-}
-
-/* packet trace format function */
-u8 *
-format_l2t_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  l2t_trace_t *t = va_arg (*args, l2t_trace_t *);
-
-  if (t->is_user_to_network)
-    s = format (s, "L2T: %U (client) -> %U (our) session %d",
-		format_ip6_address, &t->client_address,
-		format_ip6_address, &t->our_address, t->session_index);
-  else
-    s = format (s, "L2T: %U (our) -> %U (client) session %d)",
-		format_ip6_address, &t->our_address,
-		format_ip6_address, &t->client_address, t->session_index);
-  return s;
-}
-
-u8 *
-format_l2t_session (u8 * s, va_list * args)
-{
-  l2t_session_t *session = va_arg (*args, l2t_session_t *);
-  l2t_main_t *lm = &l2t_main;
-  u32 counter_index;
-  vlib_counter_t v;
-
-  s = format (s, "[%d] %U (our) %U (client) vlan-id %d rx_sw_if_index %d\n",
-	      session - lm->sessions,
-	      format_ip6_address, &session->our_address,
-	      format_ip6_address, &session->client_address,
-	      clib_net_to_host_u16 (session->vlan_id), session->sw_if_index);
-
-  s = format (s, "   local cookie %llx remote cookie %llx\n",
-	      clib_net_to_host_u64 (session->local_cookie),
-	      clib_net_to_host_u64 (session->remote_cookie));
-
-  if (session->cookie_flags & L2TP_COOKIE_ROLLOVER_LOCAL)
-    {
-      s = format (s, "   local rollover cookie %llx\n",
-		  clib_net_to_host_u64 (session->lcl_ro_cookie));
-    }
-
-  s = format (s, "   local session-id %d remote session-id %d\n",
-	      clib_net_to_host_u32 (session->local_session_id),
-	      clib_net_to_host_u32 (session->remote_session_id));
-
-  s = format (s, "   l2 specific sublayer %s\n",
-	      session->l2_sublayer_present ? "preset" : "absent");
-
-  counter_index =
-    session_index_to_counter_index (session - lm->sessions,
-				    SESSION_COUNTER_USER_TO_NETWORK);
-
-  vlib_get_combined_counter (&lm->counter_main, counter_index, &v);
-  if (v.packets != 0)
-    s = format (s, "   user-to-net: %llu pkts %llu bytes\n",
-		v.packets, v.bytes);
-
-  vlib_get_combined_counter (&lm->counter_main, counter_index + 1, &v);
-
-  if (v.packets != 0)
-    s = format (s, "   net-to-user: %llu pkts %llu bytes\n",
-		v.packets, v.bytes);
-  return s;
-}
-
-static clib_error_t *
-show_session_summary_command_fn (vlib_main_t * vm,
-				 unformat_input_t * input,
-				 vlib_cli_command_t * cmd)
-{
-  l2t_main_t *lm = &l2t_main;
-
-  vlib_cli_output (vm, "%d active sessions\n", pool_elts (lm->sessions));
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (show_session_summary_command) = {
-  .path = "show session",
-  .short_help = "show session summary",
-  .function = show_session_summary_command_fn,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-show_session_detail_command_fn (vlib_main_t * vm,
-				unformat_input_t * input,
-				vlib_cli_command_t * cmd)
-{
-  l2t_session_t *session;
-  l2t_main_t *lm = &l2t_main;
-
-  /* *INDENT-OFF* */
-  pool_foreach (session, lm->sessions,
-  ({
-    vlib_cli_output (vm, "%U", format_l2t_session, session);
-  }));
-  /* *INDENT-ON* */
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (show_session_detail_command) = {
-  .path = "show session detail",
-  .short_help = "show session table detail",
-  .function = show_session_detail_command_fn,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-test_counters_command_fn (vlib_main_t * vm,
-			  unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  l2t_session_t *session;
-  l2t_main_t *lm = &l2t_main;
-  u32 session_index;
-  u32 counter_index;
-  u32 nincr = 0;
-
-  /* *INDENT-OFF* */
-  pool_foreach (session, lm->sessions,
-  ({
-    session_index = session - lm->sessions;
-    counter_index =
-      session_index_to_counter_index (session_index,
-                                      SESSION_COUNTER_USER_TO_NETWORK);
-    vlib_increment_combined_counter (&lm->counter_main,
-                                     counter_index,
-                                     1/*pkt*/, 1111 /*bytes*/);
-    vlib_increment_combined_counter (&lm->counter_main,
-                                     counter_index+1,
-                                     1/*pkt*/, 2222 /*bytes*/);
-    nincr++;
-  }));
-  /* *INDENT-ON* */
-  vlib_cli_output (vm, "Incremented %d active counters\n", nincr);
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (test_counters_command) = {
-  .path = "test counters",
-  .short_help = "increment all active counters",
-  .function = test_counters_command_fn,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-clear_counters_command_fn (vlib_main_t * vm,
-			   unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  l2t_session_t *session;
-  l2t_main_t *lm = &l2t_main;
-  u32 session_index;
-  u32 counter_index;
-  u32 nincr = 0;
-
-  /* *INDENT-OFF* */
-  pool_foreach (session, lm->sessions,
-  ({
-    session_index = session - lm->sessions;
-    counter_index =
-      session_index_to_counter_index (session_index,
-                                      SESSION_COUNTER_USER_TO_NETWORK);
-    vlib_zero_combined_counter (&lm->counter_main, counter_index);
-    vlib_zero_combined_counter (&lm->counter_main, counter_index+1);
-    nincr++;
-  }));
-  /* *INDENT-ON* */
-  vlib_cli_output (vm, "Cleared %d active counters\n", nincr);
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (clear_counters_command) = {
-  .path = "clear counters",
-  .short_help = "clear all active counters",
-  .function = clear_counters_command_fn,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-l2tp_session_add_command_fn (vlib_main_t * vm,
-			     unformat_input_t * input,
-			     vlib_cli_command_t * cmd)
-{
-  ip6_address_t client_address, our_address;
-  ip6_address_t *dst_address_copy, *src_address_copy;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  u32 vlan_id;
-  u32 sw_if_index = (u32) ~ 0;
-  l2t_main_t *lm = &l2t_main;
-  l2t_session_t *s;
-  uword *p;
-  vnet_hw_interface_t *hi;
-  vnet_sw_interface_t *si;
-  u32 next_index;
-  uword vlan_and_sw_if_index_key;
-  u32 counter_index;
-  u64 local_cookie = (u64) ~ 0, remote_cookie = (u64) ~ 0;
-  u32 local_session_id = 1, remote_session_id = 1;
-  int our_address_set = 0, client_address_set = 0;
-  int l2_sublayer_present = 0;
-  clib_error_t *error = NULL;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "client %U",
-		    unformat_ip6_address, &client_address))
-	client_address_set = 1;
-      else if (unformat (line_input, "our %U",
-			 unformat_ip6_address, &our_address))
-	our_address_set = 1;
-      else if (unformat (line_input, "vlan %d", &vlan_id))
-	;
-      else if (unformat (line_input, "l2-interface %U",
-			 unformat_vnet_sw_interface,
-			 vnet_get_main (), &sw_if_index))
-	;
-      else if (unformat (line_input, "interface %U",
-			 unformat_vnet_sw_interface,
-			 vnet_get_main (), &sw_if_index))
-	;
-      else if (unformat (line_input, "local-cookie %llx", &local_cookie))
-	;
-      else if (unformat (line_input, "remote-cookie %llx", &remote_cookie))
-	;
-      else if (unformat (line_input, "local-session-id %d",
-			 &local_session_id))
-	;
-      else if (unformat (line_input, "remote-session-id %d",
-			 &remote_session_id))
-	;
-      else if (unformat (line_input, "l2-sublayer-present"))
-	l2_sublayer_present = 1;
-      else
-	{
-	  error = clib_error_return (0, "parse error: '%U'",
-				     format_unformat_error, line_input);
-	  unformat_free (line_input);
-	  return error;
-	}
-    }
-
-  unformat_free (line_input);
-
-  if (sw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "l2-interface not specified");
-  if (our_address_set == 0)
-    return clib_error_return (0, "our address not specified");
-  if (client_address_set == 0)
-    return clib_error_return (0, "client address not specified");
-
-  remote_session_id = clib_host_to_net_u32 (remote_session_id);
-  local_session_id = clib_host_to_net_u32 (local_session_id);
-
-  switch (lm->lookup_type)
-    {
-    case L2T_LOOKUP_SRC_ADDRESS:
-      p = hash_get_mem (lm->session_by_src_address, &client_address);
-      if (p)
-	return clib_error_return
-	  (0, "Session w/ client address %U already exists",
-	   format_ip6_address, &client_address);
-      break;
-
-    case L2T_LOOKUP_DST_ADDRESS:
-      p = hash_get_mem (lm->session_by_dst_address, &our_address);
-      if (p)
-	return clib_error_return
-	  (0, "Session w/ our address %U already exists",
-	   format_ip6_address, &our_address);
-      break;
-
-    case L2T_LOOKUP_SESSION_ID:
-      p = hash_get (lm->session_by_session_id, local_session_id);
-      if (p)
-	return clib_error_return
-	  (0,
-	   "Session w/ local session id %d already exists",
-	   clib_net_to_host_u32 (local_session_id));
-      break;
-
-    default:
-      ASSERT (0);
-    }
-
-  pool_get (lm->sessions, s);
-  memset (s, 0, sizeof (*s));
-  clib_memcpy (&s->our_address, &our_address, sizeof (s->our_address));
-  clib_memcpy (&s->client_address, &client_address,
-	       sizeof (s->client_address));
-  s->sw_if_index = sw_if_index;
-  s->vlan_id = clib_host_to_net_u16 (vlan_id);
-  s->local_cookie = clib_host_to_net_u64 (local_cookie);
-  l2tp_session_set_remote_cookie (s, remote_cookie);
-  s->local_session_id = local_session_id;
-  s->remote_session_id = remote_session_id;
-  s->l2_sublayer_present = l2_sublayer_present;
-
-  hi = vnet_get_sup_hw_interface (lm->vnet_main, sw_if_index);
-  si = vnet_get_sup_sw_interface (lm->vnet_main, sw_if_index);
-
-  next_index = vlib_node_add_next (vm, l2t_ip6_node.index,
-				   hi->output_node_index);
-  s->l2_output_next_index = next_index;
-  s->l2_output_sw_if_index = si->sw_if_index;
-
-  /* Setup hash table entries */
-  switch (lm->lookup_type)
-    {
-    case L2T_LOOKUP_SRC_ADDRESS:
-      src_address_copy = clib_mem_alloc (sizeof (*src_address_copy));
-      clib_memcpy (src_address_copy, &client_address,
-		   sizeof (*src_address_copy));
-      hash_set_mem (lm->session_by_src_address, src_address_copy,
-		    s - lm->sessions);
-      break;
-    case L2T_LOOKUP_DST_ADDRESS:
-      dst_address_copy = clib_mem_alloc (sizeof (*dst_address_copy));
-      clib_memcpy (dst_address_copy, &our_address,
-		   sizeof (*dst_address_copy));
-      hash_set_mem (lm->session_by_dst_address, dst_address_copy,
-		    s - lm->sessions);
-      break;
-    case L2T_LOOKUP_SESSION_ID:
-      hash_set (lm->session_by_session_id, local_session_id,
-		s - lm->sessions);
-      break;
-
-    default:
-      ASSERT (0);
-    }
-
-  vlan_and_sw_if_index_key = ((uword) (s->vlan_id) << 32) | sw_if_index;
-  hash_set (lm->session_by_vlan_and_rx_sw_if_index,
-	    vlan_and_sw_if_index_key, s - lm->sessions);
-
-  /* validate counters */
-  counter_index =
-    session_index_to_counter_index (s - lm->sessions,
-				    SESSION_COUNTER_USER_TO_NETWORK);
-  vlib_validate_counter (&lm->counter_main, counter_index);
-  vlib_validate_counter (&lm->counter_main, counter_index + 1);
-
-  /* Set promiscuous mode on the l2 interface */
-  ethernet_set_flags (lm->vnet_main, hi->hw_if_index,
-		      ETHERNET_INTERFACE_FLAG_ACCEPT_ALL);
-  vnet_hw_interface_rx_redirect_to_node (lm->vnet_main, hi->hw_if_index,
-					 l2t_l2_node.index);
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (l2tp_session_add_command) = {
-  .path = "l2tp session add",
-  .short_help =
-  "l2tp session add client  our  vlan  local-cookie  remote-cookie  local-session  remote-session  l2-interface ",
-  .function = l2tp_session_add_command_fn,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-l2tp_session_del_command_fn (vlib_main_t * vm,
-			     unformat_input_t * input,
-			     vlib_cli_command_t * cmd)
-{
-  l2t_main_t *lm = &l2t_main;
-  u32 session_index;
-  l2t_session_t *s;
-  hash_pair_t *hp;
-  void *key;
-  uword vlan_and_sw_if_index_key;
-
-  if (!unformat (input, "%d", &session_index))
-    return clib_error_return (0, "missing session index: '%U'",
-			      format_unformat_error, input);
-
-  if (pool_is_free_index (lm->sessions, session_index))
-    return clib_error_return (0, "session %d not in use", session_index);
-
-  s = pool_elt_at_index (lm->sessions, session_index);
-
-  switch (lm->lookup_type)
-    {
-    case L2T_LOOKUP_SRC_ADDRESS:
-      hp = hash_get_pair_mem (lm->session_by_src_address, &s->client_address);
-      if (hp)
-	{
-	  key = (void *) (hp->key);
-	  hash_unset_mem (lm->session_by_src_address, &s->client_address);
-	  clib_mem_free (key);
-	}
-      else
-	clib_warning ("session %d src address key %U AWOL",
-		      s - lm->sessions,
-		      format_ip6_address, &s->client_address);
-      break;
-
-    case L2T_LOOKUP_DST_ADDRESS:
-      hp = hash_get_pair_mem (lm->session_by_dst_address, &s->our_address);
-      if (hp)
-	{
-	  key = (void *) (hp->key);
-	  hash_unset_mem (lm->session_by_dst_address, &s->our_address);
-	  clib_mem_free (key);
-	}
-      else
-	clib_warning ("session %d dst address key %U AWOL",
-		      s - lm->sessions, format_ip6_address, &s->our_address);
-      break;
-
-    case L2T_LOOKUP_SESSION_ID:
-      hash_unset (lm->session_by_session_id, s->local_session_id);
-      break;
-
-    default:
-      ASSERT (0);
-    }
-
-  vlan_and_sw_if_index_key = ((uword) (s->vlan_id) << 32) | s->sw_if_index;
-
-  hash_unset (lm->session_by_vlan_and_rx_sw_if_index,
-	      vlan_and_sw_if_index_key);
-
-  pool_put (lm->sessions, s);
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (l2tp_session_del_command) = {
-  .path = "l2tp session delete",
-  .short_help =
-  "l2tp session delete ",
-  .function = l2tp_session_del_command_fn,
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-l2tp_session_cookie_command_fn (vlib_main_t * vm,
-				unformat_input_t * input,
-				vlib_cli_command_t * cmd)
-{
-  l2t_main_t *lm = &l2t_main;
-  u32 session_index;
-  l2t_session_t *s;
-  u64 lcl_ro_cookie = (u64) ~ 0, rem_ro_cookie = (u64) ~ 0;
-  u8 cookie_flags = 0;
-
-  if (!unformat (input, "%d", &session_index))
-    return clib_error_return (0, "missing session index: '%U'",
-			      format_unformat_error, input);
-
-  if (pool_is_free_index (lm->sessions, session_index))
-    return clib_error_return (0, "session %d not in use", session_index);
-
-  s = pool_elt_at_index (lm->sessions, session_index);
-
-  if (unformat (input, "commit"))
-    {
-      if (!s->cookie_flags)
-	{
-	  return clib_error_return (0, "no rollover cookie ready to commit");
-	}
-      else
-	{
-	  l2tp_session_cookie_commit (s);
-	  return 0;
-	}
-    }
-  if (!unformat (input, "rollover"))
-    return clib_error_return (0, "missing 'commit|rollover': '%U'",
-			      format_unformat_error, input);
-  if (unformat (input, "local %llx", &lcl_ro_cookie))
-    {
-      cookie_flags |= L2TP_COOKIE_ROLLOVER_LOCAL;
-      l2tp_session_set_local_rollover_cookie (s, lcl_ro_cookie);
-    }
-  if (unformat (input, "remote %llx", &rem_ro_cookie))
-    {
-      cookie_flags |= L2TP_COOKIE_ROLLOVER_REMOTE;
-      l2tp_session_set_remote_cookie (s, rem_ro_cookie);
-    }
-  if (!cookie_flags)
-    return clib_error_return (0, "no rollover cookie specified");
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-static VLIB_CLI_COMMAND (l2tp_session_cookie_command) = {
-  .path = "l2tp session cookie",
-  .short_help =
-  "l2tp session cookie  commit|rollover [local ] [remote ]",
-  .function = l2tp_session_cookie_command_fn,
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vpp/app/l2t_l2.c b/src/vpp/app/l2t_l2.c
deleted file mode 100644
index 07d30d9a..00000000
--- a/src/vpp/app/l2t_l2.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-
-#if DPDK == 0
-#include 
-#include 
-#include 
-#else
-#include 
-#endif
-
-#include 
-#include 
-#include 
-
-l2t_main_t l2t_main;
-
-/* Statistics (not really errors) */
-#define foreach_l2t_l2_error                       \
-_(NETWORK_TO_USER, "L2 network to user (ip6) pkts")
-
-static char *l2t_l2_error_strings[] = {
-#define _(sym,string) string,
-  foreach_l2t_l2_error
-#undef _
-};
-
-typedef enum
-{
-#define _(sym,str) L2T_L2_ERROR_##sym,
-  foreach_l2t_l2_error
-#undef _
-    L2T_L2_N_ERROR,
-} l2t_l2_error_t;
-
-/*
- * Packets go to ethernet-input when they don't match a mapping
- */
-typedef enum
-{
-  L2T_L2_NEXT_DROP,
-  L2T_L2_NEXT_ETHERNET_INPUT,
-  L2T_L2_NEXT_IP6_LOOKUP,
-  L2T_L2_N_NEXT,
-} l2t_l2_next_t;
-
-vlib_node_registration_t l2t_l2_node;
-
-#define NSTAGES 3
-
-static inline void
-stage0 (vlib_main_t * vm, vlib_node_runtime_t * node, u32 buffer_index)
-{
-  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
-  vlib_prefetch_buffer_header (b, STORE);
-  CLIB_PREFETCH (b->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
-}
-
-static inline void
-stage1 (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bi)
-{
-  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
-  l2t_main_t *lm = &l2t_main;
-  ethernet_header_t *eh;
-  ethernet_vlan_header_t *vh;
-  u32 session_index;
-  uword *p;
-  uword vlan_and_sw_if_index_key;
-
-  /* just in case, needed to test with the tun/tap device */
-  vlib_buffer_reset (b);
-
-  eh = vlib_buffer_get_current (b);
-
-  /* Not a VLAN pkt? send to ethernet-input... */
-  if (PREDICT_FALSE (eh->type != clib_host_to_net_u16 (0x8100)))
-    {
-      vnet_buffer (b)->l2t.next_index = L2T_L2_NEXT_ETHERNET_INPUT;
-      return;
-    }
-  vh = (ethernet_vlan_header_t *) (eh + 1);
-
-  /* look up session */
-  vlan_and_sw_if_index_key = ((uword) (vh->priority_cfi_and_id) << 32)
-    | vnet_buffer (b)->sw_if_index[VLIB_RX];
-
-  p = hash_get (lm->session_by_vlan_and_rx_sw_if_index,
-		vlan_and_sw_if_index_key);
-
-  if (PREDICT_FALSE (p == 0))
-    {
-      /* $$$ drop here if not for our MAC? */
-      vnet_buffer (b)->l2t.next_index = L2T_L2_NEXT_ETHERNET_INPUT;
-      return;
-    }
-  else
-    {
-      session_index = p[0];
-    }
-
-  /* Remember mapping index, prefetch the mini counter */
-  vnet_buffer (b)->l2t.next_index = L2T_L2_NEXT_IP6_LOOKUP;
-  vnet_buffer (b)->l2t.session_index = session_index;
-
-  /* Each mapping has 2 x (pkt, byte) counters, hence the shift */
-  CLIB_PREFETCH (lm->counter_main.mini + (p[0] << 1), CLIB_CACHE_LINE_BYTES,
-		 STORE);
-}
-
-static inline u32
-last_stage (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bi)
-{
-  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
-  l2t_main_t *lm = &l2t_main;
-  ethernet_header_t *eh = vlib_buffer_get_current (b);
-  vlib_node_t *n = vlib_get_node (vm, l2t_l2_node.index);
-  u32 node_counter_base_index = n->error_heap_index;
-  vlib_error_main_t *em = &vm->error_main;
-  l2tpv3_header_t *l2t;		/* l2 header */
-  ethernet_vlan_header_t *vh;	/* 802.1q vlan header */
-  u32 counter_index;
-  l2t_session_t *s;
-  ip6_header_t *ip6;
-  u16 payload_ethertype;
-  u8 dst_mac_address[6];
-  u8 src_mac_address[6];
-  u16 payload_length;
-  i32 backup;
-
-  /* Other-than-output pkt? We're done... */
-  if (vnet_buffer (b)->l2t.next_index != L2T_L2_NEXT_IP6_LOOKUP)
-    return vnet_buffer (b)->l2t.next_index;
-
-  vh = (ethernet_vlan_header_t *) (eh + 1);
-
-  em->counters[node_counter_base_index + L2T_L2_ERROR_NETWORK_TO_USER] += 1;
-
-  counter_index =
-    session_index_to_counter_index (vnet_buffer (b)->l2t.session_index,
-				    SESSION_COUNTER_NETWORK_TO_USER);
-
-  /* per-mapping byte stats include the ethernet header */
-  vlib_increment_combined_counter (&lm->counter_main, counter_index,
-				   1 /* packet_increment */ ,
-				   vlib_buffer_length_in_chain (vm, b) +
-				   sizeof (ethernet_header_t));
-
-  s = pool_elt_at_index (lm->sessions, vnet_buffer (b)->l2t.session_index);
-
-  /* Save src/dst MAC addresses */
-#define _(i)  dst_mac_address[i] = eh->dst_address[i];
-  _(0) _(1) _(2) _(3) _(4) _(5);
-#undef _
-#define _(i)  src_mac_address[i] = eh->src_address[i];
-  _(0) _(1) _(2) _(3) _(4) _(5);
-#undef _
-
-  payload_ethertype = vh->type;
-
-  /* Splice out the 802.1q vlan tag */
-  vlib_buffer_advance (b, 4);
-  eh = vlib_buffer_get_current (b);
-
-  /* restore src/dst MAC addresses */
-#define _(i)   eh->dst_address[i] = dst_mac_address[i];
-  _(0) _(1) _(2) _(3) _(4) _(5);
-#undef _
-#define _(i)  eh->src_address[i] = src_mac_address[i];
-  _(0) _(1) _(2) _(3) _(4) _(5);
-#undef _
-  eh->type = payload_ethertype;
-
-  /* Paint on an l2tpv3 hdr */
-  backup = sizeof (*l2t);
-#if 0
-  /* back up 4 bytes less if no l2 sublayer */
-  backup -= s->l2_sublayer_present ? 0 : 4;
-#endif
-
-  vlib_buffer_advance (b, -backup);
-  l2t = vlib_buffer_get_current (b);
-
-  l2t->session_id = s->remote_session_id;
-  l2t->cookie = s->remote_cookie;
-
-#if 0
-  if (s->l2_sublayer_present)
-    l2t->l2_specific_sublayer = 0;
-#endif
-
-  /* Paint on an ip6 header */
-  vlib_buffer_advance (b, -(sizeof (*ip6)));
-  ip6 = vlib_buffer_get_current (b);
-
-  ip6->ip_version_traffic_class_and_flow_label =
-    clib_host_to_net_u32 (0x6 << 28);
-
-  /* calculate ip6 payload length */
-  payload_length = vlib_buffer_length_in_chain (vm, b);
-  payload_length -= sizeof (*ip6);
-
-  ip6->payload_length = clib_host_to_net_u16 (payload_length);
-  ip6->protocol = 0x73;		/* l2tpv3 */
-  ip6->hop_limit = 0xff;
-  ip6->src_address.as_u64[0] = s->our_address.as_u64[0];
-  ip6->src_address.as_u64[1] = s->our_address.as_u64[1];
-  ip6->dst_address.as_u64[0] = s->client_address.as_u64[0];
-  ip6->dst_address.as_u64[1] = s->client_address.as_u64[1];
-
-  return L2T_L2_NEXT_IP6_LOOKUP;
-}
-
-#include 
-
-static uword
-l2t_l2_node_fn (vlib_main_t * vm,
-		vlib_node_runtime_t * node, vlib_frame_t * frame)
-{
-  return dispatch_pipeline (vm, node, frame);
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (l2t_l2_node) = {
-  .function = l2t_l2_node_fn,
-  .name = "l2t-l2-input",
-  .vector_size = sizeof (u32),
-  .format_trace = format_l2t_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(l2t_l2_error_strings),
-  .error_strings = l2t_l2_error_strings,
-
-  .n_next_nodes = L2T_L2_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [L2T_L2_NEXT_IP6_LOOKUP] = "ip6-lookup",
-    [L2T_L2_NEXT_ETHERNET_INPUT] = "ethernet-input",
-    [L2T_L2_NEXT_DROP] = "error-drop",
-  },
-};
-/* *INDENT-ON* */
-
-VLIB_NODE_FUNCTION_MULTIARCH (l2t_l2_node, l2t_l2_node_fn);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
-- 
cgit 1.2.3-korg


From fb38095d1c9d1b84850f345f0344f82b9ae2c375 Mon Sep 17 00:00:00 2001
From: Pablo Camarillo 
Date: Wed, 7 Dec 2016 18:34:18 +0100
Subject: Evolving SRv6 (Segment Routing for IPv6)

Implements:
1.- SR Policies with several (weighted) SID lists
2.- Binding SID
3.- SR LocalSIDs with support for the following functions
	- End
	- End.X
	- End.DX6
	- End.DX4
	- End.DX2
	- End.DT6
	- End.DT2
	- End.B6
	- End.B6.Encaps
4.- SR Steering policies (to steer a traffic through an SR Policy)
	- Support for IPv6 traffic (IPv6 Encapsulation / SRH insertion)
	- Support for IPv4 traffic (IPv6 Encapsulation)
	- Support for L2 traffic
	(T.Insert / T.Encaps)
5.- Doxygen documentation
6.- Framework (APIs) to allow the definition of new SR LocalSID behaviors by means of plugins
7.- Sample SRv6 LocalSID plugin

Change-Id: I2de3d126699d4f11f54c0f7f3b71420ea41fd389
Signed-off-by: Pablo Camarillo 
---
 src/configure.ac                                   |    1 +
 src/plugins/Makefile.am                            |    4 +
 src/plugins/sample_srv6_localsid.am                |   23 +
 src/plugins/srv6-localsid/node.c                   |  200 ++
 src/plugins/srv6-localsid/srv6_localsid_sample.c   |  179 ++
 src/plugins/srv6-localsid/srv6_localsid_sample.h   |   61 +
 .../srv6-localsid/srv6_sample_localsid_doc.md      |   38 +
 src/vat/api_format.c                               |  357 +-
 src/vnet.am                                        |    6 +-
 src/vnet/ip/ip6_packet.h                           |   33 +-
 src/vnet/sr/dir.dox                                |    4 +-
 src/vnet/sr/examples/sr_multicastmap.script        |    4 -
 src/vnet/sr/ietf_draft_05.txt                      | 1564 +++++++++
 src/vnet/sr/rfc_draft_05.txt                       | 1265 --------
 src/vnet/sr/sr.api                                 |  203 +-
 src/vnet/sr/sr.c                                   | 3397 +-------------------
 src/vnet/sr/sr.h                                   |  361 ++-
 src/vnet/sr/sr_api.c                               |  250 +-
 src/vnet/sr/sr_doc.md                              |  161 +
 src/vnet/sr/sr_error.def                           |   20 -
 src/vnet/sr/sr_fix_dst_error.def                   |   17 -
 src/vnet/sr/sr_localsid.c                          | 1478 +++++++++
 src/vnet/sr/sr_packet.h                            |  248 +-
 src/vnet/sr/sr_policy_rewrite.c                    | 3253 +++++++++++++++++++
 src/vnet/sr/sr_steering.c                          |  568 ++++
 src/vpp/api/custom_dump.c                          |  270 +-
 src/vpp/api/test_client.c                          |    1 -
 27 files changed, 8352 insertions(+), 5614 deletions(-)
 create mode 100644 src/plugins/sample_srv6_localsid.am
 create mode 100644 src/plugins/srv6-localsid/node.c
 create mode 100755 src/plugins/srv6-localsid/srv6_localsid_sample.c
 create mode 100644 src/plugins/srv6-localsid/srv6_localsid_sample.h
 create mode 100644 src/plugins/srv6-localsid/srv6_sample_localsid_doc.md
 mode change 100644 => 100755 src/vnet/sr/dir.dox
 delete mode 100644 src/vnet/sr/examples/sr_multicastmap.script
 create mode 100755 src/vnet/sr/ietf_draft_05.txt
 delete mode 100644 src/vnet/sr/rfc_draft_05.txt
 mode change 100644 => 100755 src/vnet/sr/sr.c
 mode change 100644 => 100755 src/vnet/sr/sr.h
 create mode 100644 src/vnet/sr/sr_doc.md
 delete mode 100644 src/vnet/sr/sr_error.def
 delete mode 100644 src/vnet/sr/sr_fix_dst_error.def
 create mode 100755 src/vnet/sr/sr_localsid.c
 mode change 100644 => 100755 src/vnet/sr/sr_packet.h
 create mode 100755 src/vnet/sr/sr_policy_rewrite.c
 create mode 100755 src/vnet/sr/sr_steering.c

(limited to 'src/plugins/Makefile.am')

diff --git a/src/configure.ac b/src/configure.ac
index 813fe067..c22d152e 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -153,6 +153,7 @@ PLUGIN_ENABLED(ioam)
 PLUGIN_ENABLED(lb)
 PLUGIN_ENABLED(sixrd)
 PLUGIN_ENABLED(snat)
+PLUGIN_DISABLED(srv6sample)
 
 ###############################################################################
 # Dependency checks
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index c8877899..7b36049e 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -61,6 +61,10 @@ if ENABLE_SNAT_PLUGIN
 include snat.am
 endif
 
+if ENABLE_SRV6SAMPLE_PLUGIN
+include sample_srv6_localsid.am
+endif
+
 include ../suffix-rules.mk
 
 # Remove *.la files
diff --git a/src/plugins/sample_srv6_localsid.am b/src/plugins/sample_srv6_localsid.am
new file mode 100644
index 00000000..a820ab25
--- /dev/null
+++ b/src/plugins/sample_srv6_localsid.am
@@ -0,0 +1,23 @@
+
+# Copyright (c)  
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppplugins_LTLIBRARIES += sample_srv6_localsid_plugin.la
+
+sample_srv6_localsid_plugin_la_SOURCES =	\
+	srv6-localsid/node.c					\
+	srv6-localsid/srv6_localsid_sample.c
+
+noinst_HEADERS +=	srv6-localsid/srv6_localsid_sample.h
+
+# vi:syntax=automake
diff --git a/src/plugins/srv6-localsid/node.c b/src/plugins/srv6-localsid/node.c
new file mode 100644
index 00000000..7bae9cd7
--- /dev/null
+++ b/src/plugins/srv6-localsid/node.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+typedef struct {
+  u32 localsid_index;
+} srv6_localsid_sample_trace_t;
+
+/* packet trace format function */
+static u8 * format_srv6_localsid_sample_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  srv6_localsid_sample_trace_t * t = va_arg (*args, srv6_localsid_sample_trace_t *);
+  s = format (s, "SRv6-sample-localsid: localsid_index %d\n",
+              t->localsid_index);
+  return s;
+}
+
+vlib_node_registration_t srv6_localsid_sample_node;
+
+#define foreach_srv6_localsid_counter \
+_(PROCESSED, "srv6-sample-localsid processed packets") \
+_(NO_SRH, "(Error) No SRH.")
+
+typedef enum {
+#define _(sym,str) SRV6_LOCALSID_COUNTER_##sym,
+  foreach_srv6_localsid_counter
+#undef _
+  SRV6_LOCALSID_N_COUNTERS,
+} srv6_localsid_sample_counters;
+
+static char * srv6_localsid_counter_strings[] = {
+#define _(sym,string) string,
+  foreach_srv6_localsid_counter
+#undef _
+};
+
+typedef enum {
+  SRV6_SAMPLE_LOCALSID_NEXT_ERROR,
+  SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP,
+  SRV6_SAMPLE_LOCALSID_N_NEXT,
+} srv6_localsid_sample_next_t;
+
+/**
+ * @brief Function doing End processing. 
+ */
+//Fixme: support OAM (hop-by-hop header) here!
+static_always_inline void
+end_srh_processing (vlib_node_runtime_t * node,
+                    vlib_buffer_t * b0,
+                    ip6_header_t * ip0,
+                    ip6_sr_header_t * sr0,
+                    u32 * next0)
+{
+  ip6_address_t *new_dst0;
+
+  if(PREDICT_TRUE(ip0->protocol == IP_PROTOCOL_IPV6_ROUTE))
+  {
+    if(PREDICT_TRUE(sr0->type == ROUTING_HEADER_TYPE_SR))
+    {
+      if(PREDICT_TRUE(sr0->segments_left != 0))
+      {
+        sr0->segments_left -= 1;
+        new_dst0 = (ip6_address_t *)(sr0->segments);
+        new_dst0 += sr0->segments_left;
+        ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
+        ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
+      }
+      else
+      {
+        *next0 = SRV6_SAMPLE_LOCALSID_NEXT_ERROR;
+        b0->error = node->errors[SRV6_LOCALSID_COUNTER_NO_SRH];
+      }
+    }
+    else
+    {
+      /* Error. Routing header of type != SR */
+      *next0 = SRV6_SAMPLE_LOCALSID_NEXT_ERROR;
+      b0->error = node->errors[SRV6_LOCALSID_COUNTER_NO_SRH];
+    }
+  }
+}
+
+/*
+ * @brief SRv6 Sample Localsid graph node
+ * WARNING: YOU MUST DO THE DUAL LOOP
+ */
+static uword
+srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  u32 next_index;
+  u32 pkts_swapped = 0;
+  
+  ip6_sr_main_t * sm = &sr_main;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+  u32 cpu_index = os_get_cpu_number ();
+
+  while (n_left_from > 0)
+  {
+    u32 n_left_to_next;
+
+    vlib_get_next_frame (vm, node, next_index,
+		   to_next, n_left_to_next);
+
+    while (n_left_from > 0 && n_left_to_next > 0)
+    {
+      u32 bi0;
+      vlib_buffer_t * b0;
+      ip6_header_t * ip0 = 0;
+      ip6_sr_header_t * sr0;
+      u32 next0 = SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP;
+      ip6_sr_localsid_t *ls0;
+      srv6_localsid_sample_per_sid_memory_t *ls0_mem;
+
+      bi0 = from[0];
+      to_next[0] = bi0;
+      from += 1;
+      to_next += 1;
+      n_left_from -= 1;
+      n_left_to_next -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      ip0 = vlib_buffer_get_current (b0);
+      sr0 = (ip6_sr_header_t *)(ip0+1);
+
+      /* Lookup the SR End behavior based on IP DA (adj) */
+      ls0 = pool_elt_at_index (sm->localsids, vnet_buffer(b0)->ip.adj_index[VLIB_TX]);
+      ls0_mem = ls0->plugin_mem;
+
+      /* SRH processing */
+      end_srh_processing (node, b0, ip0, sr0, &next0);
+
+      /* ==================================================================== */
+      /* INSERT CODE HERE */
+      /* Example starts here */
+      //In this example we are changing the next VRF table by the one in CLI
+      vnet_buffer(b0)->sw_if_index[VLIB_TX] = ls0_mem->fib_table;
+      /* Example finishes here */
+      /* ==================================================================== */
+
+      if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) 
+      {
+        srv6_localsid_sample_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+        tr->localsid_index = ls0 - sm->localsids;
+      }
+
+      /* This increments the SRv6 per LocalSID counters.*/
+      vlib_increment_combined_counter
+        (((next0 == SRV6_SAMPLE_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : &(sm->sr_ls_valid_counters)),
+        cpu_index,
+        ls0 - sm->localsids,
+        1, vlib_buffer_length_in_chain (vm, b0));
+
+      vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, 
+        n_left_to_next, bi0, next0);
+
+      pkts_swapped ++;
+    }
+    vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+
+  }
+
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (srv6_localsid_sample_node) = {
+  .function = srv6_localsid_sample_fn,
+  .name = "srv6-localsid-sample",
+  .vector_size = sizeof (u32),
+  .format_trace = format_srv6_localsid_sample_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SRV6_LOCALSID_N_COUNTERS,
+  .error_strings = srv6_localsid_counter_strings,
+  .n_next_nodes = SRV6_SAMPLE_LOCALSID_N_NEXT,
+  .next_nodes = {
+        [SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP] = "ip6-lookup",
+        [SRV6_SAMPLE_LOCALSID_NEXT_ERROR] = "error-drop",
+    },
+};
diff --git a/src/plugins/srv6-localsid/srv6_localsid_sample.c b/src/plugins/srv6-localsid/srv6_localsid_sample.c
new file mode 100755
index 00000000..ec16547e
--- /dev/null
+++ b/src/plugins/srv6-localsid/srv6_localsid_sample.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ *------------------------------------------------------------------
+ * srv6_localsid_sample.c - Simple SRv6 LocalSID
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+unsigned char srv6_localsid_name[32] = "Sample-SRv6-LocalSID-plugin";
+unsigned char keyword_str[32] = "new_srv6_localsid";
+unsigned char def_str[64] = "This is a definition of a sample new_srv6_localsid";
+unsigned char params_str[32] = "";
+
+/*****************************************/
+/* SRv6 LocalSID instantiation and removal functions */
+static int
+srv6_localsid_creation_fn (ip6_sr_localsid_t *localsid)
+{
+  /* 
+   * Do you want to do anything fancy upon localsid instantiation?
+   * You can do it here
+   * (If return != 0 the localsid creation will be cancelled.)
+   */
+  /* As an example Im going to do a +1 to the fib table inserted by the user */
+  srv6_localsid_sample_per_sid_memory_t *ls_mem = localsid->plugin_mem;
+  ls_mem->fib_table += 1;
+  return 0;
+}
+
+static int
+srv6_localsid_removal_fn (ip6_sr_localsid_t *localsid)
+{
+  /* Do you want to do anything fancy upon localsid removal?
+   * You can do it here
+   * (If return != 0 the localsid removal will be cancelled.)
+   */
+  /* 
+   * BTW if you stored something in localsid->plugin_mem you should clean it now
+   */
+
+  //In this example we are only cleaning the memory allocated per localsid
+  clib_mem_free(localsid->plugin_mem);
+  return 0;
+}
+
+/**********************************/
+/* SRv6 LocalSID format functions */
+/*
+ * Prints nicely the parameters of a localsid
+ * Example: print "Table 5"
+ */
+u8 *
+format_srv6_localsid_sample (u8 * s, va_list * args)
+{
+  srv6_localsid_sample_per_sid_memory_t *ls_mem = va_arg (*args, void *);
+  return (format (s, "Table: %u", ls_mem->fib_table));
+}
+
+/*
+ * Process the parameters of a localsid
+ * Example: process from:
+ * sr localsid address cafe::1 behavior new_srv6_localsid 5
+ * everything from behavior on... so in this case 'new_srv6_localsid 5'
+ * Notice that it MUST match the keyword_str and params_str defined above.
+ */
+uword
+unformat_srv6_localsid_sample (unformat_input_t * input, va_list * args)
+{
+  void **plugin_mem = va_arg (*args, void **);
+  srv6_localsid_sample_per_sid_memory_t *ls_mem;
+  u32 table_id;
+  if (unformat (input, "new_srv6_localsid %u", &table_id))
+    {
+      /* Allocate a portion of memory */
+      ls_mem = clib_mem_alloc_aligned_at_offset (
+        sizeof(srv6_localsid_sample_per_sid_memory_t), 0, 0, 1);
+
+      /* Set to zero the memory */
+      memset (ls_mem, 0, sizeof(srv6_localsid_sample_per_sid_memory_t));
+
+      /* Our brand-new car is ready */
+      ls_mem->fib_table = table_id;
+
+      /* Dont forget to add it to the localsid */
+      *plugin_mem = ls_mem;
+      return 1;
+    }
+  return 0;
+}
+
+/*************************/
+/* SRv6 LocalSID FIB DPO */
+static u8 *
+format_srv6_localsid_sample_dpo (u8 * s, va_list * args)
+{
+  index_t index = va_arg (*args, index_t);
+  CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
+
+  return (format (s, "SR: localsid_sample_index:[%u]", index));
+}
+
+void
+srv6_localsid_sample_dpo_lock (dpo_id_t * dpo)
+{
+}
+
+void
+srv6_localsid_sample_dpo_unlock (dpo_id_t * dpo)
+{
+}
+
+const static dpo_vft_t srv6_localsid_sample_vft = {
+  .dv_lock = srv6_localsid_sample_dpo_lock,
+  .dv_unlock = srv6_localsid_sample_dpo_unlock,
+  .dv_format = format_srv6_localsid_sample_dpo,
+};
+
+const static char *const srv6_localsid_sample_ip6_nodes[] = {
+  "srv6-localsid-sample",
+  NULL,
+};
+
+const static char *const *const srv6_localsid_sample_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = srv6_localsid_sample_ip6_nodes,
+};
+
+/**********************/
+static clib_error_t * srv6_localsid_sample_init (vlib_main_t * vm)
+{
+  srv6_localsid_sample_main_t * sm = &srv6_localsid_sample_main;
+  int rv = 0;
+  /* Create DPO */
+  sm->srv6_localsid_sample_dpo_type = dpo_register_new_type (
+    &srv6_localsid_sample_vft, srv6_localsid_sample_nodes);
+
+  /* Register SRv6 LocalSID */
+  rv = sr_localsid_register_function (vm, 
+                                  srv6_localsid_name,
+                                  keyword_str,
+                                  def_str,
+                                  params_str,
+                                  &sm->srv6_localsid_sample_dpo_type,
+                                  format_srv6_localsid_sample, 
+                                  unformat_srv6_localsid_sample, 
+                                  srv6_localsid_creation_fn, 
+                                  srv6_localsid_removal_fn);
+  if (rv < 0)
+    clib_error_return (0, "SRv6 LocalSID function could not be registered.");
+  else
+    sm->srv6_localsid_behavior_id = rv;
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (srv6_localsid_sample_init);
+
+VLIB_PLUGIN_REGISTER () = {
+  .version = "1.0",
+};
diff --git a/src/plugins/srv6-localsid/srv6_localsid_sample.h b/src/plugins/srv6-localsid/srv6_localsid_sample.h
new file mode 100644
index 00000000..474b5de2
--- /dev/null
+++ b/src/plugins/srv6-localsid/srv6_localsid_sample.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_srv6_localsid_sample_h__
+#define __included_srv6_localsid_sample_h__
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+typedef struct {
+    /* API message ID base */
+    u16 msg_id_base;
+
+    /* convenience */
+    vlib_main_t * vlib_main;
+    vnet_main_t * vnet_main;
+    
+    /* DPO type */
+    dpo_type_t srv6_localsid_sample_dpo_type;
+
+    /* SRv6 LocalSID behavior number */
+    u32 srv6_localsid_behavior_id;
+
+} srv6_localsid_sample_main_t;
+
+/*
+ * This is the memory that will be stored per each localsid
+ * the user instantiates
+ */
+typedef struct {
+	u32 fib_table;	/* Stupid index used as an example.. */
+} srv6_localsid_sample_per_sid_memory_t ;
+
+srv6_localsid_sample_main_t srv6_localsid_sample_main;
+
+format_function_t format_srv6_localsid_sample;
+unformat_function_t unformat_srv6_localsid_sample;
+
+void srv6_localsid_sample_dpo_lock (dpo_id_t * dpo);
+void srv6_localsid_sample_dpo_unlock (dpo_id_t * dpo);
+
+extern vlib_node_registration_t srv6_localsid_sample_node;
+
+#endif /* __included_sample_h__ */
diff --git a/src/plugins/srv6-localsid/srv6_sample_localsid_doc.md b/src/plugins/srv6-localsid/srv6_sample_localsid_doc.md
new file mode 100644
index 00000000..b60ab5da
--- /dev/null
+++ b/src/plugins/srv6-localsid/srv6_sample_localsid_doc.md
@@ -0,0 +1,38 @@
+# SRv6 Sample LocalSID documentation    {#srv6_plugin_doc}
+
+## Disclaimer
+
+This is a memo intended to contain documentation for the sample SRv6 LocalSID behavior plugin
+Everything that is not directly obvious should come here.
+For any feedback on content that should be explained please mailto:pcamaril@cisco.com
+ 
+This plugin refers to Segment Routing. Please read the SR documentation first.
+
+## Introduction
+
+This plugin is an example of how an user can create a new SRv6 LocalSID behavior by using VPP plugins with the appropiate API calls to the existing SR code.
+
+This **example** plugin registers a new localsid behavior, with cli keyword 'new_srv6_localsid' which only takes one parameter, a fib-table. Upon recival of a packet, this plugin will enforce the next IP6 lookup in the specific fib-table specified by the user. (Indeed it will do the lookup in the fib_table n+1 (since for the shake of the example we increment the fib-table.)
+
+Notice that the plugin only 'defines' a new SRv6 LocalSID behavior, but the existing SR code in VNET is the one actually instantiating new LocalSIDs. Notice that there are callback functions such that when you create or remove a LocalSID you can actually setup specific parameters through the functions in this plugin.
+
+## Variables to watch for
+
+* srv6_localsid_name: This variable is the name (used as a unique key) identifying this SR LocalSID plugin.
+* keyword_str: This is the CLI keyword to be used for the plugin. In this example 'new_srv6_localsid'. (i.e. sr localsid address cafe::1 behavior new_srv6_localsid )
+* def_str: This is a definition of this SR behavior. This is printed when you do 'show sr localsid behaviors'.
+* params_str: This is a definition of the parameters of this localsid. This is printed when you do 'show sr localsid behaviors'.
+
+## Functions to watch for
+
+* srv6_localsid_creation_fn: This function will be called every time a new SR LocalSID is instantiated with the behavior defined in this plugin. 
+* srv6_localsid_removal_fn: This function will be called every time a new SR LocalSID is removed with the behavior defined in this plugin. This function tends to be used for freeing up all the memory created in the previous function.
+* format_srv6_localsid_sample: This function prints nicely the parameters of every SR LocalSID using this behavior.
+* unformat_srv6_localsid_sample: This function parses the CLI command when initialising a new SR LocalSID using this behavior. It parses all the parameters and ensures that the parameters are correct.
+* format_srv6_localsid_sample_dpo: This function formats the 'show ip6 fib' message for the SR LocalSIDs created with this plugin behavior.
+
+## Graph node
+
+The current graph node uses the function 'end_srh_processing' to do the Segment Routing Endpoint behavior. Notice that it does not allow the cleanup of a Segment Routing header (as per the SRv6 behavior specs).
+This function is identical to the one found in /src/vnet/sr/sr_localsid.c
+In case that by some other reason you want to do decapsulation, or SRH clean_up you can use the functions 'end_decaps_srh_processing' or 'end_psp_srh_processing' respectively.
diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index 993a6e8b..53e9ca1f 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -22,7 +22,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 #include 
 #include 
@@ -3916,9 +3915,11 @@ _(sw_interface_ip6nd_ra_prefix_reply)                   \
 _(sw_interface_ip6nd_ra_config_reply)                   \
 _(set_arp_neighbor_limit_reply)                         \
 _(l2_patch_add_del_reply)                               \
-_(sr_tunnel_add_del_reply)                              \
-_(sr_policy_add_del_reply)                              \
-_(sr_multicast_map_add_del_reply)                       \
+_(sr_policy_add_reply)                                  \
+_(sr_policy_mod_reply)                                  \
+_(sr_policy_del_reply)                                  \
+_(sr_localsid_add_del_reply)                            \
+_(sr_steering_add_del_reply)                            \
 _(classify_add_del_session_reply)                       \
 _(classify_set_interface_ip_table_reply)                \
 _(classify_set_interface_l2_tables_reply)               \
@@ -4095,9 +4096,11 @@ _(SW_INTERFACE_IP6ND_RA_CONFIG_REPLY,                                   \
   sw_interface_ip6nd_ra_config_reply)                                   \
 _(SET_ARP_NEIGHBOR_LIMIT_REPLY, set_arp_neighbor_limit_reply)           \
 _(L2_PATCH_ADD_DEL_REPLY, l2_patch_add_del_reply)                       \
-_(SR_TUNNEL_ADD_DEL_REPLY, sr_tunnel_add_del_reply)                     \
-_(SR_POLICY_ADD_DEL_REPLY, sr_policy_add_del_reply)                     \
-_(SR_MULTICAST_MAP_ADD_DEL_REPLY, sr_multicast_map_add_del_reply)                     \
+_(SR_POLICY_ADD_REPLY, sr_policy_add_reply)                             \
+_(SR_POLICY_MOD_REPLY, sr_policy_mod_reply)                             \
+_(SR_POLICY_DEL_REPLY, sr_policy_del_reply)                             \
+_(SR_LOCALSID_ADD_DEL_REPLY, sr_localsid_add_del_reply)                 \
+_(SR_STEERING_ADD_DEL_REPLY, sr_steering_add_del_reply)                 \
 _(CLASSIFY_ADD_DEL_TABLE_REPLY, classify_add_del_table_reply)           \
 _(CLASSIFY_ADD_DEL_SESSION_REPLY, classify_add_del_session_reply)       \
 _(CLASSIFY_SET_INTERFACE_IP_TABLE_REPLY,                                \
@@ -8227,6 +8230,64 @@ api_l2_patch_add_del (vat_main_t * vam)
   return ret;
 }
 
+u8 is_del;
+u8 localsid_addr[16];
+u8 end_psp;
+u8 behavior;
+u32 sw_if_index;
+u32 vlan_index;
+u32 fib_table;
+u8 nh_addr[16];
+
+static int
+api_sr_localsid_add_del (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_sr_localsid_add_del_t *mp;
+
+  u8 is_del;
+  ip6_address_t localsid;
+  u8 end_psp = 0;
+  u8 behavior = ~0;
+  u32 sw_if_index;
+  u32 fib_table = ~(u32) 0;
+  ip6_address_t next_hop;
+
+  bool nexthop_set = 0;
+
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "del"))
+	is_del = 1;
+      else if (unformat (i, "address %U", unformat_ip6_address, &localsid));
+      else if (unformat (i, "next-hop %U", unformat_ip6_address, &next_hop))
+	nexthop_set = 1;
+      else if (unformat (i, "behavior %u", &behavior));
+      else if (unformat (i, "sw_if_index %u", &sw_if_index));
+      else if (unformat (i, "fib-table %u", &fib_table));
+      else if (unformat (i, "end.psp %u", &behavior));
+      else
+	break;
+    }
+
+  M (SR_LOCALSID_ADD_DEL, mp);
+
+  clib_memcpy (mp->localsid_addr, &localsid, sizeof (mp->localsid_addr));
+  if (nexthop_set)
+    clib_memcpy (mp->nh_addr, &next_hop, sizeof (mp->nh_addr));
+  mp->behavior = behavior;
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->fib_table = ntohl (fib_table);
+  mp->end_psp = end_psp;
+  mp->is_del = is_del;
+
+  S (mp);
+  W (ret);
+  return ret;
+}
+
 static int
 api_ioam_enable (vat_main_t * vam)
 {
@@ -8277,277 +8338,6 @@ api_ioam_disable (vat_main_t * vam)
   return ret;
 }
 
-static int
-api_sr_tunnel_add_del (vat_main_t * vam)
-{
-  unformat_input_t *i = vam->input;
-  vl_api_sr_tunnel_add_del_t *mp;
-  int is_del = 0;
-  int pl_index;
-  ip6_address_t src_address;
-  int src_address_set = 0;
-  ip6_address_t dst_address;
-  u32 dst_mask_width;
-  int dst_address_set = 0;
-  u16 flags = 0;
-  u32 rx_table_id = 0;
-  u32 tx_table_id = 0;
-  ip6_address_t *segments = 0;
-  ip6_address_t *this_seg;
-  ip6_address_t *tags = 0;
-  ip6_address_t *this_tag;
-  ip6_address_t next_address, tag;
-  u8 *name = 0;
-  u8 *policy_name = 0;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "del"))
-	is_del = 1;
-      else if (unformat (i, "name %s", &name))
-	;
-      else if (unformat (i, "policy %s", &policy_name))
-	;
-      else if (unformat (i, "rx_fib_id %d", &rx_table_id))
-	;
-      else if (unformat (i, "tx_fib_id %d", &tx_table_id))
-	;
-      else if (unformat (i, "src %U", unformat_ip6_address, &src_address))
-	src_address_set = 1;
-      else if (unformat (i, "dst %U/%d",
-			 unformat_ip6_address, &dst_address, &dst_mask_width))
-	dst_address_set = 1;
-      else if (unformat (i, "next %U", unformat_ip6_address, &next_address))
-	{
-	  vec_add2 (segments, this_seg, 1);
-	  clib_memcpy (this_seg->as_u8, next_address.as_u8,
-		       sizeof (*this_seg));
-	}
-      else if (unformat (i, "tag %U", unformat_ip6_address, &tag))
-	{
-	  vec_add2 (tags, this_tag, 1);
-	  clib_memcpy (this_tag->as_u8, tag.as_u8, sizeof (*this_tag));
-	}
-      else if (unformat (i, "clean"))
-	flags |= IP6_SR_HEADER_FLAG_CLEANUP;
-      else if (unformat (i, "protected"))
-	flags |= IP6_SR_HEADER_FLAG_PROTECTED;
-      else if (unformat (i, "InPE %d", &pl_index))
-	{
-	  if (pl_index <= 0 || pl_index > 4)
-	    {
-	    pl_index_range_error:
-	      errmsg ("pl index %d out of range", pl_index);
-	      return -99;
-	    }
-	  flags |=
-	    IP6_SR_HEADER_FLAG_PL_ELT_INGRESS_PE << (3 * (pl_index - 1));
-	}
-      else if (unformat (i, "EgPE %d", &pl_index))
-	{
-	  if (pl_index <= 0 || pl_index > 4)
-	    goto pl_index_range_error;
-	  flags |=
-	    IP6_SR_HEADER_FLAG_PL_ELT_EGRESS_PE << (3 * (pl_index - 1));
-	}
-      else if (unformat (i, "OrgSrc %d", &pl_index))
-	{
-	  if (pl_index <= 0 || pl_index > 4)
-	    goto pl_index_range_error;
-	  flags |=
-	    IP6_SR_HEADER_FLAG_PL_ELT_ORIG_SRC_ADDR << (3 * (pl_index - 1));
-	}
-      else
-	break;
-    }
-
-  if (!src_address_set)
-    {
-      errmsg ("src address required");
-      return -99;
-    }
-
-  if (!dst_address_set)
-    {
-      errmsg ("dst address required");
-      return -99;
-    }
-
-  if (!segments)
-    {
-      errmsg ("at least one sr segment required");
-      return -99;
-    }
-
-  M2 (SR_TUNNEL_ADD_DEL, mp,
-      vec_len (segments) * sizeof (ip6_address_t)
-      + vec_len (tags) * sizeof (ip6_address_t));
-
-  clib_memcpy (mp->src_address, &src_address, sizeof (mp->src_address));
-  clib_memcpy (mp->dst_address, &dst_address, sizeof (mp->dst_address));
-  mp->dst_mask_width = dst_mask_width;
-  mp->flags_net_byte_order = clib_host_to_net_u16 (flags);
-  mp->n_segments = vec_len (segments);
-  mp->n_tags = vec_len (tags);
-  mp->is_add = is_del == 0;
-  clib_memcpy (mp->segs_and_tags, segments,
-	       vec_len (segments) * sizeof (ip6_address_t));
-  clib_memcpy (mp->segs_and_tags +
-	       vec_len (segments) * sizeof (ip6_address_t), tags,
-	       vec_len (tags) * sizeof (ip6_address_t));
-
-  mp->outer_vrf_id = ntohl (rx_table_id);
-  mp->inner_vrf_id = ntohl (tx_table_id);
-  memcpy (mp->name, name, vec_len (name));
-  memcpy (mp->policy_name, policy_name, vec_len (policy_name));
-
-  vec_free (segments);
-  vec_free (tags);
-
-  S (mp);
-  W (ret);
-  return ret;
-}
-
-static int
-api_sr_policy_add_del (vat_main_t * vam)
-{
-  unformat_input_t *input = vam->input;
-  vl_api_sr_policy_add_del_t *mp;
-  int is_del = 0;
-  u8 *name = 0;
-  u8 *tunnel_name = 0;
-  u8 **tunnel_names = 0;
-
-  int name_set = 0;
-  int tunnel_set = 0;
-  int j = 0;
-  int tunnel_names_length = 1;	// Init to 1 to offset the #tunnel_names counter byte
-  int tun_name_len = 0;		// Different naming convention used as confusing these would be "bad" (TM)
-  int ret;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "del"))
-	is_del = 1;
-      else if (unformat (input, "name %s", &name))
-	name_set = 1;
-      else if (unformat (input, "tunnel %s", &tunnel_name))
-	{
-	  if (tunnel_name)
-	    {
-	      vec_add1 (tunnel_names, tunnel_name);
-	      /* For serializer:
-	         - length = #bytes to store in serial vector
-	         - +1 = byte to store that length
-	       */
-	      tunnel_names_length += (vec_len (tunnel_name) + 1);
-	      tunnel_set = 1;
-	      tunnel_name = 0;
-	    }
-	}
-      else
-	break;
-    }
-
-  if (!name_set)
-    {
-      errmsg ("policy name required");
-      return -99;
-    }
-
-  if ((!tunnel_set) && (!is_del))
-    {
-      errmsg ("tunnel name required");
-      return -99;
-    }
-
-  M2 (SR_POLICY_ADD_DEL, mp, tunnel_names_length);
-
-
-
-  mp->is_add = !is_del;
-
-  memcpy (mp->name, name, vec_len (name));
-  // Since mp->tunnel_names is of type u8[0] and not a u8 *, u8 ** needs to be serialized
-  u8 *serial_orig = 0;
-  vec_validate (serial_orig, tunnel_names_length);
-  *serial_orig = vec_len (tunnel_names);	// Store the number of tunnels as length in first byte of serialized vector
-  serial_orig += 1;		// Move along one byte to store the length of first tunnel_name
-
-  for (j = 0; j < vec_len (tunnel_names); j++)
-    {
-      tun_name_len = vec_len (tunnel_names[j]);
-      *serial_orig = tun_name_len;	// Store length of tunnel name in first byte of Length/Value pair
-      serial_orig += 1;		// Move along one byte to store the actual tunnel name
-      memcpy (serial_orig, tunnel_names[j], tun_name_len);
-      serial_orig += tun_name_len;	// Advance past the copy
-    }
-  memcpy (mp->tunnel_names, serial_orig - tunnel_names_length, tunnel_names_length);	// Regress serial_orig to head then copy fwd
-
-  vec_free (tunnel_names);
-  vec_free (tunnel_name);
-
-  S (mp);
-  W (ret);
-  return ret;
-}
-
-static int
-api_sr_multicast_map_add_del (vat_main_t * vam)
-{
-  unformat_input_t *input = vam->input;
-  vl_api_sr_multicast_map_add_del_t *mp;
-  int is_del = 0;
-  ip6_address_t multicast_address;
-  u8 *policy_name = 0;
-  int multicast_address_set = 0;
-  int ret;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "del"))
-	is_del = 1;
-      else
-	if (unformat
-	    (input, "address %U", unformat_ip6_address, &multicast_address))
-	multicast_address_set = 1;
-      else if (unformat (input, "sr-policy %s", &policy_name))
-	;
-      else
-	break;
-    }
-
-  if (!is_del && !policy_name)
-    {
-      errmsg ("sr-policy name required");
-      return -99;
-    }
-
-
-  if (!multicast_address_set)
-    {
-      errmsg ("address required");
-      return -99;
-    }
-
-  M (SR_MULTICAST_MAP_ADD_DEL, mp);
-
-  mp->is_add = !is_del;
-  memcpy (mp->policy_name, policy_name, vec_len (policy_name));
-  clib_memcpy (mp->multicast_address, &multicast_address,
-	       sizeof (mp->multicast_address));
-
-
-  vec_free (policy_name);
-
-  S (mp);
-  W (ret);
-  return ret;
-}
-
-
 #define foreach_tcp_proto_field                 \
 _(src_port)                                     \
 _(dst_port)
@@ -18166,14 +17956,9 @@ _(set_arp_neighbor_limit, "arp_nbr_limit  [ipv6]")                   \
 _(l2_patch_add_del,                                                     \
   "rx  | rx_sw_if_index  tx  | tx_sw_if_index \n" \
   "enable | disable")                                                   \
-_(sr_tunnel_add_del,                                                    \
-  "[name ] src  dst / \n"                 \
-  "(next )+ [tag ]* [clean] [reroute] \n"           \
-  "[policy ]")						\
-_(sr_policy_add_del,                                                    \
-  "name  tunnel  [tunnel ]* [del]")	\
-_(sr_multicast_map_add_del,                                             \
-  "address [ip6 multicast address] sr-policy [policy name] [del]")	\
+_(sr_localsid_add_del,                                                  \
+  "(del) address  next_hop  behavior \n"               \
+  "fib-table  (end.psp) sw_if_index ")                        \
 _(classify_add_del_table,                                               \
   "buckets  [skip ] [match ] [memory_size ]\n"	\
   " [del] [del-chain] mask \n"                              \
diff --git a/src/vnet.am b/src/vnet.am
index d89d516e..7125a122 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -710,13 +710,15 @@ nobase_include_HEADERS +=			\
 
 if WITH_LIBSSL
 libvnet_la_SOURCES +=				\
- vnet/sr/sr.c					\
+ vnet/sr/sr.c						\
+ vnet/sr/sr_localsid.c				\
+ vnet/sr/sr_policy_rewrite.c		\
+ vnet/sr/sr_steering.c				\
  vnet/sr/sr_api.c
 endif
 
 nobase_include_HEADERS +=			\
  vnet/sr/sr_packet.h				\
- vnet/sr/sr_error.def				\
  vnet/sr/sr.h						\
  vnet/sr/sr.api.h
 
diff --git a/src/vnet/ip/ip6_packet.h b/src/vnet/ip/ip6_packet.h
index 4fd14b96..6eabeef1 100644
--- a/src/vnet/ip/ip6_packet.h
+++ b/src/vnet/ip/ip6_packet.h
@@ -448,20 +448,47 @@ always_inline u8 ip6_ext_hdr(u8 nexthdr)
    * find out if nexthdr is an extension header or a protocol
    */
   return   (nexthdr == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) ||
-    (nexthdr == IP_PROTOCOL_IP6_NONXT) ||
     (nexthdr == IP_PROTOCOL_IPV6_FRAGMENTATION)  ||
     (nexthdr == IP_PROTOCOL_IPSEC_AH)      ||
     (nexthdr == IP_PROTOCOL_IPV6_ROUTE)      ||
     (nexthdr == IP_PROTOCOL_IP6_DESTINATION_OPTIONS);
 }
 
-#define ip6_ext_header_len(p)  (((p)->n_data_u64s+1) << 3)
-#define ip6_ext_authhdr_len(p) (((p)->n_data_u64s+2) << 2)
+#define ip6_ext_header_len(p)  ((((ip6_ext_header_t *)(p))->n_data_u64s+1) << 3)
+#define ip6_ext_authhdr_len(p) ((((ip6_ext_header_t *)(p))->n_data_u64s+2) << 2)
 
 always_inline void *
 ip6_ext_next_header (ip6_ext_header_t *ext_hdr )
 { return (void *)((u8 *) ext_hdr + ip6_ext_header_len(ext_hdr)); }
 
+/*
+ * Macro to find the IPv6 ext header of type t
+ * I is the IPv6 header
+ * P is the previous IPv6 ext header (NULL if none)
+ * M is the matched IPv6 ext header of type t
+ */
+#define ip6_ext_header_find_t(i, p, m, t)               \
+if ((i)->protocol == t)                                 \
+{                                                       \
+  (m) = (void *)((i)+1);                                \
+  (p) = NULL;                                           \
+}                                                       \
+else                                                    \
+{                                                       \
+  (m) = NULL;                                           \
+  (p) = (void *)((i)+1);                                \
+  while (ip6_ext_hdr((p)->next_hdr) &&                  \
+    ((ip6_ext_header_t *)(p))->next_hdr != (t))         \
+  {                                                     \
+    (p) = ip6_ext_next_header((p));                     \
+  }                                                     \
+  if ( ip6_ext_hdr((p)->next_hdr) == (t))               \
+  {                                                     \
+    (m) = (void *)(ip6_ext_next_header((p)));           \
+  }                                                     \
+}
+
+
 typedef CLIB_PACKED (struct {
   u8 next_hdr;
   /* Length of this header plus option data in 8 byte units. */
diff --git a/src/vnet/sr/dir.dox b/src/vnet/sr/dir.dox
old mode 100644
new mode 100755
index a98b202c..3f539a58
--- a/src/vnet/sr/dir.dox
+++ b/src/vnet/sr/dir.dox
@@ -18,8 +18,8 @@
  @brief Segment Routing code
 
  An implementation of Segment Routing as per:
- draft-previdi-6man-segment-routing-header-05
+ draft-ietf-6man-segment-routing-header-05
 
- See file: rfc_draft_05.txt 
+ @see ietf_draft_05.txt 
  
 */
\ No newline at end of file
diff --git a/src/vnet/sr/examples/sr_multicastmap.script b/src/vnet/sr/examples/sr_multicastmap.script
deleted file mode 100644
index 20bf7dc0..00000000
--- a/src/vnet/sr/examples/sr_multicastmap.script
+++ /dev/null
@@ -1,4 +0,0 @@
-sr_tunnel_add_del name sr2 src ::a:1:1:0:6 dst ff15::2/128 next ::a:1:1:0:f next ::a:1:1:0:1a next ff15::1 tag ::a:1:1:0:7  clean 
-sr_tunnel_add_del name sr3 src ::b:1:1:0:6 dst ff16::2/128 next ::a:1:1:0:13 next ::a:1:1:0:1a next ff15::1 tag ::a:1:1:0:7  clean
-sr_policy_add_del name pol1 tunnel sr2 tunnel sr3
-sr_multicast_map_add_del address ff15::1 sr-policy pol1
diff --git a/src/vnet/sr/ietf_draft_05.txt b/src/vnet/sr/ietf_draft_05.txt
new file mode 100755
index 00000000..e9bff04f
--- /dev/null
+++ b/src/vnet/sr/ietf_draft_05.txt
@@ -0,0 +1,1564 @@
+Network Working Group                                    S. Previdi, Ed.
+Internet-Draft                                               C. Filsfils
+Intended status: Standards Track                     Cisco Systems, Inc.
+Expires: August 5, 2017                                         B. Field
+                                                                 Comcast
+                                                                I. Leung
+                                                   Rogers Communications
+                                                              J. Linkova
+                                                                  Google
+                                                                E. Aries
+                                                                Facebook
+                                                               T. Kosugi
+                                                                     NTT
+                                                               E. Vyncke
+                                                     Cisco Systems, Inc.
+                                                               D. Lebrun
+                                        Universite Catholique de Louvain
+                                                        February 1, 2017
+
+
+                   IPv6 Segment Routing Header (SRH)
+               draft-ietf-6man-segment-routing-header-05
+
+Abstract
+
+   Segment Routing (SR) allows a node to steer a packet through a
+   controlled set of instructions, called segments, by prepending an SR
+   header to the packet.  A segment can represent any instruction,
+   topological or service-based.  SR allows to enforce a flow through
+   any path (topological, or application/service based) while
+   maintaining per-flow state only at the ingress node to the SR domain.
+
+   Segment Routing can be applied to the IPv6 data plane with the
+   addition of a new type of Routing Extension Header.  This draft
+   describes the Segment Routing Extension Header Type and how it is
+   used by SR capable nodes.
+
+Requirements Language
+
+   The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
+   "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
+   document are to be interpreted as described in RFC 2119 [RFC2119].
+
+Status of This Memo
+
+   This Internet-Draft is submitted in full conformance with the
+   provisions of BCP 78 and BCP 79.
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 1]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   Internet-Drafts are working documents of the Internet Engineering
+   Task Force (IETF).  Note that other groups may also distribute
+   working documents as Internet-Drafts.  The list of current Internet-
+   Drafts is at http://datatracker.ietf.org/drafts/current/.
+
+   Internet-Drafts are draft documents valid for a maximum of six months
+   and may be updated, replaced, or obsoleted by other documents at any
+   time.  It is inappropriate to use Internet-Drafts as reference
+   material or to cite them other than as "work in progress."
+
+   This Internet-Draft will expire on August 5, 2017.
+
+Copyright Notice
+
+   Copyright (c) 2017 IETF Trust and the persons identified as the
+   document authors.  All rights reserved.
+
+   This document is subject to BCP 78 and the IETF Trust's Legal
+   Provisions Relating to IETF Documents
+   (http://trustee.ietf.org/license-info) in effect on the date of
+   publication of this document.  Please review these documents
+   carefully, as they describe your rights and restrictions with respect
+   to this document.  Code Components extracted from this document must
+   include Simplified BSD License text as described in Section 4.e of
+   the Trust Legal Provisions and are provided without warranty as
+   described in the Simplified BSD License.
+
+Table of Contents
+
+   1.  Segment Routing Documents . . . . . . . . . . . . . . . . . .   3
+   2.  Introduction  . . . . . . . . . . . . . . . . . . . . . . . .   3
+     2.1.  Data Planes supporting Segment Routing  . . . . . . . . .   4
+     2.2.  Segment Routing (SR) Domain . . . . . . . . . . . . . . .   4
+       2.2.1.  SR Domain in a Service Provider Network . . . . . . .   5
+       2.2.2.  SR Domain in a Overlay Network  . . . . . . . . . . .   6
+   3.  Segment Routing Extension Header (SRH)  . . . . . . . . . . .   7
+     3.1.  SRH TLVs  . . . . . . . . . . . . . . . . . . . . . . . .   9
+       3.1.1.  Ingress Node TLV  . . . . . . . . . . . . . . . . . .  10
+       3.1.2.  Egress Node TLV . . . . . . . . . . . . . . . . . . .  11
+       3.1.3.  Opaque Container TLV  . . . . . . . . . . . . . . . .  11
+       3.1.4.  Padding TLV . . . . . . . . . . . . . . . . . . . . .  12
+       3.1.5.  HMAC TLV  . . . . . . . . . . . . . . . . . . . . . .  13
+     3.2.  SRH and RFC2460 behavior  . . . . . . . . . . . . . . . .  14
+   4.  SRH Procedures  . . . . . . . . . . . . . . . . . . . . . . .  14
+     4.1.  Source SR Node  . . . . . . . . . . . . . . . . . . . . .  14
+     4.2.  Transit Node  . . . . . . . . . . . . . . . . . . . . . .  15
+     4.3.  SR Segment Endpoint Node  . . . . . . . . . . . . . . . .  16
+   5.  Security Considerations . . . . . . . . . . . . . . . . . . .  16
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 2]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+     5.1.  Threat model  . . . . . . . . . . . . . . . . . . . . . .  17
+       5.1.1.  Source routing threats  . . . . . . . . . . . . . . .  17
+       5.1.2.  Applicability of RFC 5095 to SRH  . . . . . . . . . .  17
+       5.1.3.  Service stealing threat . . . . . . . . . . . . . . .  18
+       5.1.4.  Topology disclosure . . . . . . . . . . . . . . . . .  18
+       5.1.5.  ICMP Generation . . . . . . . . . . . . . . . . . . .  18
+     5.2.  Security fields in SRH  . . . . . . . . . . . . . . . . .  19
+       5.2.1.  Selecting a hash algorithm  . . . . . . . . . . . . .  20
+       5.2.2.  Performance impact of HMAC  . . . . . . . . . . . . .  21
+       5.2.3.  Pre-shared key management . . . . . . . . . . . . . .  21
+     5.3.  Deployment Models . . . . . . . . . . . . . . . . . . . .  22
+       5.3.1.  Nodes within the SR domain  . . . . . . . . . . . . .  22
+       5.3.2.  Nodes outside of the SR domain  . . . . . . . . . . .  22
+       5.3.3.  SR path exposure  . . . . . . . . . . . . . . . . . .  23
+       5.3.4.  Impact of BCP-38  . . . . . . . . . . . . . . . . . .  23
+   6.  IANA Considerations . . . . . . . . . . . . . . . . . . . . .  24
+   7.  Manageability Considerations  . . . . . . . . . . . . . . . .  24
+   8.  Contributors  . . . . . . . . . . . . . . . . . . . . . . . .  24
+   9.  Acknowledgements  . . . . . . . . . . . . . . . . . . . . . .  24
+   10. References  . . . . . . . . . . . . . . . . . . . . . . . . .  25
+     10.1.  Normative References . . . . . . . . . . . . . . . . . .  25
+     10.2.  Informative References . . . . . . . . . . . . . . . . .  25
+   Authors' Addresses  . . . . . . . . . . . . . . . . . . . . . . .  27
+
+1.  Segment Routing Documents
+
+   Segment Routing terminology is defined in
+   [I-D.ietf-spring-segment-routing].
+
+   Segment Routing use cases are described in [RFC7855] and
+   [I-D.ietf-spring-ipv6-use-cases].
+
+   Segment Routing protocol extensions are defined in
+   [I-D.ietf-isis-segment-routing-extensions], and
+   [I-D.ietf-ospf-ospfv3-segment-routing-extensions].
+
+2.  Introduction
+
+   Segment Routing (SR), defined in [I-D.ietf-spring-segment-routing],
+   allows a node to steer a packet through a controlled set of
+   instructions, called segments, by prepending an SR header to the
+   packet.  A segment can represent any instruction, topological or
+   service-based.  SR allows to enforce a flow through any path
+   (topological or service/application based) while maintaining per-flow
+   state only at the ingress node to the SR domain.  Segments can be
+   derived from different components: IGP, BGP, Services, Contexts,
+   Locators, etc.  The list of segment forming the path is called the
+   Segment List and is encoded in the packet header.
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 3]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   SR allows the use of strict and loose source based routing paradigms
+   without requiring any additional signaling protocols in the
+   infrastructure hence delivering an excellent scalability property.
+
+   The source based routing model described in
+   [I-D.ietf-spring-segment-routing] is inherited from the ones proposed
+   by [RFC1940] and [RFC2460].  The source based routing model offers
+   the support for explicit routing capability.
+
+2.1.  Data Planes supporting Segment Routing
+
+   Segment Routing (SR), can be instantiated over MPLS
+   ([I-D.ietf-spring-segment-routing-mpls]) and IPv6.  This document
+   defines its instantiation over the IPv6 data-plane based on the use-
+   cases defined in [I-D.ietf-spring-ipv6-use-cases].
+
+   This document defines a new type of Routing Header (originally
+   defined in [RFC2460]) called the Segment Routing Header (SRH) in
+   order to convey the Segment List in the packet header as defined in
+   [I-D.ietf-spring-segment-routing].  Mechanisms through which segment
+   are known and advertised are outside the scope of this document.
+
+   A segment is materialized by an IPv6 address.  A segment identifies a
+   topological instruction or a service instruction.  A segment can be
+   either:
+
+   o  global: a global segment represents an instruction supported by
+      all nodes in the SR domain and it is instantiated through an IPv6
+      address globally known in the SR domain.
+
+   o  local: a local segment represents an instruction supported only by
+      the node who originates it and it is instantiated through an IPv6
+      address that is known only by the local node.
+
+2.2.  Segment Routing (SR) Domain
+
+   We define the concept of the Segment Routing Domain (SR Domain) as
+   the set of nodes participating into the source based routing model.
+   These nodes may be connected to the same physical infrastructure
+   (e.g.: a Service Provider's network) as well as nodes remotely
+   connected to each other (e.g.: an enterprise VPN or an overlay).
+
+   A non-exhaustive list of examples of SR Domains is:
+
+   o  The network of an operator, service provider, content provider,
+      enterprise including nodes, links and Autonomous Systems.
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 4]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   o  A set of nodes connected as an overlay over one or more transit
+      providers.  The overlay nodes exchange SR-enabled traffic with
+      segments belonging solely to the overlay routers (the SR domain).
+      None of the segments in the SR-enabled packets exchanged by the
+      overlay belong to the transit networks
+
+   The source based routing model through its instantiation of the
+   Segment Routing Header (SRH) defined in this document equally applies
+   to all the above examples.
+
+   It is assumed in this document that the SRH is added to the packet by
+   its source, consistently with the source routing model defined in
+   [RFC2460].  For example:
+
+   o  At the node originating the packet (host, server).
+
+   o  At the ingress node of an SR domain where the ingress node
+      receives an IPv6 packet and encapsulates it into an outer IPv6
+      header followed by a Segment Routing header.
+
+2.2.1.  SR Domain in a Service Provider Network
+
+   The following figure illustrates an SR domain consisting of an
+   operator's network infrastructure.
+
+     (-------------------------- Operator 1 -----------------------)
+     (                                                             )
+     (  (-----AS 1-----)  (-------AS 2-------)  (----AS 3-------)  )
+     (  (              )  (                  )  (               )  )
+ A1--(--(--11---13--14-)--(-21---22---23--24-)--(-31---32---34--)--)--Z1
+     (  ( /|\  /|\  /| )  ( |\  /|\  /|\  /| )  ( |\  /|\  /| \ )  )
+ A2--(--(/ | \/ | \/ | )  ( | \/ | \/ | \/ | )  ( | \/ | \/ |  \)--)--Z2
+     (  (  | /\ | /\ | )  ( | /\ | /\ | /\ | )  ( | /\ | /\ |   )  )
+     (  (  |/  \|/  \| )  ( |/  \|/  \|/  \| )  ( |/  \|/  \|   )  )
+ A3--(--(--15---17--18-)--(-25---26---27--28-)--(-35---36---38--)--)--Z3
+     (  (              )  (                  )  (               )  )
+     (  (--------------)  (------------------)  (---------------)  )
+     (                                                             )
+     (-------------------------------------------------------------)
+
+                   Figure 1: Service Provider SR Domain
+
+   Figure 1 describes an operator network including several ASes and
+   delivering connectivity between endpoints.  In this scenario, Segment
+   Routing is used within the operator networks and across the ASes
+   boundaries (all being under the control of the same operator).  In
+   this case segment routing can be used in order to address use cases
+   such as end-to-end traffic engineering, fast re-route, egress peer
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 5]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   engineering, data-center traffic engineering as described in
+   [RFC7855], [I-D.ietf-spring-ipv6-use-cases] and
+   [I-D.ietf-spring-resiliency-use-cases].
+
+   Typically, an IPv6 packet received at ingress (i.e.: from outside the
+   SR domain), is classified according to network operator policies and
+   such classification results into an outer header with an SRH applied
+   to the incoming packet.  The SRH contains the list of segment
+   representing the path the packet must take inside the SR domain.
+   Thus, the SA of the packet is the ingress node, the DA (due to SRH
+   procedures described in Section 4) is set as the first segment of the
+   path and the last segment of the path is the egress node of the SR
+   domain.
+
+   The path may include intra-AS as well as inter-AS segments.  It has
+   to be noted that all nodes within the SR domain are under control of
+   the same administration.  When the packet reaches the egress point of
+   the SR domain, the outer header and its SRH are removed so that the
+   destination of the packet is unaware of the SR domain the packet has
+   traversed.
+
+   The outer header with the SRH is no different from any other
+   tunneling encapsulation mechanism and allows a network operator to
+   implement traffic engineering mechanisms so to efficiently steer
+   traffic across his infrastructure.
+
+2.2.2.  SR Domain in a Overlay Network
+
+   The following figure illustrates an SR domain consisting of an
+   overlay network over multiple operator's networks.
+
+       (--Operator 1---)  (-----Operator 2-----)  (--Operator 3---)
+       (               )  (                    )  (               )
+   A1--(--11---13--14--)--(--21---22---23--24--)--(-31---32---34--)--C1
+       ( /|\  /|\  /|  )  (  |\  /|\  /|\  /|  )  ( |\  /|\  /| \ )
+   A2--(/ | \/ | \/ |  )  (  | \/ | \/ | \/ |  )  ( | \/ | \/ |  \)--C2
+       (  | /\ | /\ |  )  (  | /\ | /\ | /\ |  )  ( | /\ | /\ |   )
+       (  |/  \|/  \|  )  (  |/  \|/  \|/  \|  )  ( |/  \|/  \|   )
+   A3--(--15---17--18--)--(--25---26---27--28--)--(-35---36---38--)--C3
+       (               )  (  |    |         |  )  (               )
+       (---------------)  (--|----|---------|--)  (---------------)
+                             |    |         |
+                             B1   B2        B3
+
+                        Figure 2: Overlay SR Domain
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 6]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   Figure 2 describes an overlay consisting of nodes connected to three
+   different network operators and forming a single overlay network
+   where Segment routing packets are exchanged.
+
+   The overlay consists of nodes A1, A2, A3, B1, B2, B3, C1, C2 and C3.
+   These nodes are connected to their respective network operator and
+   form an overlay network.
+
+   Each node may originate packets with an SRH which contains, in the
+   segment list of the SRH or in the DA, segments identifying other
+   overlay nodes.  This implies that packets with an SRH may traverse
+   operator's networks but, obviously, these SRHs cannot contain an
+   address/segment of the transit operators 1, 2 and 3.  The SRH
+   originated by the overlay can only contain address/segment under the
+   administration of the overlay (e.g. address/segments supported by A1,
+   A2, A3, B1, B2, B3, C1,C2 or C3).
+
+   In this model, the operator network nodes are transit nodes and,
+   according to [RFC2460], MUST NOT inspect the routing extension header
+   since they are not the DA of the packet.
+
+   It is a common practice in operators networks to filter out, at
+   ingress, any packet whose DA is the address of an internal node and
+   it is also possible that an operator would filter out any packet
+   destined to an internal address and having an extension header in it.
+
+   This common practice does not impact the SR-enabled traffic between
+   the overlay nodes as the intermediate transit networks never see a
+   destination address belonging to their infrastructure.  These SR-
+   enabled overlay packets will thus never be filtered by the transit
+   operators.
+
+   In all cases, transit packets (i.e.: packets whose DA is outside the
+   domain of the operator's network) will be forwarded accordingly
+   without introducing any security concern in the operator's network.
+   This is similar to tunneled packets.
+
+3.  Segment Routing Extension Header (SRH)
+
+   A new type of the Routing Header (originally defined in [RFC2460]) is
+   defined: the Segment Routing Header (SRH) which has a new Routing
+   Type, (suggested value 4) to be assigned by IANA.
+
+   The Segment Routing Header (SRH) is defined as follows:
+
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 7]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    | Next Header   |  Hdr Ext Len  | Routing Type  | Segments Left |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    | First Segment |     Flags     |           RESERVED            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |            Segment List[0] (128 bits IPv6 address)            |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                                                               |
+                                  ...
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |            Segment List[n] (128 bits IPv6 address)            |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    //                                                             //
+    //         Optional Type Length Value objects (variable)       //
+    //                                                             //
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+   where:
+
+   o  Next Header: 8-bit selector.  Identifies the type of header
+      immediately following the SRH.
+
+   o  Hdr Ext Len: 8-bit unsigned integer, is the length of the SRH
+      header in 8-octet units, not including the first 8 octets.
+
+   o  Routing Type: TBD, to be assigned by IANA (suggested value: 4).
+
+   o  Segments Left.  Defined in [RFC2460], it contains the index, in
+      the Segment List, of the next segment to inspect.  Segments Left
+      is decremented at each segment.
+
+   o  First Segment: contains the index, in the Segment List, of the
+      first segment of the path which is in fact the last element of the
+      Segment List.
+
+   o  Flags: 8 bits of flags.  Following flags are defined:
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 8]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+          0 1 2 3 4 5 6 7
+         +-+-+-+-+-+-+-+-+
+         |U|P|O|A|H|  U  |
+         +-+-+-+-+-+-+-+-+
+
+         U: Unused and for future use.  SHOULD be unset on transmission
+         and MUST be ignored on receipt.
+
+         P-flag: Protected flag.  Set when the packet has been rerouted
+         through FRR mechanism by an SR endpoint node.
+
+         O-flag: OAM flag.  When set, it indicates that this packet is
+         an operations and management (OAM) packet.
+
+         A-flag: Alert flag.  If present, it means important Type Length
+         Value (TLV) objects are present.  See Section 3.1 for details
+         on TLVs objects.
+
+         H-flag: HMAC flag.  If set, the HMAC TLV is present and is
+         encoded as the last TLV of the SRH.  In other words, the last
+         36 octets of the SRH represent the HMAC information.  See
+         Section 3.1.5 for details on the HMAC TLV.
+
+   o  RESERVED: SHOULD be unset on transmission and MUST be ignored on
+      receipt.
+
+   o  Segment List[n]: 128 bit IPv6 addresses representing the nth
+      segment in the Segment List.  The Segment List is encoded starting
+      from the last segment of the path.  I.e., the first element of the
+      segment list (Segment List [0]) contains the last segment of the
+      path while the last segment of the Segment List (Segment List[n])
+      contains the first segment of the path.  The index contained in
+      "Segments Left" identifies the current active segment.
+
+   o  Type Length Value (TLV) are described in Section 3.1.
+
+3.1.  SRH TLVs
+
+   This section defines TLVs of the Segment Routing Header.
+
+   Type Length Value (TLV) contain optional information that may be used
+   by the node identified in the DA of the packet.  It has to be noted
+   that the information carried in the TLVs is not intended to be used
+   by the routing layer.  Typically, TLVs carry information that is
+   consumed by other components (e.g.: OAM) than the routing function.
+
+   Each TLV has its own length, format and semantic.  The code-point
+   allocated (by IANA) to each TLV defines both the format and the
+
+
+
+Previdi, et al.          Expires August 5, 2017                 [Page 9]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   semantic of the information carried in the TLV.  Multiple TLVs may be
+   encoded in the same SRH.
+
+   The "Length" field of the TLV is primarily used to skip the TLV while
+   inspecting the SRH in case the node doesn't support or recognize the
+   TLV codepoint.  The "Length" defines the TLV length in octets and not
+   including the "Type" and "Length" fields.
+
+   The primary scope of TLVs is to give the receiver of the packet
+   information related to the source routed path (e.g.: where the packet
+   entered in the SR domain and where it is expected to exit).
+
+   Additional TLVs may be defined in the future.
+
+3.1.1.  Ingress Node TLV
+
+   The Ingress Node TLV is optional and identifies the node this packet
+   traversed when entered the SR domain.  The Ingress Node TLV has
+   following format:
+
+    0                   1                   2                   3
+    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |      Type     |    Length     |   RESERVED    |     Flags     |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |                                                               |
+   |                 Ingress Node (16 octets)                      |
+   |                                                               |
+   |                                                               |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+   where:
+
+   o  Type: to be assigned by IANA (suggested value 1).
+
+   o  Length: 18.
+
+   o  RESERVED: 8 bits.  SHOULD be unset on transmission and MUST be
+      ignored on receipt.
+
+   o  Flags: 8 bits.  No flags are defined in this document.
+
+   o  Ingress Node: 128 bits.  Defines the node where the packet is
+      expected to enter the SR domain.  In the encapsulation case
+      described in Section 2.2.1, this information corresponds to the SA
+      of the encapsulating header.
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 10]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+3.1.2.  Egress Node TLV
+
+   The Egress Node TLV is optional and identifies the node this packet
+   is expected to traverse when exiting the SR domain.  The Egress Node
+   TLV has following format:
+
+    0                   1                   2                   3
+    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |      Type     |    Length     |   RESERVED    |     Flags     |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |                                                               |
+   |                  Egress Node (16 octets)                      |
+   |                                                               |
+   |                                                               |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+   where:
+
+   o  Type: to be assigned by IANA (suggested value 2).
+
+   o  Length: 18.
+
+   o  RESERVED: 8 bits.  SHOULD be unset on transmission and MUST be
+      ignored on receipt.
+
+   o  Flags: 8 bits.  No flags are defined in this document.
+
+   o  Egress Node: 128 bits.  Defines the node where the packet is
+      expected to exit the SR domain.  In the encapsulation case
+      described in Section 2.2.1, this information corresponds to the
+      last segment of the SRH in the encapsulating header.
+
+3.1.3.  Opaque Container TLV
+
+   The Opaque Container TLV is optional and has the following format:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 11]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+    0                   1                   2                   3
+    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |      Type     |    Length     |   RESERVED    |     Flags     |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |                                                               |
+   |             Opaque Container (16 octets)                      |
+   |                                                               |
+   |                                                               |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+   where:
+
+   o  Type: to be assigned by IANA (suggested value 3).
+
+   o  Length: 18.
+
+   o  RESERVED: 8 bits.  SHOULD be unset on transmission and MUST be
+      ignored on receipt.
+
+   o  Flags: 8 bits.  No flags are defined in this document.
+
+   o  Opaque Container: 128 bits of opaque data not relevant for the
+      routing layer.  Typically, this information is consumed by a non-
+      routing component of the node receiving the packet (i.e.: the node
+      in the DA).
+
+3.1.4.  Padding TLV
+
+   The Padding TLV is optional and with the purpose of aligning the SRH
+   on a 8 octet boundary.  The Padding TLV has the following format:
+
+    0                   1                   2                   3
+    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |     Type      |    Length     |      Padding (variable)       |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   //                    Padding (variable)                       //
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+   where:
+
+   o  Type: to be assigned by IANA (suggested value 4).
+
+   o  Length: 1 to 7
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 12]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   o  Padding: from 1 to 7 octets of padding.  Padding bits have no
+      semantic.  They SHOULD be set to 0 on transmission and MUST be
+      ignored on receipt.
+
+   The following applies to the Padding TLV:
+
+   o  Padding TLV is optional and MAY only appear once in the SRH.  If
+      present, it MUST have a length between 1 and 7 octets.
+
+   o  The Padding TLV is used in order to align the SRH total length on
+      the 8 octet boundary.
+
+   o  When present, the Padding TLV MUST appear as the last TLV before
+      the HMAC TLV (if HMAC TLV is present).
+
+   o  When present, the Padding TLV MUST have a length from 1 to 7 in
+      order to align the SRH total lenght on a 8-octet boundary.
+
+   o  When a router inspecting the SRH encounters the Padding TLV, it
+      MUST assume that no other TLV (other than the HMAC) follow the
+      Padding TLV.
+
+3.1.5.  HMAC TLV
+
+   HMAC TLV is optional and contains the HMAC information.  The HMAC TLV
+   has the following format:
+
+    0                   1                   2                   3
+    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |      Type     |     Length    |          RESERVED             |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |                      HMAC Key ID (4 octets)                   |
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+   |                                                              //
+   |                      HMAC (32 octets)                        //
+   |                                                              //
+   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+   where:
+
+   o  Type: to be assigned by IANA (suggested value 5).
+
+   o  Length: 38.
+
+   o  RESERVED: 2 octets.  SHOULD be unset on transmission and MUST be
+      ignored on receipt.
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 13]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   o  HMAC Key ID: 4 octets.
+
+   o  HMAC: 32 octets.
+
+   o  HMAC and HMAC Key ID usage is described in Section 5
+
+   The Following applies to the HMAC TLV:
+
+   o  When present, the HMAC TLV MUST be encoded as the last TLV of the
+      SRH.
+
+   o  If the HMAC TLV is present, the SRH H-Flag (Figure 4) MUST be set.
+
+   o  When the H-flag is set in the SRH, the router inspecting the SRH
+      MUST find the HMAC TLV in the last 38 octets of the SRH.
+
+3.2.  SRH and RFC2460 behavior
+
+   The SRH being a new type of the Routing Header, it also has the same
+   properties:
+
+      SHOULD only appear once in the packet.
+
+      Only the router whose address is in the DA field of the packet
+      header MUST inspect the SRH.
+
+   Therefore, Segment Routing in IPv6 networks implies that the segment
+   identifier (i.e.: the IPv6 address of the segment) is moved into the
+   DA of the packet.
+
+   The DA of the packet changes at each segment termination/completion
+   and therefore the final DA of the packet MUST be encoded as the last
+   segment of the path.
+
+4.  SRH Procedures
+
+   In this section we describe the different procedures on the SRH.
+
+4.1.  Source SR Node
+
+   A Source SR Node can be any node originating an IPv6 packet with its
+   IPv6 and Segment Routing Headers.  This include either:
+
+      A host originating an IPv6 packet.
+
+      An SR domain ingress router encapsulating a received IPv6 packet
+      into an outer IPv6 header followed by an SRH.
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 14]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   The mechanism through which a Segment List is derived is outside of
+   the scope of this document.  As an example, the Segment List may be
+   obtained through:
+
+      Local path computation.
+
+      Local configuration.
+
+      Interaction with a centralized controller delivering the path.
+
+      Any other mechanism.
+
+   The following are the steps of the creation of the SRH:
+
+      Next Header and Hdr Ext Len fields are set according to [RFC2460].
+
+      Routing Type field is set as TBD (to be allocated by IANA,
+      suggested value 4).
+
+      The Segment List is built with the FIRST segment of the path
+      encoded in the LAST element of the Segment List.  Subsequent
+      segments are encoded on top of the first segment.  Finally, the
+      LAST segment of the path is encoded in the FIRST element of the
+      Segment List.  In other words, the Segment List is encoded in the
+      reverse order of the path.
+
+      The final DA of the packet is encoded as the last segment of the
+      path (encoded in the first element of the Segment List).
+
+      The DA of the packet is set with the value of the first segment
+      (found in the last element of the segment list).
+
+      The Segments Left field is set to n-1 where n is the number of
+      elements in the Segment List.
+
+      The First Segment field is set to n-1 where n is the number of
+      elements in the Segment List.
+
+      The packet is sent out towards the first segment (i.e.:
+      represented in the packet DA).
+
+      HMAC TLV may be set according to Section 5.
+
+4.2.  Transit Node
+
+   According to [RFC2460], the only node who is allowed to inspect the
+   Routing Extension Header (and therefore the SRH), is the node
+   corresponding to the DA of the packet.  Any other transit node MUST
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 15]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   NOT inspect the underneath routing header and MUST forward the packet
+   towards the DA and according to the IPv6 routing table.
+
+   In the example case described in Section 2.2.2, when SR capable nodes
+   are connected through an overlay spanning multiple third-party
+   infrastructure, it is safe to send SRH packets (i.e.: packet having a
+   Segment Routing Header) between each other overlay/SR-capable nodes
+   as long as the segment list does not include any of the transit
+   provider nodes.  In addition, as a generic security measure, any
+   service provider will block any packet destined to one of its
+   internal routers, especially if these packets have an extended header
+   in it.
+
+4.3.  SR Segment Endpoint Node
+
+   The SR segment endpoint node is the node whose address is in the DA.
+   The segment endpoint node inspects the SRH and does:
+
+   1.   IF DA = myself (segment endpoint)
+   2.      IF Segments Left > 0 THEN
+              decrement Segments Left
+              update DA with Segment List[Segments Left]
+   3.      ELSE continue IPv6 processing of the packet
+                End of processing.
+   4.   Forward the packet out
+
+5.  Security Considerations
+
+   This section analyzes the security threat model, the security issues
+   and proposed solutions related to the new Segment Routing Header.
+
+   The Segment Routing Header (SRH) is simply another type of the
+   routing header as described in RFC 2460 [RFC2460] and is:
+
+   o  Added by an SR edge router when entering the segment routing
+      domain or by the originating host itself.  The source host can
+      even be outside the SR domain;
+
+   o  inspected and acted upon when reaching the destination address of
+      the IP header per RFC 2460 [RFC2460].
+
+   Per RFC2460 [RFC2460], routers on the path that simply forward an
+   IPv6 packet (i.e. the IPv6 destination address is none of theirs)
+   will never inspect and process the content of the SRH.  Routers whose
+   one interface IPv6 address equals the destination address field of
+   the IPv6 packet MUST parse the SRH and, if supported and if the local
+   configuration allows it, MUST act accordingly to the SRH content.
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 16]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   According to RFC2460 [RFC2460], the default behavior of a non SR-
+   capable router upon receipt of an IPv6 packet with SRH destined to an
+   address of its, is to:
+
+   o  ignore the SRH completely if the Segment Left field is 0 and
+      proceed to process the next header in the IPv6 packet;
+
+   o  discard the IPv6 packet if Segment Left field is greater than 0,
+      it MAY send a Parameter Problem ICMP message back to the Source
+      Address.
+
+5.1.  Threat model
+
+5.1.1.  Source routing threats
+
+   Using an SRH is similar to source routing, therefore it has some
+   well-known security issues as described in RFC4942 [RFC4942] section
+   2.1.1 and RFC5095 [RFC5095]:
+
+   o  amplification attacks: where a packet could be forged in such a
+      way to cause looping among a set of SR-enabled routers causing
+      unnecessary traffic, hence a Denial of Service (DoS) against
+      bandwidth;
+
+   o  reflection attack: where a hacker could force an intermediate node
+      to appear as the immediate attacker, hence hiding the real
+      attacker from naive forensic;
+
+   o  bypass attack: where an intermediate node could be used as a
+      stepping stone (for example in a De-Militarized Zone) to attack
+      another host (for example in the datacenter or any back-end
+      server).
+
+5.1.2.  Applicability of RFC 5095 to SRH
+
+   First of all, the reader must remember this specific part of section
+   1 of RFC5095 [RFC5095], "A side effect is that this also eliminates
+   benign RH0 use-cases; however, such applications may be facilitated
+   by future Routing Header specifications.".  In short, it is not
+   forbidden to create new secure type of Routing Header; for example,
+   RFC 6554 (RPL) [RFC6554] also creates a new Routing Header type for a
+   specific application confined in a single network.
+
+   In the segment routing architecture described in
+   [I-D.ietf-spring-segment-routing] there are basically two kinds of
+   nodes (routers and hosts):
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 17]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   o  nodes within the SR domain, which is within one single
+      administrative domain, i.e., where all nodes are trusted anyway
+      else the damage caused by those nodes could be worse than
+      amplification attacks: traffic interception, man-in-the-middle
+      attacks, more server DoS by dropping packets, and so on.
+
+   o  nodes outside of the SR domain, which is outside of the
+      administrative segment routing domain hence they cannot be trusted
+      because there is no physical security for those nodes, i.e., they
+      can be replaced by hostile nodes or can be coerced in wrong
+      behaviors.
+
+   The main use case for SR consists of the single administrative domain
+   where only trusted nodes with SR enabled and configured participate
+   in SR: this is the same model as in RFC6554 [RFC6554].  All non-
+   trusted nodes do not participate as either SR processing is not
+   enabled by default or because they only process SRH from nodes within
+   their domain.
+
+   Moreover, all SR nodes ignore SRH created by outsiders based on
+   topology information (received on a peering or internal interface) or
+   on presence and validity of the HMAC field.  Therefore, if
+   intermediate nodes ONLY act on valid and authorized SRH (such as
+   within a single administrative domain), then there is no security
+   threat similar to RH-0.  Hence, the RFC 5095 [RFC5095] attacks are
+   not applicable.
+
+5.1.3.  Service stealing threat
+
+   Segment routing is used for added value services, there is also a
+   need to prevent non-participating nodes to use those services; this
+   is called 'service stealing prevention'.
+
+5.1.4.  Topology disclosure
+
+   The SRH may also contains IPv6 addresses of some intermediate SR-
+   nodes in the path towards the destination, this obviously reveals
+   those addresses to the potentially hostile attackers if those
+   attackers are able to intercept packets containing SRH.  On the other
+   hand, if the attacker can do a traceroute whose probes will be
+   forwarded along the SR path, then there is little learned by
+   intercepting the SRH itself.
+
+5.1.5.  ICMP Generation
+
+   Per section 4.4 of RFC2460 [RFC2460], when destination nodes (i.e.
+   where the destination address is one of theirs) receive a Routing
+   Header with unsupported Routing Type, the required behavior is:
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 18]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   o  If Segments Left is zero, the node must ignore the Routing header
+      and proceed to process the next header in the packet.
+
+   o  If Segments Left is non-zero, the node must discard the packet and
+      send an ICMP Parameter Problem, Code 0, message to the packet's
+      Source Address, pointing to the unrecognized Routing Type.
+
+   This required behavior could be used by an attacker to force the
+   generation of ICMP message by any node.  The attacker could send
+   packets with SRH (with Segment Left set to 0) destined to a node not
+   supporting SRH.  Per RFC2460 [RFC2460], the destination node could
+   generate an ICMP message, causing a local CPU utilization and if the
+   source of the offending packet with SRH was spoofed could lead to a
+   reflection attack without any amplification.
+
+   It must be noted that this is a required behavior for any unsupported
+   Routing Type and not limited to SRH packets.  So, it is not specific
+   to SRH and the usual rate limiting for ICMP generation is required
+   anyway for any IPv6 implementation and has been implemented and
+   deployed for many years.
+
+5.2.  Security fields in SRH
+
+   This section summarizes the use of specific fields in the SRH.  They
+   are based on a key-hashed message authentication code (HMAC).
+
+   The security-related fields in the SRH are instantiated by the HMAC
+   TLV, containing:
+
+   o  HMAC Key-id, 32 bits wide;
+
+   o  HMAC, 256 bits wide (optional, exists only if HMAC Key-id is not
+      0).
+
+   The HMAC field is the output of the HMAC computation (per RFC 2104
+   [RFC2104]) using a pre-shared key identified by HMAC Key-id and of
+   the text which consists of the concatenation of:
+
+   o  the source IPv6 address;
+
+   o  First Segment field;
+
+   o  an octet of bit flags;
+
+   o  HMAC Key-id;
+
+   o  all addresses in the Segment List.
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 19]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   The purpose of the HMAC TLV is to verify the validity, the integrity
+   and the authorization of the SRH itself.  If an outsider of the SR
+   domain does not have access to a current pre-shared secret, then it
+   cannot compute the right HMAC field and the first SR router on the
+   path processing the SRH and configured to check the validity of the
+   HMAC will simply reject the packet.
+
+   The HMAC TLV is located at the end of the SRH simply because only the
+   router on the ingress of the SR domain needs to process it, then all
+   other SR nodes can ignore it (based on local policy) because they
+   trust the upstream router.  This is to speed up forwarding operations
+   because SR routers which do not validate the SRH do not need to parse
+   the SRH until the end.
+
+   The HMAC Key-id field allows for the simultaneous existence of
+   several hash algorithms (SHA-256, SHA3-256 ... or future ones) as
+   well as pre-shared keys.  The HMAC Key-id field is opaque, i.e., it
+   has neither syntax nor semantic except as an index to the right
+   combination of pre-shared key and hash algorithm and except that a
+   value of 0 means that there is no HMAC field.  Having an HMAC Key-id
+   field allows for pre-shared key roll-over when two pre-shared keys
+   are supported for a while when all SR nodes converged to a fresher
+   pre-shared key.  It could also allow for interoperation among
+   different SR domains if allowed by local policy and assuming a
+   collision-free HMAC Key Id allocation.
+
+   When a specific SRH is linked to a time-related service (such as
+   turbo-QoS for a 1-hour period) where the DA, Segment ID (SID) are
+   identical, then it is important to refresh the shared-secret
+   frequently as the HMAC validity period expires only when the HMAC
+   Key-id and its associated shared-secret expires.
+
+5.2.1.  Selecting a hash algorithm
+
+   The HMAC field in the HMAC TLV is 256 bit wide.  Therefore, the HMAC
+   MUST be based on a hash function whose output is at least 256 bits.
+   If the output of the hash function is 256, then this output is simply
+   inserted in the HMAC field.  If the output of the hash function is
+   larger than 256 bits, then the output value is truncated to 256 by
+   taking the least-significant 256 bits and inserting them in the HMAC
+   field.
+
+   SRH implementations can support multiple hash functions but MUST
+   implement SHA-2 [FIPS180-4] in its SHA-256 variant.
+
+   NOTE: SHA-1 is currently used by some early implementations used for
+   quick interoperations testing, the 160-bit hash value must then be
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 20]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   right-hand padded with 96 bits set to 0.  The authors understand that
+   this is not secure but is ok for limited tests.
+
+5.2.2.  Performance impact of HMAC
+
+   While adding an HMAC to each and every SR packet increases the
+   security, it has a performance impact.  Nevertheless, it must be
+   noted that:
+
+   o  the HMAC field is used only when SRH is added by a device (such as
+      a home set-up box) which is outside of the segment routing domain.
+      If the SRH is added by a router in the trusted segment routing
+      domain, then, there is no need for an HMAC field, hence no
+      performance impact.
+
+   o  when present, the HMAC field MUST only be checked and validated by
+      the first router of the segment routing domain, this router is
+      named 'validating SR router'.  Downstream routers may not inspect
+      the HMAC field.
+
+   o  this validating router can also have a cache of  to improve the performance.  It is not the
+      same use case as in IPsec where HMAC value was unique per packet,
+      in SRH, the HMAC value is unique per flow.
+
+   o  Last point, hash functions such as SHA-2 have been optimized for
+      security and performance and there are multiple implementations
+      with good performance.
+
+   With the above points in mind, the performance impact of using HMAC
+   is minimized.
+
+5.2.3.  Pre-shared key management
+
+   The field HMAC Key-id allows for:
+
+   o  key roll-over: when there is a need to change the key (the hash
+      pre-shared secret), then multiple pre-shared keys can be used
+      simultaneously.  The validating routing can have a table of  for the currently active and future
+      keys.
+
+   o  different algorithms: by extending the previous table to , the validating router
+      can also support simultaneously several hash algorithms (see
+      section Section 5.2.1)
+
+   The pre-shared secret distribution can be done:
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 21]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   o  in the configuration of the validating routers, either by static
+      configuration or any SDN oriented approach;
+
+   o  dynamically using a trusted key distribution such as [RFC6407]
+
+   The intent of this document is NOT to define yet-another-key-
+   distribution-protocol.
+
+5.3.  Deployment Models
+
+5.3.1.  Nodes within the SR domain
+
+   An SR domain is defined as a set of interconnected routers where all
+   routers at the perimeter are configured to add and act on SRH.  Some
+   routers inside the SR domain can also act on SRH or simply forward
+   IPv6 packets.
+
+   The routers inside an SR domain can be trusted to generate SRH and to
+   process SRH received on interfaces that are part of the SR domain.
+   These nodes MUST drop all SRH packets received on an interface that
+   is not part of the SR domain and containing an SRH whose HMAC field
+   cannot be validated by local policies.  This includes obviously
+   packet with an SRH generated by a non-cooperative SR domain.
+
+   If the validation fails, then these packets MUST be dropped, ICMP
+   error messages (parameter problem) SHOULD be generated (but rate
+   limited) and SHOULD be logged.
+
+5.3.2.  Nodes outside of the SR domain
+
+   Nodes outside of the SR domain cannot be trusted for physical
+   security; hence, they need to request by some trusted means (outside
+   of the scope of this document) a complete SRH for each new connection
+   (i.e. new destination address).  The received SRH MUST include an
+   HMAC TLV which is computed correctly (see Section 5.2).
+
+   When an outside node sends a packet with an SRH and towards an SR
+   domain ingress node, the packet MUST contain the HMAC TLV (with a
+   Key-id and HMAC fields) and the the destination address MUST be an
+   address of an SR domain ingress node .
+
+   The ingress SR router, i.e., the router with an interface address
+   equals to the destination address, MUST verify the HMAC TLV.
+
+   If the validation is successful, then the packet is simply forwarded
+   as usual for an SR packet.  As long as the packet travels within the
+   SR domain, no further HMAC check needs to be done.  Subsequent
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 22]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   routers in the SR domain MAY verify the HMAC TLV when they process
+   the SRH (i.e. when they are the destination).
+
+   If the validation fails, then this packet MUST be dropped, an ICMP
+   error message (parameter problem) SHOULD be generated (but rate
+   limited) and SHOULD be logged.
+
+5.3.3.  SR path exposure
+
+   As the intermediate SR nodes addresses appears in the SRH, if this
+   SRH is visible to an outsider then he/she could reuse this knowledge
+   to launch an attack on the intermediate SR nodes or get some insider
+   knowledge on the topology.  This is especially applicable when the
+   path between the source node and the first SR domain ingress router
+   is on the public Internet.
+
+   The first remark is to state that 'security by obscurity' is never
+   enough; in other words, the security policy of the SR domain MUST
+   assume that the internal topology and addressing is known by the
+   attacker.  A simple traceroute will also give the same information
+   (with even more information as all intermediate nodes between SID
+   will also be exposed).  IPsec Encapsulating Security Payload
+   [RFC4303] cannot be use to protect the SRH as per RFC4303 the ESP
+   header must appear after any routing header (including SRH).
+
+   To prevent a user to leverage the gained knowledge by intercepting
+   SRH, it it recommended to apply an infrastructure Access Control List
+   (iACL) at the edge of the SR domain.  This iACL will drop all packets
+   from outside the SR-domain whose destination is any address of any
+   router inside the domain.  This security policy should be tuned for
+   local operations.
+
+5.3.4.  Impact of BCP-38
+
+   BCP-38 [RFC2827], also known as "Network Ingress Filtering", checks
+   whether the source address of packets received on an interface is
+   valid for this interface.  The use of loose source routing such as
+   SRH forces packets to follow a path which differs from the expected
+   routing.  Therefore, if BCP-38 was implemented in all routers inside
+   the SR domain, then SR packets could be received by an interface
+   which is not expected one and the packets could be dropped.
+
+   As an SR domain is usually a subset of one administrative domain, and
+   as BCP-38 is only deployed at the ingress routers of this
+   administrative domain and as packets arriving at those ingress
+   routers have been normally forwarded using the normal routing
+   information, then there is no reason why this ingress router should
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 23]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   drop the SRH packet based on BCP-38.  Routers inside the domain
+   commonly do not apply BCP-38; so, this is not a problem.
+
+6.  IANA Considerations
+
+   This document makes the following registrations in the Internet
+   Protocol Version 6 (IPv6) Parameters "Routing Type" registry
+   maintained by IANA:
+
+   Suggested            Description             Reference
+     Value
+   ----------------------------------------------------------
+      4         Segment Routing Header (SRH)    This document
+
+   In addition, this document request IANA to create and maintain a new
+   Registry: "Segment Routing Header Type-Value Objects".  The following
+   code-points are requested from the registry:
+
+   Registry: Segment Routing Header Type-Value Objects
+
+   Suggested         Description            Reference
+     Value
+   -----------------------------------------------------
+      1         Ingress Node TLV          This document
+      2         Egress Node  TLV          This document
+      3         Opaque Container TLV      This document
+      4         Padding TLV               This document
+      5         HMAC TLV                  This document
+
+7.  Manageability Considerations
+
+   TBD
+
+8.  Contributors
+
+   Dave Barach, John Leddy, John Brzozowski, Pierre Francois, Nagendra
+   Kumar, Mark Townsley, Christian Martin, Roberta Maglione, James
+   Connolly, Aloys Augustin contributed to the content of this document.
+
+9.  Acknowledgements
+
+   The authors would like to thank Ole Troan, Bob Hinden, Fred Baker,
+   Brian Carpenter, Alexandru Petrescu and Punit Kumar Jaiswal for their
+   comments to this document.
+
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 24]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+10.  References
+
+10.1.  Normative References
+
+   [FIPS180-4]
+              National Institute of Standards and Technology, "FIPS
+              180-4 Secure Hash Standard (SHS)", March 2012,
+              .
+
+   [RFC2119]  Bradner, S., "Key words for use in RFCs to Indicate
+              Requirement Levels", BCP 14, RFC 2119,
+              DOI 10.17487/RFC2119, March 1997,
+              .
+
+   [RFC2460]  Deering, S. and R. Hinden, "Internet Protocol, Version 6
+              (IPv6) Specification", RFC 2460, DOI 10.17487/RFC2460,
+              December 1998, .
+
+   [RFC4303]  Kent, S., "IP Encapsulating Security Payload (ESP)",
+              RFC 4303, DOI 10.17487/RFC4303, December 2005,
+              .
+
+   [RFC5095]  Abley, J., Savola, P., and G. Neville-Neil, "Deprecation
+              of Type 0 Routing Headers in IPv6", RFC 5095,
+              DOI 10.17487/RFC5095, December 2007,
+              .
+
+   [RFC6407]  Weis, B., Rowles, S., and T. Hardjono, "The Group Domain
+              of Interpretation", RFC 6407, DOI 10.17487/RFC6407,
+              October 2011, .
+
+10.2.  Informative References
+
+   [I-D.ietf-isis-segment-routing-extensions]
+              Previdi, S., Filsfils, C., Bashandy, A., Gredler, H.,
+              Litkowski, S., Decraene, B., and j. jefftant@gmail.com,
+              "IS-IS Extensions for Segment Routing", draft-ietf-isis-
+              segment-routing-extensions-09 (work in progress), October
+              2016.
+
+   [I-D.ietf-ospf-ospfv3-segment-routing-extensions]
+              Psenak, P., Previdi, S., Filsfils, C., Gredler, H.,
+              Shakir, R., Henderickx, W., and J. Tantsura, "OSPFv3
+              Extensions for Segment Routing", draft-ietf-ospf-ospfv3-
+              segment-routing-extensions-07 (work in progress), October
+              2016.
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 25]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   [I-D.ietf-spring-ipv6-use-cases]
+              Brzozowski, J., Leddy, J., Townsley, W., Filsfils, C., and
+              R. Maglione, "IPv6 SPRING Use Cases", draft-ietf-spring-
+              ipv6-use-cases-08 (work in progress), January 2017.
+
+   [I-D.ietf-spring-resiliency-use-cases]
+              Filsfils, C., Previdi, S., Decraene, B., and R. Shakir,
+              "Resiliency use cases in SPRING networks", draft-ietf-
+              spring-resiliency-use-cases-08 (work in progress), October
+              2016.
+
+   [I-D.ietf-spring-segment-routing]
+              Filsfils, C., Previdi, S., Decraene, B., Litkowski, S.,
+              and R. Shakir, "Segment Routing Architecture", draft-ietf-
+              spring-segment-routing-10 (work in progress), November
+              2016.
+
+   [I-D.ietf-spring-segment-routing-mpls]
+              Filsfils, C., Previdi, S., Bashandy, A., Decraene, B.,
+              Litkowski, S., Horneffer, M., Shakir, R.,
+              jefftant@gmail.com, j., and E. Crabbe, "Segment Routing
+              with MPLS data plane", draft-ietf-spring-segment-routing-
+              mpls-06 (work in progress), January 2017.
+
+   [RFC1940]  Estrin, D., Li, T., Rekhter, Y., Varadhan, K., and D.
+              Zappala, "Source Demand Routing: Packet Format and
+              Forwarding Specification (Version 1)", RFC 1940,
+              DOI 10.17487/RFC1940, May 1996,
+              .
+
+   [RFC2104]  Krawczyk, H., Bellare, M., and R. Canetti, "HMAC: Keyed-
+              Hashing for Message Authentication", RFC 2104,
+              DOI 10.17487/RFC2104, February 1997,
+              .
+
+   [RFC2827]  Ferguson, P. and D. Senie, "Network Ingress Filtering:
+              Defeating Denial of Service Attacks which employ IP Source
+              Address Spoofing", BCP 38, RFC 2827, DOI 10.17487/RFC2827,
+              May 2000, .
+
+   [RFC4942]  Davies, E., Krishnan, S., and P. Savola, "IPv6 Transition/
+              Co-existence Security Considerations", RFC 4942,
+              DOI 10.17487/RFC4942, September 2007,
+              .
+
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 26]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   [RFC6554]  Hui, J., Vasseur, JP., Culler, D., and V. Manral, "An IPv6
+              Routing Header for Source Routes with the Routing Protocol
+              for Low-Power and Lossy Networks (RPL)", RFC 6554,
+              DOI 10.17487/RFC6554, March 2012,
+              .
+
+   [RFC7855]  Previdi, S., Ed., Filsfils, C., Ed., Decraene, B.,
+              Litkowski, S., Horneffer, M., and R. Shakir, "Source
+              Packet Routing in Networking (SPRING) Problem Statement
+              and Requirements", RFC 7855, DOI 10.17487/RFC7855, May
+              2016, .
+
+Authors' Addresses
+
+   Stefano Previdi (editor)
+   Cisco Systems, Inc.
+   Via Del Serafico, 200
+   Rome  00142
+   Italy
+
+   Email: sprevidi@cisco.com
+
+
+   Clarence Filsfils
+   Cisco Systems, Inc.
+   Brussels
+   BE
+
+   Email: cfilsfil@cisco.com
+
+
+   Brian Field
+   Comcast
+   4100 East Dry Creek Road
+   Centennial, CO  80122
+   US
+
+   Email: Brian_Field@cable.comcast.com
+
+
+   Ida Leung
+   Rogers Communications
+   8200 Dixie Road
+   Brampton, ON  L6T 0C1
+   CA
+
+   Email: Ida.Leung@rci.rogers.com
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 27]
+ 
+Internet-Draft      IPv6 Segment Routing Header (SRH)      February 2017
+
+
+   Jen Linkova
+   Google
+   1600 Amphitheatre Parkway
+   Mountain View, CA 94043
+   US
+
+   Email: furry@google.com
+
+
+   Ebben Aries
+   Facebook
+   US
+
+   Email: exa@fb.com
+
+
+   Tomoya Kosugi
+   NTT
+   3-9-11, Midori-Cho Musashino-Shi,
+   Tokyo  180-8585
+   JP
+
+   Email: kosugi.tomoya@lab.ntt.co.jp
+
+
+   Eric Vyncke
+   Cisco Systems, Inc.
+   De Kleetlaann 6A
+   Diegem  1831
+   Belgium
+
+   Email: evyncke@cisco.com
+
+
+   David Lebrun
+   Universite Catholique de Louvain
+   Place Ste Barbe, 2
+   Louvain-la-Neuve, 1348
+   Belgium
+
+   Email: david.lebrun@uclouvain.be
+
+
+
+
+
+
+
+
+
+
+Previdi, et al.          Expires August 5, 2017                [Page 28]
\ No newline at end of file
diff --git a/src/vnet/sr/rfc_draft_05.txt b/src/vnet/sr/rfc_draft_05.txt
deleted file mode 100644
index bc41c181..00000000
--- a/src/vnet/sr/rfc_draft_05.txt
+++ /dev/null
@@ -1,1265 +0,0 @@
-Network Working Group                                    S. Previdi, Ed.
-Internet-Draft                                               C. Filsfils
-Intended status: Standards Track                     Cisco Systems, Inc.
-Expires: June 12, 2015                                          B. Field
-                                                                 Comcast
-                                                                I. Leung
-                                                   Rogers Communications
-                                                        December 9, 2014
-
-
-                   IPv6 Segment Routing Header (SRH)
-              draft-previdi-6man-segment-routing-header-05
-
-Abstract
-
-   Segment Routing (SR) allows a node to steer a packet through a
-   controlled set of instructions, called segments, by prepending a SR
-   header to the packet.  A segment can represent any instruction,
-   topological or service-based.  SR allows to enforce a flow through
-   any path (topological, or application/service based) while
-   maintaining per-flow state only at the ingress node to the SR domain.
-
-   Segment Routing can be applied to the IPv6 data plane with the
-   addition of a new type of Routing Extension Header.  This draft
-   describes the Segment Routing Extension Header Type and how it is
-   used by SR capable nodes.
-
-Requirements Language
-
-   The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
-   "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
-   document are to be interpreted as described in RFC 2119 [RFC2119].
-
-Status of This Memo
-
-   This Internet-Draft is submitted in full conformance with the
-   provisions of BCP 78 and BCP 79.
-
-   Internet-Drafts are working documents of the Internet Engineering
-   Task Force (IETF).  Note that other groups may also distribute
-   working documents as Internet-Drafts.  The list of current Internet-
-   Drafts is at http://datatracker.ietf.org/drafts/current/.
-
-   Internet-Drafts are draft documents valid for a maximum of six months
-   and may be updated, replaced, or obsoleted by other documents at any
-   time.  It is inappropriate to use Internet-Drafts as reference
-   material or to cite them other than as "work in progress."
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 1]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   This Internet-Draft will expire on June 12, 2015.
-
-Copyright Notice
-
-   Copyright (c) 2014 IETF Trust and the persons identified as the
-   document authors.  All rights reserved.
-
-   This document is subject to BCP 78 and the IETF Trust's Legal
-   Provisions Relating to IETF Documents
-   (http://trustee.ietf.org/license-info) in effect on the date of
-   publication of this document.  Please review these documents
-   carefully, as they describe your rights and restrictions with respect
-   to this document.  Code Components extracted from this document must
-   include Simplified BSD License text as described in Section 4.e of
-   the Trust Legal Provisions and are provided without warranty as
-   described in the Simplified BSD License.
-
-Table of Contents
-
-   1.  Structure of this document  . . . . . . . . . . . . . . . . .   3
-   2.  Segment Routing Documents . . . . . . . . . . . . . . . . . .   3
-   3.  Introduction  . . . . . . . . . . . . . . . . . . . . . . . .   3
-     3.1.  Data Planes supporting Segment Routing  . . . . . . . . .   4
-     3.2.  Illustration  . . . . . . . . . . . . . . . . . . . . . .   4
-   4.  Abstract Routing Model  . . . . . . . . . . . . . . . . . . .   7
-     4.1.  Segment Routing Global Block (SRGB) . . . . . . . . . . .   8
-     4.2.  Traffic Engineering with SR . . . . . . . . . . . . . . .   9
-     4.3.  Segment Routing Database  . . . . . . . . . . . . . . . .  10
-   5.  IPv6 Instantiation of Segment Routing . . . . . . . . . . . .  10
-     5.1.  Segment Identifiers (SIDs) and SRGB . . . . . . . . . . .  10
-       5.1.1.  Node-SID  . . . . . . . . . . . . . . . . . . . . . .  11
-       5.1.2.  Adjacency-SID . . . . . . . . . . . . . . . . . . . .  11
-     5.2.  Segment Routing Extension Header (SRH)  . . . . . . . . .  11
-       5.2.1.  SRH and RFC2460 behavior  . . . . . . . . . . . . . .  15
-   6.  SRH Procedures  . . . . . . . . . . . . . . . . . . . . . . .  15
-     6.1.  Segment Routing Operations  . . . . . . . . . . . . . . .  15
-     6.2.  Segment Routing Node Functions  . . . . . . . . . . . . .  16
-       6.2.1.  Ingress SR Node . . . . . . . . . . . . . . . . . . .  16
-       6.2.2.  Transit Non-SR Capable Node . . . . . . . . . . . . .  18
-       6.2.3.  SR Intra Segment Transit Node . . . . . . . . . . . .  18
-       6.2.4.  SR Segment Endpoint Node  . . . . . . . . . . . . . .  18
-     6.3.  FRR Flag Settings . . . . . . . . . . . . . . . . . . . .  18
-   7.  SR and Tunneling  . . . . . . . . . . . . . . . . . . . . . .  18
-   8.  Example Use Case  . . . . . . . . . . . . . . . . . . . . . .  19
-   9.  IANA Considerations . . . . . . . . . . . . . . . . . . . . .  21
-   10. Manageability Considerations  . . . . . . . . . . . . . . . .  21
-   11. Security Considerations . . . . . . . . . . . . . . . . . . .  21
-   12. Contributors  . . . . . . . . . . . . . . . . . . . . . . . .  21
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 2]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   13. Acknowledgements  . . . . . . . . . . . . . . . . . . . . . .  21
-   14. References  . . . . . . . . . . . . . . . . . . . . . . . . .  21
-     14.1.  Normative References . . . . . . . . . . . . . . . . . .  21
-     14.2.  Informative References . . . . . . . . . . . . . . . . .  21
-   Authors' Addresses  . . . . . . . . . . . . . . . . . . . . . . .  22
-
-1.  Structure of this document
-
-   Section 3 gives an introduction on SR for IPv6 networks.
-
-   Section 4 describes the Segment Routing abstract model.
-
-   Section 5 defines the Segment Routing Header (SRH) allowing
-   instantiation of SR over IPv6 dataplane.
-
-   Section 6 details the procedures of the Segment Routing Header.
-
-2.  Segment Routing Documents
-
-   Segment Routing terminology is defined in
-   [I-D.filsfils-spring-segment-routing].
-
-   Segment Routing use cases are described in
-   [I-D.filsfils-spring-segment-routing-use-cases].
-
-   Segment Routing IPv6 use cases are described in
-   [I-D.ietf-spring-ipv6-use-cases].
-
-   Segment Routing protocol extensions are defined in
-   [I-D.ietf-isis-segment-routing-extensions], and
-   [I-D.psenak-ospf-segment-routing-ospfv3-extension].
-
-   The security mechanisms of the Segment Routing Header (SRH) are
-   described in [I-D.vyncke-6man-segment-routing-security].
-
-3.  Introduction
-
-   Segment Routing (SR), defined in
-   [I-D.filsfils-spring-segment-routing], allows a node to steer a
-   packet through a controlled set of instructions, called segments, by
-   prepending a SR header to the packet.  A segment can represent any
-   instruction, topological or service-based.  SR allows to enforce a
-   flow through any path (topological or service/application based)
-   while maintaining per-flow state only at the ingress node to the SR
-   domain.  Segments can be derived from different components: IGP, BGP,
-   Services, Contexts, Locators, etc.  The list of segment forming the
-   path is called the Segment List and is encoded in the packet header.
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 3]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   SR allows the use of strict and loose source based routing paradigms
-   without requiring any additional signaling protocols in the
-   infrastructure hence delivering an excellent scalability property.
-
-   The source based routing model described in
-   [I-D.filsfils-spring-segment-routing] is inherited from the ones
-   proposed by [RFC1940] and [RFC2460].  The source based routing model
-   offers the support for explicit routing capability.
-
-3.1.  Data Planes supporting Segment Routing
-
-   Segment Routing (SR), can be instantiated over MPLS
-   ([I-D.filsfils-spring-segment-routing-mpls]) and IPv6.  This document
-   defines its instantiation over the IPv6 data-plane based on the use-
-   cases defined in [I-D.ietf-spring-ipv6-use-cases].
-
-   Segment Routing for IPv6 (SR-IPv6) is required in networks where MPLS
-   data-plane is not used or, when combined with SR-MPLS, in networks
-   where MPLS is used in the core and IPv6 is used at the edge (home
-   networks, datacenters).
-
-   This document defines a new type of Routing Header (originally
-   defined in [RFC2460]) called the Segment Routing Header (SRH) in
-   order to convey the Segment List in the packet header as defined in
-   [I-D.filsfils-spring-segment-routing].  Mechanisms through which
-   segment are known and advertised are outside the scope of this
-   document.
-
-3.2.  Illustration
-
-   In the context of Figure 1 where all the links have the same IGP
-   cost, let us assume that a packet P enters the SR domain at an
-   ingress edge router I and that the operator requests the following
-   requirements for packet P:
-
-      The local service S offered by node B must be applied to packet P.
-
-      The links AB and CE cannot be used to transport the packet P.
-
-      Any node N along the journey of the packet should be able to
-      determine where the packet P entered the SR domain and where it
-      will exit.  The intermediate node should be able to determine the
-      paths from the ingress edge router to itself, and from itself to
-      the egress edge router.
-
-      Per-flow State for packet P should only be created at the ingress
-      edge router.
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 4]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-      The operator can forbid, for security reasons, anyone outside the
-      operator domain to exploit its intra-domain SR capabilities.
-
-   I---A---B---C---E
-        \  |  / \ /
-         \ | /   F
-          \|/
-           D
-
-                Figure 1: An illustration of SR properties
-
-   All these properties may be realized by instructing the ingress SR
-   edge router I to push the following abstract SR header on the packet
-   P.
-
-   +---------------------------------------------------------------+
-   |                                   |                           |
-   |      Abstract SR Header           |                           |
-   |                                   |                           |
-   | {SD, SB, SS, SF, SE}, Ptr, SI, SE |        Transported        |
-   |  ^                     |          |           Packet          |
-   |  |                     |          |             P             |
-   |  +---------------------+          |                           |
-   |                                   |                           |
-   +---------------------------------------------------------------+
-
-                       Figure 2: Packet P at node I
-
-   The abstract SR header contains a source route encoded as a list of
-   segments {SD, SB, SS, SF, SE}, a pointer (Ptr) and the identification
-   of the ingress and egress SR edge routers (segments SI and SE).
-
-   A segment identifies a topological instruction or a service
-   instruction.  A segment can either be global or local.  The
-   instruction associated with a global segment is recognized and
-   executed by any SR-capable node in the domain.  The instruction
-   associated with a local segment is only supported by the specific
-   node that originates it.
-
-   Let us assume some IGP (i.e.: ISIS and OSPF) extensions to define a
-   "Node Segment" as a global instruction within the IGP domain to
-   forward a packet along the shortest path to the specified node.  Let
-   us further assume that within the SR domain illustrated in Figure 1,
-   segments SI, SD, SB, SE and SF respectively identify IGP node
-   segments to I, D, B, E and F.
-
-   Let us assume that node B identifies its local service S with local
-   segment SS.
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 5]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   With all of this in mind, let us describe the journey of the packet
-   P.
-
-   The packet P reaches the ingress SR edge router.  I pushes the SR
-   header illustrated in Figure 2 and sets the pointer to the first
-   segment of the list (SD).
-
-   SD is an instruction recognized by all the nodes in the SR domain
-   which causes the packet to be forwarded along the shortest path to D.
-
-   Once at D, the pointer is incremented and the next segment is
-   executed (SB).
-
-   SB is an instruction recognized by all the nodes in the SR domain
-   which causes the packet to be forwarded along the shortest path to B.
-
-   Once at B, the pointer is incremented and the next segment is
-   executed (SS).
-
-   SS is an instruction only recognized by node B which causes the
-   packet to receive service S.
-
-   Once the service applied, the next segment is executed (SF) which
-   causes the packet to be forwarded along the shortest path to F.
-
-   Once at F, the pointer is incremented and the next segment is
-   executed (SE).
-
-   SE is an instruction recognized by all the nodes in the SR domain
-   which causes the packet to be forwarded along the shortest path to E.
-
-   E then removes the SR header and the packet continues its journey
-   outside the SR domain.
-
-   All of the requirements are met.
-
-   First, the packet P has not used links AB and CE: the shortest-path
-   from I to D is I-A-D, the shortest-path from D to B is D-B, the
-   shortest-path from B to F is B-C-F and the shortest-path from F to E
-   is F-E, hence the packet path through the SR domain is I-A-D-B-C-F-E
-   and the links AB and CE have been avoided.
-
-   Second, the service S supported by B has been applied on packet P.
-
-   Third, any node along the packet path is able to identify the service
-   and topological journey of the packet within the SR domain.  For
-   example, node C receives the packet illustrated in Figure 3 and hence
-   is able to infer where the packet entered the SR domain (SI), how it
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 6]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   got up to itself {SD, SB, SS, SE}, where it will exit the SR domain
-   (SE) and how it will do so {SF, SE}.
-
-   +---------------------------------------------------------------+
-   |                                   |                           |
-   |           SR Header               |                           |
-   |                                   |                           |
-   | {SD, SB, SS, SF, SE}, Ptr, SI, SE |        Transported        |
-   |               ^        |          |           Packet          |
-   |               |        |          |             P             |
-   |               +--------+          |                           |
-   |                                   |                           |
-   +---------------------------------------------------------------+
-
-                       Figure 3: Packet P at node C
-
-   Fourth, only node I maintains per-flow state for packet P.  The
-   entire program of topological and service instructions to be executed
-   by the SR domain on packet P is encoded by the ingress edge router I
-   in the SR header in the form of a list of segments where each segment
-   identifies a specific instruction.  No further per-flow state is
-   required along the packet path.  The per-flow state is in the SR
-   header and travels with the packet.  Intermediate nodes only hold
-   states related to the IGP global node segments and the local IGP
-   adjacency segments.  These segments are not per-flow specific and
-   hence scale very well.  Typically, an intermediate node would
-   maintain in the order of 100's to 1000's global node segments and in
-   the order of 10's to 100 of local adjacency segments.  Typically the
-   SR IGP forwarding table is expected to be much less than 10000
-   entries.
-
-   Fifth, the SR header is inserted at the entrance to the domain and
-   removed at the exit of the operator domain.  For security reasons,
-   the operator can forbid anyone outside its domain to use its intra-
-   domain SR capability.
-
-4.  Abstract Routing Model
-
-   At the entrance of the SR domain, the ingress SR edge router pushes
-   the SR header on top of the packet.  At the exit of the SR domain,
-   the egress SR edge router removes the SR header.
-
-   The abstract SR header contains an ordered list of segments, a
-   pointer identifying the next segment to process and the
-   identifications of the ingress and egress SR edge routers on the path
-   of this packet.  The pointer identifies the segment that MUST be used
-   by the receiving router to process the packet.  This segment is
-   called the active segment.
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 7]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   A property of SR is that the entire source route of the packet,
-   including the identity of the ingress and egress edge routers is
-   always available with the packet.  This allows for interesting
-   accounting and service applications.
-
-   We define three SR-header operations:
-
-      "PUSH": an SR header is pushed on an IP packet, or additional
-      segments are added at the head of the segment list.  The pointer
-      is moved to the first entry of the added segments.
-
-      "NEXT": the active segment is completed, the pointer is moved to
-      the next segment in the list.
-
-      "CONTINUE": the active segment is not completed, the pointer is
-      left unchanged.
-
-   In the future, other SR-header management operations may be defined.
-
-   As the packet travels through the SR domain, the pointer is
-   incremented through the ordered list of segments and the source route
-   encoded by the SR ingress edge node is executed.
-
-   A node processes an incoming packet according to the instruction
-   associated with the active segment.
-
-   Any instruction might be associated with a segment: for example, an
-   intra-domain topological strict or loose forwarding instruction, a
-   service instruction, etc.
-
-   At minimum, a segment instruction must define two elements: the
-   identity of the next-hop to forward the packet to (this could be the
-   same node or a context within the node) and which SR-header
-   management operation to execute.
-
-   Each segment is known in the network through a Segment Identifier
-   (SID).  The terms "segment" and "SID" are interchangeable.
-
-4.1.  Segment Routing Global Block (SRGB)
-
-   In the SR abstract model, a segment is identified by a Segment
-   Routing Identifier (SID).  The SR abstract model doesn't mandate a
-   specific format for the SID (IPv6 address or other formats).
-
-   In Segment Routing IPv6 the SID is an IPv6 address.  Therefore, the
-   SRGB is materialized by the global IPv6 address space which
-   represents the set of IPv6 routable addresses in the SR domain.  The
-   following rules apply:
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 8]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   o  Each node of the SR domain MUST be configured with the Segment
-      Routing Global Block (SRGB).
-
-   o  All global segments must be allocated from the SRGB.  Any SR
-      capable node MUST be able to process any global segment advertised
-      by any other node within the SR domain.
-
-   o  Any segment outside the SRGB has a local significance and is
-      called a "local segment".  An SR-capable node MUST be able to
-      process the local segments it originates.  An SR-capable node MUST
-      NOT support the instruction associated with a local segment
-      originated by a remote node.
-
-4.2.  Traffic Engineering with SR
-
-   An SR Traffic Engineering policy is composed of two elements: a flow
-   classification and a segment-list to prepend on the packets of the
-   flow.
-
-   In SR, this per-flow state only exists at the ingress edge node where
-   the policy is defined and the SR header is pushed.
-
-   It is outside the scope of the document to define the process that
-   leads to the instantiation at a node N of an SR Traffic Engineering
-   policy.
-
-   [I-D.filsfils-spring-segment-routing-use-cases] illustrates various
-   alternatives:
-
-      N is deriving this policy automatically (e.g.  FRR).
-
-      N is provisioned explicitly by the operator.
-
-      N is provisioned by a controller or server (e.g.: SDN Controller).
-
-      N is provisioned by the operator with a high-level policy which is
-      mapped into a path thanks to a local CSPF-based computation (e.g.
-      affinity/SRLG exclusion).
-
-      N could also be provisioned by other means.
-
-   [I-D.filsfils-spring-segment-routing-use-cases] explains why the
-   majority of use-cases require very short segment-lists, hence
-   minimizing the performance impact, if any, of inserting and
-   transporting the segment list.
-
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                 [Page 9]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   A SDN controller, which desires to instantiate at node N an SR
-   Traffic Engineering policy, collects the SR capability of node N such
-   as to ensure that the policy meets its capability.
-
-4.3.  Segment Routing Database
-
-   The Segment routing Database (SRDB) is a set of entries where each
-   entry is identified by a SID.  The instruction associated with each
-   entry at least defines the identity of the next-hop to which the
-   packet should be forwarded and what operation should be performed on
-   the SR header (PUSH, CONTINUE, NEXT).
-
-   +---------+-----------+---------------------------------+
-   | Segment |  Next-Hop |  SR Header operation            |
-   +---------+-----------+---------------------------------+
-   |   Sk    |     M     | CONTINUE                        |
-   |   Sj    |     N     | NEXT                            |
-   |   Sl    | NAT Srvc  | NEXT                            |
-   |   Sm    |  FW srvc  | NEXT                            |
-   |   Sn    |     Q     | NEXT                            |
-   |  etc.   |   etc.    | etc.                            |
-   +---------+-----------+---------------------------------+
-
-                           Figure 4: SR Database
-
-   Each SR-capable node maintains its local SRDB.  SRDB entries can
-   either derive from local policy or from protocol segment
-   advertisement.
-
-5.  IPv6 Instantiation of Segment Routing
-
-5.1.  Segment Identifiers (SIDs) and SRGB
-
-   Segment Routing, as described in
-   [I-D.filsfils-spring-segment-routing], defines Node-SID and
-   Adjacency-SID.  When SR is used over IPv6 data-plane the following
-   applies.
-
-   The SRGB is the global IPv6 address space which represents the set of
-   IPv6 routable addresses in the SR domain.
-
-   Node SIDs are IPv6 addresses part of the SRGB (i.e.: routable
-   addresses).  Adjacency-SIDs are IPv6 addresses which may not be part
-   of the global IPv6 address space.
-
-
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 10]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-5.1.1.  Node-SID
-
-   The Node-SID identifies a node.  With SR-IPv6 the Node-SID is an IPv6
-   prefix that the operator configured on the node and that is used as
-   the node identifier.  Typically, in case of a router, this is the
-   IPv6 address of the node loopback interface.  Therefore, SR-IPv6 does
-   not require any additional SID advertisement for the Node Segment.
-   The Node-SID is in fact the IPv6 address of the node.
-
-5.1.2.  Adjacency-SID
-
-   In the SR architecture defined in
-   [I-D.filsfils-spring-segment-routing] the Adjacency-SID (or Adj-SID)
-   identifies a given interface and may be local or global (depending on
-   how it is advertised).  A node may advertise one (or more) Adj-SIDs
-   allocated to a given interface so to force the forwarding of the
-   packet (when received with that particular Adj-SID) into the
-   interface regardless the routing entry for the packet destination.
-   The semantic of the Adj-SID is:
-
-      Send out the packet to the interface this prefix is allocated to.
-
-   When SR is applied to IPv6, any SID is in a global IPv6 address and
-   therefore, an Adj-SID has a global significance (i.e.: the IPv6
-   address representing the SID is a global address).  In other words, a
-   node that advertises the Adj-SID in the form of a global IPv6 address
-   representing the link/adjacency the packet has to be forwarded to,
-   will apply to the Adj-SID a global significance.
-
-   Advertisement of Adj-SID may be done using multiple mechanisms among
-   which the ones described in ISIS and OSPF protocol extensions:
-   [I-D.ietf-isis-segment-routing-extensions] and
-   [I-D.psenak-ospf-segment-routing-ospfv3-extension].  The distinction
-   between local and global significance of the Adj-SID is given in the
-   encoding of the Adj-SID advertisement.
-
-5.2.  Segment Routing Extension Header (SRH)
-
-   A new type of the Routing Header (originally defined in [RFC2460]) is
-   defined: the Segment Routing Header (SRH) which has a new Routing
-   Type, (suggested value 4) to be assigned by IANA.
-
-   As an example, if an explicit path is to be constructed across a core
-   network running ISIS or OSPF, the segment list will contain SIDs
-   representing the nodes across the path (loose or strict) which,
-   usually, are the IPv6 loopback interface address of each node.  If
-   the path is across service or application entities, the segment list
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 11]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   contains the IPv6 addresses of these services or application
-   instances.
-
-   The Segment Routing Header (SRH) is defined as follows:
-
-
-     0                   1                   2                   3
-     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    | Next Header   |  Hdr Ext Len  | Routing Type  | Segments Left |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    | First Segment |             Flags             |  HMAC Key ID  |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |            Segment List[0] (128 bits ipv6 address)            |
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |                                                               |
-                                  ...
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |            Segment List[n] (128 bits ipv6 address)            |
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |            Policy List[0] (optional)                          |
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |            Policy List[1] (optional)                          |
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |            Policy List[2] (optional)                          |
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                                                               |
-    |                                                               |
-    |                                                               |
-    |                       HMAC (256 bits)                         |
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 12]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-    |                        (optional)                             |
-    |                                                               |
-    |                                                               |
-    |                                                               |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-   where:
-
-   o  Next Header: 8-bit selector.  Identifies the type of header
-      immediately following the SRH.
-
-   o  Hdr Ext Len: 8-bit unsigned integer, is the length of the SRH
-      header in 8-octet units, not including the first 8 octets.
-
-   o  Routing Type: TBD, to be assigned by IANA (suggested value: 4).
-
-   o  Segments Left.  Defined in [RFC2460], it contains the index, in
-      the Segment List, of the next segment to inspect.  Segments Left
-      is decremented at each segment and it is used as an index in the
-      segment list.
-
-   o  First Segment: offset in the SRH, not including the first 8 octets
-      and expressed in 16-octet units, pointing to the last element of
-      the segment list, which is in fact the first segment of the
-      segment routing path.
-
-   o  Flags: 16 bits of flags.  Following flags are defined:
-
-                              1
-          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-         |C|P|R|R|    Policy Flags       |
-         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-         C-flag: Clean-up flag.  Set when the SRH has to be removed from
-         the packet when packet reaches the last segment.
-
-         P-flag: Protected flag.  Set when the packet has been rerouted
-         through FRR mechanism by a SR endpoint node.  See Section 6.3
-         for more details.
-
-         R-flags.  Reserved and for future use.
-
-         Policy Flags.  Define the type of the IPv6 addresses encoded
-         into the Policy List (see below).  The following have been
-         defined:
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 13]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-            Bits 4-6: determine the type of the first element after the
-            segment list.
-
-            Bits 7-9: determine the type of the second element.
-
-            Bits 10-12: determine the type of the third element.
-
-            Bits 13-15: determine the type of the fourth element.
-
-         The following values are used for the type:
-
-            0x0: Not present.  If value is set to 0x0, it means the
-            element represented by these bits is not present.
-
-            0x1: SR Ingress.
-
-            0x2: SR Egress.
-
-            0x3: Original Source Address.
-
-   o  HMAC Key ID and HMAC field, and their use are defined in
-      [I-D.vyncke-6man-segment-routing-security].
-
-   o  Segment List[n]: 128 bit IPv6 addresses representing the nth
-      segment in the Segment List.  The Segment List is encoded starting
-      from the last segment of the path.  I.e., the first element of the
-      segment list (Segment List [0]) contains the last segment of the
-      path while the last segment of the Segment List (Segment List[n])
-      contains the first segment of the path.  The index contained in
-      "Segments Left" identifies the current active segment.
-
-   o  Policy List.  Optional addresses representing specific nodes in
-      the SR path such as:
-
-         SR Ingress: a 128 bit generic identifier representing the
-         ingress in the SR domain (i.e.: it needs not to be a valid IPv6
-         address).
-
-         SR Egress: a 128 bit generic identifier representing the egress
-         in the SR domain (i.e.: it needs not to be a valid IPv6
-         address).
-
-         Original Source Address: IPv6 address originally present in the
-         SA field of the packet.
-
-      The segments in the Policy List are encoded after the segment list
-      and they are optional.  If none are in the SRH, all bits of the
-      Policy List Flags MUST be set to 0x0.
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 14]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-5.2.1.  SRH and RFC2460 behavior
-
-   The SRH being a new type of the Routing Header, it also has the same
-   properties:
-
-      SHOULD only appear once in the packet.
-
-      Only the router whose address is in the DA field of the packet
-      header MUST inspect the SRH.
-
-   Therefore, Segment Routing in IPv6 networks implies that the segment
-   identifier (i.e.: the IPv6 address of the segment) is moved into the
-   DA of the packet.
-
-   The DA of the packet changes at each segment termination/completion
-   and therefore the original DA of the packet MUST be encoded as the
-   last segment of the path.
-
-   As illustrated in Section 3.2, nodes that are within the path of a
-   segment will forward packets based on the DA of the packet without
-   inspecting the SRH.  This ensures full interoperability between SR-
-   capable and non-SR-capable nodes.
-
-6.  SRH Procedures
-
-   In this section we describe the different procedures on the SRH.
-
-6.1.  Segment Routing Operations
-
-   When Segment Routing is instantiated over the IPv6 data plane the
-   following applies:
-
-   o  The segment list is encoded in the SRH.
-
-   o  The active segment is in the destination address of the packet.
-
-   o  The Segment Routing CONTINUE operation (as described in
-      [I-D.filsfils-spring-segment-routing]) is implemented as a
-      regular/plain IPv6 operation consisting of DA based forwarding.
-
-   o  The NEXT operation is implemented through the update of the DA
-      with the value represented by the Next Segment field in the SRH.
-
-   o  The PUSH operation is implemented through the insertion of the SRH
-      or the insertion of additional segments in the SRH segment list.
-
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 15]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-6.2.  Segment Routing Node Functions
-
-   SR packets are forwarded to segments endpoints (i.e.: nodes whose
-   address is in the DA field of the packet).  The segment endpoint,
-   when receiving a SR packet destined to itself, does:
-
-   o  Inspect the SRH.
-
-   o  Determine the next active segment.
-
-   o  Update the Segments Left field (or, if requested, remove the SRH
-      from the packet).
-
-   o  Update the DA.
-
-   o  Send the packet to the next segment.
-
-   The procedures applied to the SRH are related to the node function.
-   Following nodes functions are defined:
-
-      Ingress SR Node.
-
-      Transit Non-SR Node.
-
-      Transit SR Intra Segment Node.
-
-      SR Endpoint Node.
-
-6.2.1.  Ingress SR Node
-
-   Ingress Node can be a router at the edge of the SR domain or a SR-
-   capable host.  The ingress SR node may obtain the segment list by
-   either:
-
-      Local path computation.
-
-      Local configuration.
-
-      Interaction with an SDN controller delivering the path as a
-      complete SRH.
-
-      Any other mechanism (mechanisms through which the path is acquired
-      are outside the scope of this document).
-
-   When creating the SRH (either at ingress node or in the SDN
-   controller) the following is done:
-
-      Next Header and Hdr Ext Len fields are set according to [RFC2460].
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 16]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-      Routing Type field is set as TBD (SRH).
-
-      The Segment List is built with the FIRST segment of the path
-      encoded in the LAST element of the Segment List.  Subsequent
-      segments are encoded on top of the first segment.  Finally, the
-      LAST segment of the path is encoded in the FIRST element of the
-      Segment List.  In other words, the Segment List is encoded in the
-      reverse order of the path.
-
-      The original DA of the packet is encoded as the last segment of
-      the path (encoded in the first element of the Segment List).
-
-      the DA of the packet is set with the value of the first segment
-      (found in the last element of the segment list).
-
-      the Segments Left field is set to n-1 where n is the number of
-      elements in the Segment List.
-
-      The packet is sent out towards the first segment (i.e.:
-      represented in the packet DA).
-
-6.2.1.1.  Security at Ingress
-
-   The procedures related to the Segment Routing security are detailed
-   in [I-D.vyncke-6man-segment-routing-security].
-
-   In the case where the SR domain boundaries are not under control of
-   the network operator (e.g.: when the SR domain edge is in a home
-   network), it is important to authenticate and validate the content of
-   any SRH being received by the network operator.  In such case, the
-   security procedure described in
-   [I-D.vyncke-6man-segment-routing-security] is to be used.
-
-   The ingress node (e.g.: the host in the home network) requests the
-   SRH from a control system (e.g.: an SDN controller) which delivers
-   the SRH with its HMAC signature on it.
-
-   Then, the home network host can send out SR packets (with an SRH on
-   it) that will be validated at the ingress of the network operator
-   infrastructure.
-
-   The ingress node of the network operator infrastructure, is
-   configured in order to validate the incoming SRH HMACs in order to
-   allow only packets having correct SRH according to their SA/DA
-   addresses.
-
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 17]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-6.2.2.  Transit Non-SR Capable Node
-
-   SR is interoperable with plain IPv6 forwarding.  Any non SR-capable
-   node will forward SR packets solely based on the DA.  There's no SRH
-   inspection.  This ensures full interoperability between SR and non-SR
-   nodes.
-
-6.2.3.  SR Intra Segment Transit Node
-
-   Only the node whose address is in DA inspects and processes the SRH
-   (according to [RFC2460]).  An intra segment transit node is not in
-   the DA and its forwarding is based on DA and its SR-IPv6 FIB.
-
-6.2.4.  SR Segment Endpoint Node
-
-   The SR segment endpoint node is the node whose address is in the DA.
-   The segment endpoint node inspects the SRH and does:
-
-   1.   IF DA = myself (segment endpoint)
-   2.      IF Segments Left > 0 THEN
-              decrement Segments Left
-              update DA with Segment List[Segments Left]
-   3.      ELSE IF Segments List[Segments Left] <> DA THEN
-              update DA with Segments List[Segments Left]
-              IF Clean-up bit is set THEN remove the SRH
-   4.      ELSE give the packet to next PID (application)
-                End of processing.
-   5.   Forward the packet out
-
-6.3.  FRR Flag Settings
-
-   A node supporting SR and doing Fast Reroute (as described in
-   [I-D.filsfils-spring-segment-routing-use-cases], when rerouting
-   packets through FRR mechanisms, SHOULD inspect the rerouted packet
-   header and look for the SRH.  If the SRH is present, the rerouting
-   node SHOULD set the Protected bit on all rerouted packets.
-
-7.  SR and Tunneling
-
-   Encapsulation can be realized in two different ways with SR-IPv6:
-
-      Outer encapsulation.
-
-      SRH with SA/DA original addresses.
-
-   Outer encapsulation tunneling is the traditional method where an
-   additional IPv6 header is prepended to the packet.  The original IPv6
-   header being encapsulated, everything is preserved and the packet is
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 18]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   switched/routed according to the outer header (that could contain a
-   SRH).
-
-   SRH allows encoding both original SA and DA, hence an operator may
-   decide to change the SA/DA at ingress and restore them at egress.
-   This can be achieved without outer encapsulation, by changing SA/DA
-   and encoding the original SA in the Policy List and in the original
-   DA in the Segment List.
-
-8.  Example Use Case
-
-   A more detailed description of use cases are available in
-   [I-D.ietf-spring-ipv6-use-cases].  In this section, a simple SR-IPv6
-   example is illustrated.
-
-   In the topology described in Figure 6 it is assumed an end-to-end SR
-   deployment.  Therefore SR is supported by all nodes from A to J.
-
-    Home Network |          Backbone         |    Datacenter
-                 |                           |
-                 |   +---+   +---+   +---+   |   +---+   |
-             +---|---| C |---| D |---| E |---|---| I |---|
-             |   |   +---+   +---+   +---+   |   +---+   |
-             |   |     |       |       |     |     |     |  +---+
-   +---+   +---+ |     |       |       |     |     |     |--| X |
-   | A |---| B | |   +---+   +---+   +---+   |   +---+   |  +---+
-   +---+   +---+ |   | F |---| G |---| H |---|---| J |---|
-                 |   +---+   +---+   +---+   |   +---+   |
-                 |                           |
-                 |        +-----------+
-                          |    SDN    |
-                          | Orch/Ctlr |
-                          +-----------+
-
-                       Figure 6: Sample SR topology
-
-   The following workflow applies to packets sent by host A and destined
-   to server X.
-
-
-
-
-
-
-
-
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 19]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   . Host A sends a request for a path to server X to the SDN
-     controller or orchestration system.
-
-   . The SDN controller/orchestrator builds a SRH with:
-      . Segment List: C, F, J, X
-      . HMAC
-     that satisfies the requirements expressed in the request
-     by host A and based on policies applicable to host A.
-
-   . Host A receives the SRH and insert it into the packet.
-     The packet has now:
-      . SA: A
-      . DA: C
-      . SRH with
-         . SL: X, J, F, C
-         . Segments Left: 3 (i.e.: Segment List size - 1)
-         . PL: C (ingress), J (egress)
-        Note that X is the last segment and C is the
-        first segment (i.e.: the SL is encoded in the reverse
-        path order).
-      . HMAC
-
-   . When packet arrives in C (first segment), C does:
-      . Validate the HMAC of the SRH.
-      . Decrement Segments Left by one: 2
-      . Update the DA with the next segment found in
-        Segment List[2]. DA is set to F.
-      . Forward the packet to F.
-
-   . When packet arrives in F (second segment), F does:
-      . Decrement Segments Left by one: 1
-      . Update the DA with the next segment found in
-        Segment List[1]. DA is set to J.
-      . Forward the packet to J.
-
-   . Packet travels across G and H nodes which do plain
-     IPv6 forwarding based on DA. No inspection of SRH needs
-     to be done in these nodes. However, any SR capable node
-     is allowed to set the Protected bit in case of FRR
-     protection.
-
-   . When packet arrives in J (third segment), J does:
-      . Decrement Segments Left by one: 0
-      . Update the DA with the next segment found in
-        Segment List[0]. DA is set to X.
-      . If the cleanup bit is set, then node J will strip out
-        the SRH from the packet.
-      . Forward the packet to X.
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 20]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   The packet arrives in the server that may or may not support SR.  The
-   return traffic, from server to host, may be sent using the same
-   procedures.
-
-9.  IANA Considerations
-
-   TBD
-
-10.  Manageability Considerations
-
-   TBD
-
-11.  Security Considerations
-
-   Security mechanisms applied to Segment Routing over IPv6 networks are
-   detailed in [I-D.vyncke-6man-segment-routing-security].
-
-12.  Contributors
-
-   The authors would like to thank Dave Barach, John Leddy, John
-   Brzozowski, Pierre Francois, Nagendra Kumar, Mark Townsley, Christian
-   Martin, Roberta Maglione, Eric Vyncke, James Connolly, David Lebrun
-   and Fred Baker for their contribution to this document.
-
-13.  Acknowledgements
-
-   TBD
-
-14.  References
-
-14.1.  Normative References
-
-   [RFC2119]  Bradner, S., "Key words for use in RFCs to Indicate
-              Requirement Levels", BCP 14, RFC 2119, March 1997.
-
-   [RFC2460]  Deering, S. and R. Hinden, "Internet Protocol, Version 6
-              (IPv6) Specification", RFC 2460, December 1998.
-
-14.2.  Informative References
-
-   [I-D.filsfils-spring-segment-routing]
-              Filsfils, C., Previdi, S., Bashandy, A., Decraene, B.,
-              Litkowski, S., Horneffer, M., Milojevic, I., Shakir, R.,
-              Ytti, S., Henderickx, W., Tantsura, J., and E. Crabbe,
-              "Segment Routing Architecture", draft-filsfils-spring-
-              segment-routing-04 (work in progress), July 2014.
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 21]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   [I-D.filsfils-spring-segment-routing-mpls]
-              Filsfils, C., Previdi, S., Bashandy, A., Decraene, B.,
-              Litkowski, S., Horneffer, M., Milojevic, I., Shakir, R.,
-              Ytti, S., Henderickx, W., Tantsura, J., and E. Crabbe,
-              "Segment Routing with MPLS data plane", draft-filsfils-
-              spring-segment-routing-mpls-03 (work in progress), August
-              2014.
-
-   [I-D.filsfils-spring-segment-routing-use-cases]
-              Filsfils, C., Francois, P., Previdi, S., Decraene, B.,
-              Litkowski, S., Horneffer, M., Milojevic, I., Shakir, R.,
-              Ytti, S., Henderickx, W., Tantsura, J., Kini, S., and E.
-              Crabbe, "Segment Routing Use Cases", draft-filsfils-
-              spring-segment-routing-use-cases-01 (work in progress),
-              October 2014.
-
-   [I-D.ietf-isis-segment-routing-extensions]
-              Previdi, S., Filsfils, C., Bashandy, A., Gredler, H.,
-              Litkowski, S., Decraene, B., and J. Tantsura, "IS-IS
-              Extensions for Segment Routing", draft-ietf-isis-segment-
-              routing-extensions-03 (work in progress), October 2014.
-
-   [I-D.ietf-spring-ipv6-use-cases]
-              Brzozowski, J., Leddy, J., Leung, I., Previdi, S.,
-              Townsley, W., Martin, C., Filsfils, C., and R. Maglione,
-              "IPv6 SPRING Use Cases", draft-ietf-spring-ipv6-use-
-              cases-03 (work in progress), November 2014.
-
-   [I-D.psenak-ospf-segment-routing-ospfv3-extension]
-              Psenak, P., Previdi, S., Filsfils, C., Gredler, H.,
-              Shakir, R., Henderickx, W., and J. Tantsura, "OSPFv3
-              Extensions for Segment Routing", draft-psenak-ospf-
-              segment-routing-ospfv3-extension-02 (work in progress),
-              July 2014.
-
-   [I-D.vyncke-6man-segment-routing-security]
-              Vyncke, E. and S. Previdi, "IPv6 Segment Routing Header
-              (SRH) Security Considerations", July 2014.
-
-   [RFC1940]  Estrin, D., Li, T., Rekhter, Y., Varadhan, K., and D.
-              Zappala, "Source Demand Routing: Packet Format and
-              Forwarding Specification (Version 1)", RFC 1940, May 1996.
-
-Authors' Addresses
-
-
-
-
-
-
-
-Previdi, et al.           Expires June 12, 2015                [Page 22]
-
-Internet-Draft      IPv6 Segment Routing Header (SRH)      December 2014
-
-
-   Stefano Previdi (editor)
-   Cisco Systems, Inc.
-   Via Del Serafico, 200
-   Rome  00142
-   Italy
-
-   Email: sprevidi@cisco.com
-
-
-   Clarence Filsfils
-   Cisco Systems, Inc.
-   Brussels
-   BE
-
-   Email: cfilsfil@cisco.com
-
-
-   Brian Field
-   Comcast
-   4100 East Dry Creek Road
-   Centennial, CO  80122
-   US
-
-   Email: Brian_Field@cable.comcast.com
-
-
-   Ida Leung
-   Rogers Communications
-   8200 Dixie Road
-   Brampton, ON  L6T 0C1
-   CA
-
-   Email: Ida.Leung@rci.rogers.com
diff --git a/src/vnet/sr/sr.api b/src/vnet/sr/sr.api
index 3d017ce5..5feadcb0 100644
--- a/src/vnet/sr/sr.api
+++ b/src/vnet/sr/sr.api
@@ -12,108 +12,207 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
- 
-/** \brief IPv6 segment routing tunnel add / del request
+
+/** \brief IPv6 SR LocalSID add/del request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_del Boolean of whether its a delete instruction
+    @param localsid_addr IPv6 address of the localsid
+    @param end_psp Boolean of whether decapsulation is allowed in this function
+    @param behavior Type of behavior (function) for this localsid
+    @param sw_if_index Only for L2/L3 xconnect. OIF. In VRF variant the fib_table.
+    @param vlan_index Only for L2 xconnect. Outgoing VLAN tag.
+    @param fib_table  FIB table in which we should install the localsid entry
+    @param nh_addr Next Hop IPv4/IPv6 address. Only for L2/L3 xconnect.
+*/
+define sr_localsid_add_del
+{
+  u32 client_index;
+  u32 context;
+  u8 is_del;
+  u8 localsid_addr[16];
+  u8 end_psp;
+  u8 behavior;
+  u32 sw_if_index;
+  u32 vlan_index;
+  u32 fib_table;
+  u8 nh_addr[16];
+};
+
+/** \brief IPv6 SR LocalSID add/del request response
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+*/
+define sr_localsid_add_del_reply
+{
+  u32 context;
+  i32 retval;
+};
+
+/** \brief IPv6 SR policy add
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param bsid is the bindingSID of the SR Policy
+    @param weight is the weight of the sid list. optional.
+    @param is_encap is the behavior of the SR policy. (0.SRH insert // 1.Encapsulation)
+    @param type is the type of the SR policy. (0.Default // 1.Spray)
+    @param fib_table is the VRF where to install the FIB entry for the BSID
+    @param segments is a vector of IPv6 address composing the segment list
+*/
+define sr_policy_add
+{
+  u32 client_index;
+  u32 context;
+  u8 bsid_addr[16];
+  u32 weight;
+  u8 is_encap;
+  u8 type;
+  u32 fib_table;
+  u8 n_segments;
+  u8 segments[0];
+};
+
+/** \brief IPv6 SR Policy add request response
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+*/
+define sr_policy_add_reply
+{
+  u32 context;
+  i32 retval;
+};
+
+/** \brief IPv6 SR policy modification
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
-    @param is_add - add the tunnel if non-zero, else delete it
-    @param name[] - tunnel name (len. 64)
-    @param src_address[] -
-    @param dst_address[] -
-    @param dst_mask_width -
-    @param inner_vrf_id -
-    @param outer_vrf_id -
-    @param flags_net_byte_order -
-    @param n_segments -
-    @param n_tags -
-    @param segs_and_tags[] -
-    @param policy_name[] - name of policy to associate this tunnel to (len. 64)
+    @param bsid is the bindingSID of the SR Policy
+    @param sr_policy_index is the index of the SR policy
+    @param fib_table is the VRF where to install the FIB entry for the BSID
+    @param operation is the operation to perform (among the top ones)
+    @param segments is a vector of IPv6 address composing the segment list
+    @param sl_index is the index of the Segment List to modify/delete
+    @param weight is the weight of the sid list. optional.
+    @param is_encap Mode. Encapsulation or SRH insertion.
 */
-define sr_tunnel_add_del
+define sr_policy_mod
 {
   u32 client_index;
   u32 context;
-  u8 is_add;
-  u8 name[64];
-  u8 src_address[16];
-  u8 dst_address[16];
-  u8 dst_mask_width;
-  u32 inner_vrf_id;
-  u32 outer_vrf_id;
-  u16 flags_net_byte_order;
+  u8 bsid_addr[16];
+  u32 sr_policy_index;
+  u32 fib_table;
+  u8 operation;
+  u32 sl_index;
+  u32 weight;
   u8 n_segments;
-  u8 n_tags;
-  u8 policy_name[64];
-  u8 segs_and_tags[0];
+  u8 segments[0];
 };
 
-/** \brief IPv6 segment routing tunnel add / del response
+/** \brief IPv6 SR Policy modification request response
     @param context - sender context, to match reply w/ request
     @param retval - return value for request
 */
-define sr_tunnel_add_del_reply
+define sr_policy_mod_reply
 {
   u32 context;
   i32 retval;
 };
 
-/** \brief IPv6 segment routing policy add / del request
+/** \brief IPv6 SR policy deletion
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
-    @param is_add - add the tunnel if non-zero, else delete it
-    @param name[] - policy name (len. 64)
-    @param tunnel_names[] -
+    @param bsid is the bindingSID of the SR Policy
+    @param index is the index of the SR policy
 */
-define sr_policy_add_del
+define sr_policy_del
 {
   u32 client_index;
   u32 context;
-  u8 is_add;
-  u8 name[64];
-  u8 tunnel_names[0];
+  u8 bsid_addr[16];
+  u32 sr_policy_index;
 };
 
-/** \brief IPv6 segment routing policy add / del response
+/** \brief IPv6 SR Policy deletion request response
     @param context - sender context, to match reply w/ request
-    @param retval - return value for request 
- 
-    
+    @param retval - return value for request
 */
-define sr_policy_add_del_reply
+define sr_policy_del_reply
 {
   u32 context;
   i32 retval;
 };
 
-/** \brief IPv6 segment routing multicast map to policy add / del request
+/** \brief IPv6 SR steering add/del
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
-    @param is_add - add the tunnel if non-zero, else delete it
-    @param multicast_address[] - IP6 multicast address
-    @param policy_name[] = policy name (len.64)
+    @param is_del
+    @param bsid is the bindingSID of the SR Policy (alt to sr_policy_index)
+    @param sr_policy is the index of the SR Policy (alt to bsid)
+    @param table_id is the VRF where to install the FIB entry for the BSID
+    @param prefix is the IPv4/v6 address for L3 traffic type
+    @param mask_width is the mask for L3 traffic type
+    @param sw_if_index is the incoming interface for L2 traffic
+    @param traffic_type describes the type of traffic
 */
-define sr_multicast_map_add_del
+define sr_steering_add_del
 {
   u32 client_index;
   u32 context;
-  u8 is_add;
-  u8 multicast_address[16];
-  u8 policy_name[64];
+  u8 is_del;
+  u8 bsid_addr[16];
+  u32 sr_policy_index;
+  u32 table_id;
+  u8 prefix_addr[16];
+  u32 mask_width;
+  u32 sw_if_index;
+  u8 traffic_type;
 };
 
-/** \brief IPv6 segment routing multicast map to policy add / del response
+/** \brief IPv6 SR steering add/del request response
     @param context - sender context, to match reply w/ request
     @param retval - return value for request
 */
-define sr_multicast_map_add_del_reply
+define sr_steering_add_del_reply
 {
   u32 context;
   i32 retval;
 };
 
+/** \brief Dump the list of SR LocalSIDs
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+/**define sr_localsids_dump
+{
+  u32 client_index;
+  u32 context;
+};*/
+
+/** \brief Details about a single SR LocalSID
+    @param context - returned sender context, to match reply w/ request
+    @param localsid_addr IPv6 address of the localsid
+    @param behavior Type of behavior (function) for this localsid
+    @param end_psp Boolean of whether decapsulation is allowed in this function
+    @param sw_if_index Only for L2/L3 xconnect. OIF. In VRF variant the fib_table.
+    @param vlan_index Only for L2 xconnect. Outgoing VLAN tag.
+    @param fib_table  FIB table in which we should install the localsid entry
+    @param nh_addr Next Hop IPv4/IPv6 address. Only for L2/L3 xconnect.
+*/
+/**manual_endian define sr_localsid_details
+{
+  u32 context;
+  u8 localsid_addr[16];
+  u8 behavior;
+  u8 end_psp;
+  u32 sw_if_index;
+  u32 vlan_index;
+  u32 fib_table;
+  u8 nh_addr[16];
+};*/
+
 /*
+ * fd.io coding-style-patch-verification: ON
  * Local Variables:
  * eval: (c-set-style "gnu")
  * End:
  */
- 
\ No newline at end of file
diff --git a/src/vnet/sr/sr.c b/src/vnet/sr/sr.c
old mode 100644
new mode 100755
index 012d4542..34344fce
--- a/src/vnet/sr/sr.c
+++ b/src/vnet/sr/sr.c
@@ -17,9 +17,10 @@
 
 /**
  * @file
- * @brief Segment Routing main functions
+ * @brief Segment Routing initialization
  *
  */
+
 #include 
 #include 
 #include 
@@ -27,1179 +28,13 @@
 #include 
 #include 
 
-#include 
-
 ip6_sr_main_t sr_main;
-static vlib_node_registration_t sr_local_node;
-
-/**
- * @brief Dynamically added SR DPO type
- */
-static dpo_type_t sr_dpo_type;
-
-/**
- * @brief Dynamically added SR FIB Node type
- */
-static fib_node_type_t sr_fib_node_type;
-
-/**
- * @brief Use passed HMAC key in ip6_sr_header_t in OpenSSL HMAC routines
- *
- * @param sm ip6_sr_main_t *
- * @param ip ip6_header_t *
- * @param sr ip6_sr_header_t *
- */
-void
-sr_fix_hmac (ip6_sr_main_t * sm, ip6_header_t * ip, ip6_sr_header_t * sr)
-{
-  u32 key_index;
-  static u8 *keybuf;
-  u8 *copy_target;
-  int first_segment;
-  ip6_address_t *addrp;
-  int i;
-  ip6_sr_hmac_key_t *hmac_key;
-  u32 sig_len;
-
-  key_index = sr->hmac_key;
-
-  /* No signature? Pass... */
-  if (key_index == 0)
-    return;
-
-  /* We don't know about this key? Fail... */
-  if (key_index >= vec_len (sm->hmac_keys))
-    return;
-
-  hmac_key = sm->hmac_keys + key_index;
-
-  vec_reset_length (keybuf);
-
-  /* pkt ip6 src address */
-  vec_add2 (keybuf, copy_target, sizeof (ip6_address_t));
-  clib_memcpy (copy_target, ip->src_address.as_u8, sizeof (ip6_address_t));
-
-  /* first segment */
-  vec_add2 (keybuf, copy_target, 1);
-  copy_target[0] = sr->first_segment;
-
-  /* octet w/ bit 0 = "clean" flag */
-  vec_add2 (keybuf, copy_target, 1);
-  copy_target[0]
-    = (sr->flags & clib_host_to_net_u16 (IP6_SR_HEADER_FLAG_CLEANUP))
-    ? 0x80 : 0;
-
-  /* hmac key id */
-  vec_add2 (keybuf, copy_target, 1);
-  copy_target[0] = sr->hmac_key;
-
-  first_segment = sr->first_segment;
-
-  addrp = sr->segments;
-
-  /* segments */
-  for (i = 0; i <= first_segment; i++)
-    {
-      vec_add2 (keybuf, copy_target, sizeof (ip6_address_t));
-      clib_memcpy (copy_target, addrp->as_u8, sizeof (ip6_address_t));
-      addrp++;
-    }
-
-  addrp++;
-
-  HMAC_CTX_init (sm->hmac_ctx);
-  if (!HMAC_Init (sm->hmac_ctx, hmac_key->shared_secret,
-		  vec_len (hmac_key->shared_secret), sm->md))
-    clib_warning ("barf1");
-  if (!HMAC_Update (sm->hmac_ctx, keybuf, vec_len (keybuf)))
-    clib_warning ("barf2");
-  if (!HMAC_Final (sm->hmac_ctx, (unsigned char *) addrp, &sig_len))
-    clib_warning ("barf3");
-  HMAC_CTX_cleanup (sm->hmac_ctx);
-}
-
-/**
- * @brief Format function for decoding various SR flags
- *
- * @param s u8 * - formatted string
- * @param args va_list * - u16 flags
- *
- * @return formatted output string u8 *
- */
-u8 *
-format_ip6_sr_header_flags (u8 * s, va_list * args)
-{
-  u16 flags = (u16) va_arg (*args, int);
-  u8 pl_flag;
-  int bswap_needed = va_arg (*args, int);
-  int i;
-
-  if (bswap_needed)
-    flags = clib_host_to_net_u16 (flags);
-
-  if (flags & IP6_SR_HEADER_FLAG_CLEANUP)
-    s = format (s, "cleanup ");
-
-  if (flags & IP6_SR_HEADER_FLAG_PROTECTED)
-    s = format (s, "reroute ");
-
-  s = format (s, "pl: ");
-  for (i = 1; i <= 4; i++)
-    {
-      pl_flag = ip6_sr_policy_list_flags (flags, i);
-      s = format (s, "[%d] ", i);
-
-      switch (pl_flag)
-	{
-	case IP6_SR_HEADER_FLAG_PL_ELT_NOT_PRESENT:
-	  s = format (s, "NotPr ");
-	  break;
-	case IP6_SR_HEADER_FLAG_PL_ELT_INGRESS_PE:
-	  s = format (s, "InPE ");
-	  break;
-	case IP6_SR_HEADER_FLAG_PL_ELT_EGRESS_PE:
-	  s = format (s, "EgPE ");
-	  break;
-
-	case IP6_SR_HEADER_FLAG_PL_ELT_ORIG_SRC_ADDR:
-	  s = format (s, "OrgSrc ");
-	  break;
-	}
-    }
-  return s;
-}
-
-/**
- * @brief Format function for decoding ip6_sr_header_t
- *
- * @param s u8 * - formatted string
- * @param args va_list * - ip6_sr_header_t
- *
- * @return formatted output string u8 *
- */
-u8 *
-format_ip6_sr_header (u8 * s, va_list * args)
-{
-  ip6_sr_header_t *h = va_arg (*args, ip6_sr_header_t *);
-  ip6_address_t placeholder_addr =
-    { {254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
-       254, 254}
-  };
-  int print_hmac = va_arg (*args, int);
-  int i, pl_index, max_segs;
-  int flags_host_byte_order = clib_net_to_host_u16 (h->flags);
-
-  s = format (s, "next proto %d, len %d, type %d",
-	      h->protocol, (h->length << 3) + 8, h->type);
-  s = format (s, "\n      segs left %d, first_segment %d, hmac key %d",
-	      h->segments_left, h->first_segment, h->hmac_key);
-  s = format (s, "\n      flags %U", format_ip6_sr_header_flags,
-	      flags_host_byte_order, 0 /* bswap needed */ );
-
-  /*
-   * Header length is in 8-byte units (minus one), so
-   * divide by 2 to ascertain the number of ip6 addresses in the
-   * segment list
-   */
-  max_segs = (h->length >> 1);
-
-  if (!print_hmac && h->hmac_key)
-    max_segs -= 2;
-
-  s = format (s, "\n  Segments (in processing order):");
-
-  for (i = h->first_segment; i >= 1; i--)
-    s = format (s, "\n  %U", format_ip6_address, h->segments + i);
-  if (ip6_address_is_equal (&placeholder_addr, h->segments))
-    s = format (s, "\n  (empty placeholder)");
-  else
-    s = format (s, "\n  %U", format_ip6_address, h->segments);
-
-  s = format (s, "\n  Policy List:");
-
-  pl_index = 1;			/* to match the RFC text */
-  for (i = (h->first_segment + 1); i < max_segs; i++, pl_index++)
-    {
-      char *tag;
-      char *tags[] = { " ", "InPE: ", "EgPE: ", "OrgSrc: " };
-
-      tag = tags[0];
-      if (pl_index >= 1 && pl_index <= 4)
-	{
-	  int this_pl_flag = ip6_sr_policy_list_flags
-	    (flags_host_byte_order, pl_index);
-	  tag = tags[this_pl_flag];
-	}
-
-      s = format (s, "\n  %s%U", tag, format_ip6_address, h->segments + i);
-    }
-
-  return s;
-}
-
-/**
- * @brief Format function for decoding ip6_sr_header_t with length
- *
- * @param s u8 * - formatted string
- * @param args va_list * - ip6_header_t + ip6_sr_header_t
- *
- * @return formatted output string u8 *
- */
-u8 *
-format_ip6_sr_header_with_length (u8 * s, va_list * args)
-{
-  ip6_header_t *h = va_arg (*args, ip6_header_t *);
-  u32 max_header_bytes = va_arg (*args, u32);
-  uword header_bytes;
-
-  header_bytes = sizeof (h[0]) + sizeof (ip6_sr_header_t);
-  if (max_header_bytes != 0 && header_bytes > max_header_bytes)
-    return format (s, "ip6_sr header truncated");
-
-  s = format (s, "IP6: %U\n", format_ip6_header, h, max_header_bytes);
-  s =
-    format (s, "SR: %U\n", format_ip6_sr_header, (ip6_sr_header_t *) (h + 1),
-	    0 /* print_hmac */ , max_header_bytes);
-  return s;
-}
-
-/**
- * @brief Defined valid next nodes
-*/
-#define foreach_sr_rewrite_next                 \
-_(ERROR, "error-drop")                          \
-_(SR_LOCAL, "sr-local")
-
-/**
- * @brief Struct for defined valid next nodes
-*/
-typedef enum
-{
-#define _(s,n) SR_REWRITE_NEXT_##s,
-  foreach_sr_rewrite_next
-#undef _
-    SR_REWRITE_N_NEXT,
-} sr_rewrite_next_t;
-
-/**
- * @brief Struct for data for SR rewrite packet trace
- */
-typedef struct
-{
-  ip6_address_t src, dst;
-  u16 length;
-  u32 next_index;
-  u32 tunnel_index;
-  u8 sr[256];
-} sr_rewrite_trace_t;
-
-/**
- * @brief Error strings for SR rewrite
- */
-static char *sr_rewrite_error_strings[] = {
-#define sr_error(n,s) s,
-#include "sr_error.def"
-#undef sr_error
-};
-
-/**
- * @brief Struct for SR rewrite error strings
- */
-typedef enum
-{
-#define sr_error(n,s) SR_REWRITE_ERROR_##n,
-#include "sr_error.def"
-#undef sr_error
-  SR_REWRITE_N_ERROR,
-} sr_rewrite_error_t;
-
-
-/**
- * @brief Format function for SR rewrite trace.
- */
-u8 *
-format_sr_rewrite_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  sr_rewrite_trace_t *t = va_arg (*args, sr_rewrite_trace_t *);
-  ip6_sr_main_t *sm = &sr_main;
-  ip6_sr_tunnel_t *tun = pool_elt_at_index (sm->tunnels, t->tunnel_index);
-  ip6_fib_t *rx_fib, *tx_fib;
-
-  rx_fib = ip6_fib_get (tun->rx_fib_index);
-  tx_fib = ip6_fib_get (tun->tx_fib_index);
-
-  s = format
-    (s, "SR-REWRITE: next %s ip6 src %U dst %U len %u\n"
-     "           rx-fib-id %d tx-fib-id %d\n%U",
-     (t->next_index == SR_REWRITE_NEXT_SR_LOCAL)
-     ? "sr-local" : "ip6-lookup",
-     format_ip6_address, &t->src,
-     format_ip6_address, &t->dst, t->length,
-     rx_fib->table_id, tx_fib->table_id,
-     format_ip6_sr_header, t->sr, 0 /* print_hmac */ );
-  return s;
-}
-
-/**
- * @brief Main processing dual-loop for Segment Routing Rewrite
- * @node sr-rewrite
- *
- * @param vm vlib_main_t *
- * @param node vlib_node_runtime_t *
- * @param from_frame vlib_frame_t *
- *
- * @return from_frame->n_vectors uword
- */
-static uword
-sr_rewrite (vlib_main_t * vm,
-	    vlib_node_runtime_t * node, vlib_frame_t * from_frame)
-{
-  u32 n_left_from, next_index, *from, *to_next;
-  ip6_sr_main_t *sm = &sr_main;
-  u32 (*sr_local_cb) (vlib_main_t *, vlib_node_runtime_t *,
-		      vlib_buffer_t *, ip6_header_t *, ip6_sr_header_t *);
-  sr_local_cb = sm->sr_local_cb;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      /* Note 2x loop disabled */
-      while (0 && n_left_from >= 4 && n_left_to_next >= 2)
-	{
-	  u32 bi0, bi1;
-	  vlib_buffer_t *b0, *b1;
-	  ip6_header_t *ip0, *ip1;
-	  ip6_sr_header_t *sr0, *sr1;
-	  ip6_sr_tunnel_t *t0, *t1;
-	  u32 next0;
-	  u32 next1;
-	  u16 new_l0 = 0;
-	  u16 new_l1 = 0;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t *p2, *p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-	  }
-
-	  bi0 = from[0];
-	  bi1 = from[1];
-	  to_next[0] = bi0;
-	  to_next[1] = bi1;
-	  from += 2;
-	  to_next += 2;
-	  n_left_to_next -= 2;
-	  n_left_from -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-	  /*
-	   * $$$ parse through header(s) to pick the point
-	   * where we punch in the SR extention header
-	   */
-	  t0 =
-	    pool_elt_at_index (sm->tunnels,
-			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
-	  t1 =
-	    pool_elt_at_index (sm->tunnels,
-			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
-
-	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE
-		  >= ((word) vec_len (t0->rewrite)) + b0->current_data);
-	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE
-		  >= ((word) vec_len (t1->rewrite)) + b1->current_data);
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = t0->tx_fib_index;
-	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = t1->tx_fib_index;
-
-	  ip0 = vlib_buffer_get_current (b0);
-	  ip1 = vlib_buffer_get_current (b1);
-
-	  /*
-	   * SR-unaware service chaining case: pkt coming back from
-	   * service has the original dst address, and will already
-	   * have an SR header. If so, send it to sr-local
-	   */
-	  if (PREDICT_FALSE (ip0->protocol == IPPROTO_IPV6_ROUTE))
-	    {
-	      vlib_buffer_advance (b0, sizeof (ip0));
-	      sr0 = (ip6_sr_header_t *) (ip0 + 1);
-	      new_l0 = clib_net_to_host_u16 (ip0->payload_length);
-	      next0 = SR_REWRITE_NEXT_SR_LOCAL;
-	    }
-	  else
-	    {
-	      u32 len_bytes = sizeof (ip6_header_t);
-	      u8 next_hdr = ip0->protocol;
-
-	      /* HBH must immediately follow ipv6 header */
-	      if (PREDICT_FALSE
-		  (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-		{
-		  ip6_hop_by_hop_ext_t *ext_hdr =
-		    (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-		  len_bytes +=
-		    ip6_ext_header_len ((ip6_ext_header_t *) ext_hdr);
-		  /* Ignoring the sr_local for now, if RH follows HBH here */
-		  next_hdr = ext_hdr->next_hdr;
-		  ext_hdr->next_hdr = IPPROTO_IPV6_ROUTE;
-		}
-	      else
-		{
-		  ip0->protocol = IPPROTO_IPV6_ROUTE;	/* routing extension header */
-		}
-	      /*
-	       * Copy data before the punch-in point left by the
-	       * required amount. Assume (for the moment) that only
-	       * the main packet header needs to be copied.
-	       */
-	      clib_memcpy (((u8 *) ip0) - vec_len (t0->rewrite),
-			   ip0, len_bytes);
-	      vlib_buffer_advance (b0, -(word) vec_len (t0->rewrite));
-	      ip0 = vlib_buffer_get_current (b0);
-	      sr0 = (ip6_sr_header_t *) ((u8 *) ip0 + len_bytes);
-	      /* $$$ tune */
-	      clib_memcpy (sr0, t0->rewrite, vec_len (t0->rewrite));
-
-	      /* Fix the next header chain */
-	      sr0->protocol = next_hdr;
-
-	      new_l0 = clib_net_to_host_u16 (ip0->payload_length) +
-		vec_len (t0->rewrite);
-	      ip0->payload_length = clib_host_to_net_u16 (new_l0);
-
-	      /* Copy dst address into the DA slot in the segment list */
-	      clib_memcpy (sr0->segments, ip0->dst_address.as_u64,
-			   sizeof (ip6_address_t));
-	      /* Rewrite the ip6 dst address with the first hop */
-	      clib_memcpy (ip0->dst_address.as_u64, t0->first_hop.as_u64,
-			   sizeof (ip6_address_t));
-
-	      sr_fix_hmac (sm, ip0, sr0);
-
-	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
-		t0->first_hop_dpo.dpoi_index;
-	      next0 = t0->first_hop_dpo.dpoi_next_node;
-	      next0 = (sr_local_cb ?
-		       sr_local_cb (vm, node, b0, ip0, sr0) : next0);
-
-	      /*
-	       * Ignore "do not rewrite" shtik in this path
-	       */
-	      if (PREDICT_FALSE (next0 & 0x80000000))
-		{
-		  next0 ^= 0xFFFFFFFF;
-		  if (PREDICT_FALSE (next0 == SR_REWRITE_NEXT_ERROR))
-		    b0->error = node->errors[SR_REWRITE_ERROR_APP_CALLBACK];
-		}
-	    }
-
-	  if (PREDICT_FALSE (ip1->protocol == IPPROTO_IPV6_ROUTE))
-	    {
-	      vlib_buffer_advance (b1, sizeof (ip1));
-	      sr1 = (ip6_sr_header_t *) (ip1 + 1);
-	      new_l1 = clib_net_to_host_u16 (ip1->payload_length);
-	      next1 = SR_REWRITE_NEXT_SR_LOCAL;
-	    }
-	  else
-	    {
-	      u32 len_bytes = sizeof (ip6_header_t);
-	      u8 next_hdr = ip1->protocol;
-
-	      /* HBH must immediately follow ipv6 header */
-	      if (PREDICT_FALSE
-		  (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-		{
-		  ip6_hop_by_hop_ext_t *ext_hdr =
-		    (ip6_hop_by_hop_ext_t *) ip6_next_header (ip1);
-		  len_bytes +=
-		    ip6_ext_header_len ((ip6_ext_header_t *) ext_hdr);
-		  /* Ignoring the sr_local for now, if RH follows HBH here */
-		  next_hdr = ext_hdr->next_hdr;
-		  ext_hdr->next_hdr = IPPROTO_IPV6_ROUTE;
-		}
-	      else
-		{
-		  ip1->protocol = IPPROTO_IPV6_ROUTE;
-		}
-	      /*
-	       * Copy data before the punch-in point left by the
-	       * required amount. Assume (for the moment) that only
-	       * the main packet header needs to be copied.
-	       */
-	      clib_memcpy (((u8 *) ip1) - vec_len (t1->rewrite),
-			   ip1, len_bytes);
-	      vlib_buffer_advance (b1, -(word) vec_len (t1->rewrite));
-	      ip1 = vlib_buffer_get_current (b1);
-	      sr1 = (ip6_sr_header_t *) ((u8 *) ip1 + len_bytes);
-	      clib_memcpy (sr1, t1->rewrite, vec_len (t1->rewrite));
-
-	      sr1->protocol = next_hdr;
-	      new_l1 = clib_net_to_host_u16 (ip1->payload_length) +
-		vec_len (t1->rewrite);
-	      ip1->payload_length = clib_host_to_net_u16 (new_l1);
-
-	      /* Copy dst address into the DA slot in the segment list */
-	      clib_memcpy (sr1->segments, ip1->dst_address.as_u64,
-			   sizeof (ip6_address_t));
-	      /* Rewrite the ip6 dst address with the first hop */
-	      clib_memcpy (ip1->dst_address.as_u64, t1->first_hop.as_u64,
-			   sizeof (ip6_address_t));
-
-	      sr_fix_hmac (sm, ip1, sr1);
-
-	      vnet_buffer (b1)->ip.adj_index[VLIB_TX] =
-		t1->first_hop_dpo.dpoi_index;
-	      next1 = t1->first_hop_dpo.dpoi_next_node;
-	      next1 = (sr_local_cb ?
-		       sr_local_cb (vm, node, b1, ip1, sr1) : next1);
-
-	      /*
-	       * Ignore "do not rewrite" shtik in this path
-	       */
-	      if (PREDICT_FALSE (next1 & 0x80000000))
-		{
-		  next1 ^= 0xFFFFFFFF;
-		  if (PREDICT_FALSE (next1 == SR_REWRITE_NEXT_ERROR))
-		    b1->error = node->errors[SR_REWRITE_ERROR_APP_CALLBACK];
-		}
-	    }
-
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_rewrite_trace_t *tr = vlib_add_trace (vm, node,
-						       b0, sizeof (*tr));
-	      tr->tunnel_index = t0 - sm->tunnels;
-	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
-			   sizeof (tr->src.as_u8));
-	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
-			   sizeof (tr->dst.as_u8));
-	      tr->length = new_l0;
-	      tr->next_index = next0;
-	      if (sr0)
-		clib_memcpy (tr->sr, sr0, sizeof (tr->sr));
-	    }
-	  if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_rewrite_trace_t *tr = vlib_add_trace (vm, node,
-						       b1, sizeof (*tr));
-	      tr->tunnel_index = t1 - sm->tunnels;
-	      clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
-			   sizeof (tr->src.as_u8));
-	      clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
-			   sizeof (tr->dst.as_u8));
-	      tr->length = new_l1;
-	      tr->next_index = next1;
-	      if (sr1)
-		clib_memcpy (tr->sr, sr1, sizeof (tr->sr));
-	    }
-	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, bi1, next0, next1);
-	}
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  ip6_header_t *ip0 = 0;
-	  ip6_sr_header_t *sr0 = 0;
-	  ip6_sr_tunnel_t *t0;
-	  u32 next0;
-	  u16 new_l0 = 0;
-
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-
-	  /*
-	   * $$$ parse through header(s) to pick the point
-	   * where we punch in the SR extention header
-	   */
-	  t0 =
-	    pool_elt_at_index (sm->tunnels,
-			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
-
-	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE
-		  >= ((word) vec_len (t0->rewrite)) + b0->current_data);
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = t0->tx_fib_index;
-
-	  ip0 = vlib_buffer_get_current (b0);
-
-	  /*
-	   * SR-unaware service chaining case: pkt coming back from
-	   * service has the original dst address, and will already
-	   * have an SR header. If so, send it to sr-local
-	   */
-	  if (PREDICT_FALSE (ip0->protocol == IPPROTO_IPV6_ROUTE))
-	    {
-	      vlib_buffer_advance (b0, sizeof (ip0));
-	      sr0 = (ip6_sr_header_t *) (ip0 + 1);
-	      new_l0 = clib_net_to_host_u16 (ip0->payload_length);
-	      next0 = SR_REWRITE_NEXT_SR_LOCAL;
-	    }
-	  else
-	    {
-	      u32 len_bytes = sizeof (ip6_header_t);
-	      u8 next_hdr = ip0->protocol;
-
-	      /* HBH must immediately follow ipv6 header */
-	      if (PREDICT_FALSE
-		  (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-		{
-		  ip6_hop_by_hop_ext_t *ext_hdr =
-		    (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-		  len_bytes +=
-		    ip6_ext_header_len ((ip6_ext_header_t *) ext_hdr);
-		  next_hdr = ext_hdr->next_hdr;
-		  ext_hdr->next_hdr = IPPROTO_IPV6_ROUTE;
-		  /* Ignoring the sr_local for now, if RH follows HBH here */
-		}
-	      else
-		{
-		  ip0->protocol = IPPROTO_IPV6_ROUTE;	/* routing extension header */
-		}
-	      /*
-	       * Copy data before the punch-in point left by the
-	       * required amount. Assume (for the moment) that only
-	       * the main packet header needs to be copied.
-	       */
-	      clib_memcpy (((u8 *) ip0) - vec_len (t0->rewrite),
-			   ip0, len_bytes);
-	      vlib_buffer_advance (b0, -(word) vec_len (t0->rewrite));
-	      ip0 = vlib_buffer_get_current (b0);
-	      sr0 = (ip6_sr_header_t *) ((u8 *) ip0 + len_bytes);
-	      /* $$$ tune */
-	      clib_memcpy (sr0, t0->rewrite, vec_len (t0->rewrite));
-
-	      /* Fix the next header chain */
-	      sr0->protocol = next_hdr;
-	      new_l0 = clib_net_to_host_u16 (ip0->payload_length) +
-		vec_len (t0->rewrite);
-	      ip0->payload_length = clib_host_to_net_u16 (new_l0);
-
-	      /* Copy dst address into the DA slot in the segment list */
-	      clib_memcpy (sr0->segments, ip0->dst_address.as_u64,
-			   sizeof (ip6_address_t));
-	      /* Rewrite the ip6 dst address with the first hop */
-	      clib_memcpy (ip0->dst_address.as_u64, t0->first_hop.as_u64,
-			   sizeof (ip6_address_t));
-
-	      sr_fix_hmac (sm, ip0, sr0);
-
-	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
-		t0->first_hop_dpo.dpoi_index;
-	      next0 = t0->first_hop_dpo.dpoi_next_node;
-	      next0 = (sr_local_cb ?
-		       sr_local_cb (vm, node, b0, ip0, sr0) : next0);
-
-	      /*
-	       * Ignore "do not rewrite" shtik in this path
-	       */
-	      if (PREDICT_FALSE (next0 & 0x80000000))
-		{
-		  next0 ^= 0xFFFFFFFF;
-		  if (PREDICT_FALSE (next0 == SR_REWRITE_NEXT_ERROR))
-		    b0->error = node->errors[SR_REWRITE_ERROR_APP_CALLBACK];
-		}
-	    }
-
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_rewrite_trace_t *tr = vlib_add_trace (vm, node,
-						       b0, sizeof (*tr));
-	      tr->tunnel_index = t0 - sm->tunnels;
-	      if (ip0)
-		{
-		  memcpy (tr->src.as_u8, ip0->src_address.as_u8,
-			  sizeof (tr->src.as_u8));
-		  memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
-			  sizeof (tr->dst.as_u8));
-		}
-	      tr->length = new_l0;
-	      tr->next_index = next0;
-	      if (sr0)
-		clib_memcpy (tr->sr, sr0, sizeof (tr->sr));
-	    }
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  return from_frame->n_vectors;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (sr_rewrite_node) = {
-  .function = sr_rewrite,
-  .name = "sr-rewrite",
-  /* Takes a vector of packets. */
-  .vector_size = sizeof (u32),
-  .format_trace = format_sr_rewrite_trace,
-  .format_buffer = format_ip6_sr_header_with_length,
-
-  .n_errors = SR_REWRITE_N_ERROR,
-  .error_strings = sr_rewrite_error_strings,
-
-  .runtime_data_bytes = 0,
-
-  .n_next_nodes = SR_REWRITE_N_NEXT,
-  .next_nodes = {
-#define _(s,n) [SR_REWRITE_NEXT_##s] = n,
-    foreach_sr_rewrite_next
-#undef _
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (sr_rewrite_node, sr_rewrite)
-/* *INDENT-ON* */
-
-static int
-ip6_routes_add_del (ip6_sr_tunnel_t * t, int is_del)
-{
-  ip6_sr_main_t *sm = &sr_main;
-
-  /*
-   * the prefix for the tunnel's destination
-   */
-  /* *INDENT-OFF* */
-  fib_prefix_t pfx = {
-    .fp_proto = FIB_PROTOCOL_IP6,
-    .fp_len = t->dst_mask_width,
-    .fp_addr = {
-      .ip6 = t->key.dst,
-    }
-  };
-  /* *INDENT-ON* */
-
-  if (is_del)
-    {
-      fib_table_entry_delete (t->rx_fib_index, &pfx, FIB_SOURCE_SR);
-    }
-  else
-    {
-      dpo_id_t dpo = DPO_INVALID;
-
-      dpo_set (&dpo, sr_dpo_type, DPO_PROTO_IP6, t - sm->tunnels);
-      fib_table_entry_special_dpo_add (t->rx_fib_index,
-				       &pfx,
-				       FIB_SOURCE_SR,
-				       FIB_ENTRY_FLAG_EXCLUSIVE, &dpo);
-      dpo_reset (&dpo);
-    }
-
-  /*
-   * Track the first hop address so we don't need to perform an extra
-   * lookup in the data-path
-   */
-  /* *INDENT-OFF* */
-  const fib_prefix_t first_hop_pfx = {
-    .fp_len = 128,
-    .fp_proto = FIB_PROTOCOL_IP6,
-    .fp_addr = {
-      .ip6 = t->first_hop,
-    }
-  };
-  /* *INDENT-ON* */
-
-  if (is_del)
-    {
-      fib_entry_child_remove (t->fib_entry_index, t->sibling_index);
-      fib_table_entry_delete_index (t->fib_entry_index, FIB_SOURCE_RR);
-    }
-  else
-    {
-      t->fib_entry_index =
-	fib_table_entry_special_add (t->rx_fib_index,
-				     &first_hop_pfx,
-				     FIB_SOURCE_RR,
-				     FIB_ENTRY_FLAG_NONE, ADJ_INDEX_INVALID);
-      t->sibling_index =
-	fib_entry_child_add (t->fib_entry_index,
-			     sr_fib_node_type, t - sm->tunnels);
-    }
-
-  return 0;
-}
-
-/**
- * @brief Find or add if not found - HMAC shared secret
- *
- * @param sm ip6_sr_main_t *
- * @param secret u8 *
- * @param indexp u32 *
- *
- * @return ip6_sr_hmac_key_t *
- */
-static ip6_sr_hmac_key_t *
-find_or_add_shared_secret (ip6_sr_main_t * sm, u8 * secret, u32 * indexp)
-{
-  uword *p;
-  ip6_sr_hmac_key_t *key = 0;
-  int i;
-
-  p = hash_get_mem (sm->hmac_key_by_shared_secret, secret);
-
-  if (p)
-    {
-      key = vec_elt_at_index (sm->hmac_keys, p[0]);
-      if (indexp)
-	*indexp = p[0];
-      return (key);
-    }
-
-  /* Specific key ID? */
-  if (indexp && *indexp)
-    {
-      vec_validate (sm->hmac_keys, *indexp);
-      key = sm->hmac_keys + *indexp;
-    }
-  else
-    {
-      for (i = 0; i < vec_len (sm->hmac_keys); i++)
-	{
-	  if (sm->hmac_keys[i].shared_secret == 0)
-	    {
-	      key = sm->hmac_keys + i;
-	      goto found;
-	    }
-	}
-      vec_validate (sm->hmac_keys, i);
-      key = sm->hmac_keys + i;
-    found:
-      ;
-    }
-
-  key->shared_secret = vec_dup (secret);
-
-  hash_set_mem (sm->hmac_key_by_shared_secret, key->shared_secret,
-		key - sm->hmac_keys);
-
-  if (indexp)
-    *indexp = key - sm->hmac_keys;
-  return (key);
-}
-
-/**
- * @brief Stack a tunnel on the forwarding chain of the first-hop
- */
-static void
-sr_tunnel_stack (ip6_sr_tunnel_t * st)
-{
-  dpo_stack (sr_dpo_type,
-	     DPO_PROTO_IP6,
-	     &st->first_hop_dpo,
-	     fib_entry_contribute_ip_forwarding (st->fib_entry_index));
-}
-
-/**
- * @brief Add or Delete a Segment Routing tunnel.
- *
- * @param a ip6_sr_add_del_tunnel_args_t *
- *
- * @return retval int
- */
-int
-ip6_sr_add_del_tunnel (ip6_sr_add_del_tunnel_args_t * a)
-{
-  ip6_main_t *im = &ip6_main;
-  ip6_sr_tunnel_key_t key;
-  ip6_sr_tunnel_t *t;
-  uword *p, *n;
-  ip6_sr_header_t *h = 0;
-  u32 header_length;
-  ip6_address_t *addrp, *this_address;
-  ip6_sr_main_t *sm = &sr_main;
-  u8 *key_copy;
-  u32 rx_fib_index, tx_fib_index;
-  u32 hmac_key_index_u32;
-  u8 hmac_key_index = 0;
-  ip6_sr_policy_t *pt;
-  int i;
-
-  /* Make sure that the rx FIB exists */
-  p = hash_get (im->fib_index_by_table_id, a->rx_table_id);
-
-  if (p == 0)
-    return -3;
-
-  /* remember the FIB index */
-  rx_fib_index = p[0];
-
-  /* Make sure that the supplied FIB exists */
-  p = hash_get (im->fib_index_by_table_id, a->tx_table_id);
-
-  if (p == 0)
-    return -4;
-
-  /* remember the FIB index */
-  tx_fib_index = p[0];
-
-  clib_memcpy (key.src.as_u8, a->src_address->as_u8, sizeof (key.src));
-  clib_memcpy (key.dst.as_u8, a->dst_address->as_u8, sizeof (key.dst));
-
-  /* When adding a tunnel:
-   * - If a "name" is given, it must not exist.
-   * - The "key" is always checked, and must not exist.
-   * When deleting a tunnel:
-   * - If the "name" is given, and it exists, then use it.
-   * - If the "name" is not given, use the "key".
-   * - If the "name" and the "key" are given, then both must point to the same
-   *   thing.
-   */
-
-  /* Lookup the key */
-  p = hash_get_mem (sm->tunnel_index_by_key, &key);
-
-  /* If the name is given, look it up */
-  if (a->name)
-    n = hash_get_mem (sm->tunnel_index_by_name, a->name);
-  else
-    n = 0;
-
-  /* validate key/name parameters */
-  if (!a->is_del)		/* adding a tunnel */
-    {
-      if (a->name && n)		/* name given & exists already */
-	return -1;
-      if (p)			/* key exists already */
-	return -1;
-    }
-  else				/* deleting a tunnel */
-    {
-      if (!p)			/* key doesn't exist */
-	return -2;
-      if (a->name && !n)	/* name given & it doesn't exist */
-	return -2;
-
-      if (n)			/* name given & found */
-	{
-	  if (n[0] != p[0])	/* name and key do not point to the same thing */
-	    return -2;
-	}
-    }
-
-
-  if (a->is_del)		/* delete the tunnel */
-    {
-      hash_pair_t *hp;
-
-      /* Delete existing tunnel */
-      t = pool_elt_at_index (sm->tunnels, p[0]);
-
-      ip6_routes_add_del (t, 1);
-
-      vec_free (t->rewrite);
-      /* Remove tunnel from any policy if associated */
-      if (t->policy_index != ~0)
-	{
-	  pt = pool_elt_at_index (sm->policies, t->policy_index);
-	  for (i = 0; i < vec_len (pt->tunnel_indices); i++)
-	    {
-	      if (pt->tunnel_indices[i] == t - sm->tunnels)
-		{
-		  vec_delete (pt->tunnel_indices, 1, i);
-		  goto found;
-		}
-	    }
-	  clib_warning ("Tunnel index %d not found in policy_index %d",
-			t - sm->tunnels, pt - sm->policies);
-	found:
-	  /* If this is last tunnel in the  policy, clean up the policy too */
-	  if (vec_len (pt->tunnel_indices) == 0)
-	    {
-	      hash_unset_mem (sm->policy_index_by_policy_name, pt->name);
-	      vec_free (pt->name);
-	      pool_put (sm->policies, pt);
-	    }
-	}
-
-      /* Clean up the tunnel by name */
-      if (t->name)
-	{
-	  hash_unset_mem (sm->tunnel_index_by_name, t->name);
-	  vec_free (t->name);
-	}
-      dpo_reset (&t->first_hop_dpo);
-      pool_put (sm->tunnels, t);
-      hp = hash_get_pair (sm->tunnel_index_by_key, &key);
-      key_copy = (void *) (hp->key);
-      hash_unset_mem (sm->tunnel_index_by_key, &key);
-      vec_free (key_copy);
-      return 0;
-    }
-
-  /* create a new tunnel */
-  pool_get (sm->tunnels, t);
-  memset (t, 0, sizeof (*t));
-  t->policy_index = ~0;
-  fib_node_init (&t->node, sr_fib_node_type);
-
-  clib_memcpy (&t->key, &key, sizeof (t->key));
-  t->dst_mask_width = a->dst_mask_width;
-  t->rx_fib_index = rx_fib_index;
-  t->tx_fib_index = tx_fib_index;
-
-  if (!vec_len (a->segments))
-    /* there must be at least one segment... */
-    return -4;
-
-  /* The first specified hop goes right into the dst address */
-  clib_memcpy (&t->first_hop, &a->segments[0], sizeof (ip6_address_t));
-
-  /*
-   * Create the sr header rewrite string
-   * The list of segments needs an extra slot for the ultimate destination
-   * which is taken from the packet we add the SRH to.
-   */
-  header_length = sizeof (*h) +
-    sizeof (ip6_address_t) * (vec_len (a->segments) + 1 + vec_len (a->tags));
-
-  if (a->shared_secret)
-    {
-      /* Allocate a new key slot if we don't find the secret key */
-      hmac_key_index_u32 = 0;
-      (void) find_or_add_shared_secret (sm, a->shared_secret,
-					&hmac_key_index_u32);
-
-      /* Hey Vinz Clortho: Gozzer is pissed.. you're out of keys! */
-      if (hmac_key_index_u32 >= 256)
-	return -5;
-      hmac_key_index = hmac_key_index_u32;
-      header_length += SHA256_DIGEST_LENGTH;
-    }
-
-  vec_validate (t->rewrite, header_length - 1);
-
-  h = (ip6_sr_header_t *) t->rewrite;
-
-  h->protocol = 0xFF;		/* we don't know yet */
-
-  h->length = (header_length / 8) - 1;
-  h->type = ROUTING_HEADER_TYPE_SR;
-
-  /* first_segment and segments_left need to have the index of the last
-   * element in the list; a->segments has one element less than ends up
-   * in the header (it does not have the DA in it), so vec_len(a->segments)
-   * is the value we want.
-   */
-  h->first_segment = h->segments_left = vec_len (a->segments);
-
-  if (a->shared_secret)
-    h->hmac_key = hmac_key_index & 0xFF;
-
-  h->flags = a->flags_net_byte_order;
-
-  /* Paint on the segment list, in reverse.
-   * This is offset by one to leave room at the start for the ultimate
-   * destination.
-   */
-  addrp = h->segments + vec_len (a->segments);
-
-  vec_foreach (this_address, a->segments)
-  {
-    clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t));
-    addrp--;
-  }
-
-  /*
-   * Since the ultimate destination address is not yet known, set that slot
-   * to a value we will instantly recognize as bogus.
-   */
-  memset (h->segments, 0xfe, sizeof (ip6_address_t));
-
-  /* Paint on the tag list, not reversed */
-  addrp = h->segments + vec_len (a->segments);
-
-  vec_foreach (this_address, a->tags)
-  {
-    clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t));
-    addrp++;
-  }
-
-  key_copy = vec_new (ip6_sr_tunnel_key_t, 1);
-  clib_memcpy (key_copy, &key, sizeof (ip6_sr_tunnel_key_t));
-  hash_set_mem (sm->tunnel_index_by_key, key_copy, t - sm->tunnels);
-
-  /*
-   * Stick the tunnel index into the rewrite header.
-   *
-   * Unfortunately, inserting an SR header according to the various
-   * RFC's requires parsing through the ip6 header, perhaps consing a
-   * buffer onto the head of the vlib_buffer_t, etc. We don't use the
-   * normal reverse bcopy rewrite code.
-   *
-   * We don't handle ugly RFC-related cases yet, but I'm sure PL will complain
-   * at some point...
-   */
-
-  /*
-   * Add the routes for the tunnel destination and first-hop, then stack
-   * the tunnel on the appropriate forwarding DPOs.
-   */
-  ip6_routes_add_del (t, 0);
-  sr_tunnel_stack (t);
-
-  if (a->policy_name)
-    {
-      p = hash_get_mem (sm->policy_index_by_policy_name, a->policy_name);
-      if (p)
-	{
-	  pt = pool_elt_at_index (sm->policies, p[0]);
-	}
-      else			/* no policy, lets create one */
-	{
-	  pool_get (sm->policies, pt);
-	  memset (pt, 0, sizeof (*pt));
-	  pt->name = format (0, "%s%c", a->policy_name, 0);
-	  hash_set_mem (sm->policy_index_by_policy_name, pt->name,
-			pt - sm->policies);
-	  p = hash_get_mem (sm->policy_index_by_policy_name, a->policy_name);
-	}
-      vec_add1 (pt->tunnel_indices, t - sm->tunnels);
-      if (p == 0)
-	clib_warning ("p is NULL!");
-      t->policy_index = p ? p[0] : ~0;	/* equiv. to (pt - sm->policies) */
-    }
-
-  if (a->name)
-    {
-      t->name = format (0, "%s%c", a->name, 0);
-      hash_set_mem (sm->tunnel_index_by_name, t->name, t - sm->tunnels);
-    }
-
-  return 0;
-}
 
 /**
  * @brief no-op lock function.
  * The lifetime of the SR entry is managed by the control plane
  */
-static void
+void
 sr_dpo_lock (dpo_id_t * dpo)
 {
 }
@@ -1208,2227 +43,15 @@ sr_dpo_lock (dpo_id_t * dpo)
  * @brief no-op unlock function.
  * The lifetime of the SR entry is managed by the control plane
  */
-static void
+void
 sr_dpo_unlock (dpo_id_t * dpo)
 {
 }
 
-u8 *
-format_sr_dpo (u8 * s, va_list * args)
-{
-  index_t index = va_arg (*args, index_t);
-  CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
-
-  return (format (s, "SR: tunnel:[%d]", index));
-}
-
-const static dpo_vft_t sr_dpo_vft = {
-  .dv_lock = sr_dpo_lock,
-  .dv_unlock = sr_dpo_unlock,
-  .dv_format = format_sr_dpo,
-};
-
-const static char *const sr_ip6_nodes[] = {
-  "sr-rewrite",
-  NULL,
-};
-
-const static char *const *const sr_nodes[DPO_PROTO_NUM] = {
-  [DPO_PROTO_IP6] = sr_ip6_nodes,
-};
-
-static ip6_sr_tunnel_t *
-sr_tunnel_from_fib_node (fib_node_t * node)
-{
-#if (CLIB_DEBUG > 0)
-  ASSERT (sr_fib_node_type == node->fn_type);
-#endif
-  return ((ip6_sr_tunnel_t *) (((char *) node) -
-			       STRUCT_OFFSET_OF (ip6_sr_tunnel_t, node)));
-}
-
-/**
- * Function definition to backwalk a FIB node
- */
-static fib_node_back_walk_rc_t
-sr_tunnel_back_walk (fib_node_t * node, fib_node_back_walk_ctx_t * ctx)
-{
-  sr_tunnel_stack (sr_tunnel_from_fib_node (node));
-
-  return (FIB_NODE_BACK_WALK_CONTINUE);
-}
-
-/**
- * Function definition to get a FIB node from its index
- */
-static fib_node_t *
-sr_tunnel_fib_node_get (fib_node_index_t index)
-{
-  ip6_sr_tunnel_t *st;
-  ip6_sr_main_t *sm;
-
-  sm = &sr_main;
-  st = pool_elt_at_index (sm->tunnels, index);
-
-  return (&st->node);
-}
-
-/**
- * Function definition to inform the FIB node that its last lock has gone.
- */
-static void
-sr_tunnel_last_lock_gone (fib_node_t * node)
-{
-  /*
-   * The SR tunnel is a root of the graph. As such
-   * it never has children and thus is never locked.
-   */
-  ASSERT (0);
-}
-
 /*
- * Virtual function table registered by SR tunnels
- * for participation in the FIB object graph.
- */
-const static fib_node_vft_t sr_fib_vft = {
-  .fnv_get = sr_tunnel_fib_node_get,
-  .fnv_last_lock = sr_tunnel_last_lock_gone,
-  .fnv_back_walk = sr_tunnel_back_walk,
-};
-
-/**
- * @brief CLI parser for Add or Delete a Segment Routing tunnel.
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-sr_add_del_tunnel_command_fn (vlib_main_t * vm,
-			      unformat_input_t * input,
-			      vlib_cli_command_t * cmd)
-{
-  int is_del = 0;
-  ip6_address_t src_address;
-  int src_address_set = 0;
-  ip6_address_t dst_address;
-  u32 dst_mask_width;
-  int dst_address_set = 0;
-  u16 flags = 0;
-  u8 *shared_secret = 0;
-  u8 *name = 0;
-  u8 *policy_name = 0;
-  u32 rx_table_id = 0;
-  u32 tx_table_id = 0;
-  ip6_address_t *segments = 0;
-  ip6_address_t *this_seg;
-  ip6_address_t *tags = 0;
-  ip6_address_t *this_tag;
-  ip6_sr_add_del_tunnel_args_t _a, *a = &_a;
-  ip6_address_t next_address, tag;
-  int pl_index;
-  int rv;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "del"))
-	is_del = 1;
-      else if (unformat (input, "rx-fib-id %d", &rx_table_id))
-	;
-      else if (unformat (input, "tx-fib-id %d", &tx_table_id))
-	;
-      else if (unformat (input, "src %U", unformat_ip6_address, &src_address))
-	src_address_set = 1;
-      else if (unformat (input, "name %s", &name))
-	;
-      else if (unformat (input, "policy %s", &policy_name))
-	;
-      else if (unformat (input, "dst %U/%d",
-			 unformat_ip6_address, &dst_address, &dst_mask_width))
-	dst_address_set = 1;
-      else if (unformat (input, "next %U", unformat_ip6_address,
-			 &next_address))
-	{
-	  vec_add2 (segments, this_seg, 1);
-	  clib_memcpy (this_seg->as_u8, next_address.as_u8,
-		       sizeof (*this_seg));
-	}
-      else if (unformat (input, "tag %U", unformat_ip6_address, &tag))
-	{
-	  vec_add2 (tags, this_tag, 1);
-	  clib_memcpy (this_tag->as_u8, tag.as_u8, sizeof (*this_tag));
-	}
-      else if (unformat (input, "clean"))
-	flags |= IP6_SR_HEADER_FLAG_CLEANUP;
-      else if (unformat (input, "protected"))
-	flags |= IP6_SR_HEADER_FLAG_PROTECTED;
-      else if (unformat (input, "key %s", &shared_secret))
-	/* Do not include the trailing NULL byte. Guaranteed interop issue */
-	_vec_len (shared_secret) -= 1;
-      else if (unformat (input, "InPE %d", &pl_index))
-	{
-	  if (pl_index <= 0 || pl_index > 4)
-	    {
-	    pl_index_range_error:
-	      return clib_error_return
-		(0, "Policy List Element Index %d out of range (1-4)",
-		 pl_index);
-
-	    }
-	  flags |= IP6_SR_HEADER_FLAG_PL_ELT_INGRESS_PE
-	    << ip6_sr_policy_list_shift_from_index (pl_index);
-	}
-      else if (unformat (input, "EgPE %d", &pl_index))
-	{
-	  if (pl_index <= 0 || pl_index > 4)
-	    goto pl_index_range_error;
-	  flags |= IP6_SR_HEADER_FLAG_PL_ELT_EGRESS_PE
-	    << ip6_sr_policy_list_shift_from_index (pl_index);
-	}
-      else if (unformat (input, "OrgSrc %d", &pl_index))
-	{
-	  if (pl_index <= 0 || pl_index > 4)
-	    goto pl_index_range_error;
-	  flags |= IP6_SR_HEADER_FLAG_PL_ELT_ORIG_SRC_ADDR
-	    << ip6_sr_policy_list_shift_from_index (pl_index);
-	}
-      else
-	break;
-    }
-
-  if (!src_address_set)
-    return clib_error_return (0, "src address required");
-
-  if (!dst_address_set)
-    return clib_error_return (0, "dst address required");
-
-  if (!segments)
-    return clib_error_return (0, "at least one sr segment required");
-
-  memset (a, 0, sizeof (*a));
-  a->src_address = &src_address;
-  a->dst_address = &dst_address;
-  a->dst_mask_width = dst_mask_width;
-  a->segments = segments;
-  a->tags = tags;
-  a->flags_net_byte_order = clib_host_to_net_u16 (flags);
-  a->is_del = is_del;
-  a->rx_table_id = rx_table_id;
-  a->tx_table_id = tx_table_id;
-  a->shared_secret = shared_secret;
-
-  if (vec_len (name))
-    a->name = name;
-  else
-    a->name = 0;
-
-  if (vec_len (policy_name))
-    a->policy_name = policy_name;
-  else
-    a->policy_name = 0;
-
-  rv = ip6_sr_add_del_tunnel (a);
-
-  vec_free (segments);
-  vec_free (tags);
-  vec_free (shared_secret);
-
-  switch (rv)
-    {
-    case 0:
-      break;
-
-    case -1:
-      return clib_error_return (0, "SR tunnel src %U dst %U already exists",
-				format_ip6_address, &src_address,
-				format_ip6_address, &dst_address);
-
-    case -2:
-      return clib_error_return (0, "SR tunnel src %U dst %U does not exist",
-				format_ip6_address, &src_address,
-				format_ip6_address, &dst_address);
-
-    case -3:
-      return clib_error_return (0, "FIB table %d does not exist",
-				rx_table_id);
-
-    case -4:
-      return clib_error_return (0, "At least one segment is required");
-
-    default:
-      return clib_error_return (0, "BUG: ip6_sr_add_del_tunnel returns %d",
-				rv);
-    }
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (sr_tunnel_command, static) = {
-    .path = "sr tunnel",
-    .short_help =
-      "sr tunnel [del] [name ] src  dst  [next ] "
-      "[clean] [reroute] [key ] [policy ]"
-      "[rx-fib-id ] [tx-fib-id ]",
-    .function = sr_add_del_tunnel_command_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Display Segment Routing tunnel
- *
- * @param vm vlib_main_t *
- * @param t ip6_sr_tunnel_t *
- *
- */
-void
-ip6_sr_tunnel_display (vlib_main_t * vm, ip6_sr_tunnel_t * t)
-{
-  ip6_sr_main_t *sm = &sr_main;
-  ip6_fib_t *rx_fib, *tx_fib;
-  ip6_sr_policy_t *pt;
-
-  rx_fib = ip6_fib_get (t->rx_fib_index);
-  tx_fib = ip6_fib_get (t->tx_fib_index);
-
-  if (t->name)
-    vlib_cli_output (vm, "sr tunnel name: %s", (char *) t->name);
-
-  vlib_cli_output (vm, "src %U dst %U first hop %U",
-		   format_ip6_address, &t->key.src,
-		   format_ip6_address, &t->key.dst,
-		   format_ip6_address, &t->first_hop);
-  vlib_cli_output (vm, "    rx-fib-id %d tx-fib-id %d",
-		   rx_fib->table_id, tx_fib->table_id);
-  vlib_cli_output (vm, "  sr: %U", format_ip6_sr_header, t->rewrite,
-		   0 /* print_hmac */ );
-
-  if (t->policy_index != ~0)
-    {
-      pt = pool_elt_at_index (sm->policies, t->policy_index);
-      vlib_cli_output (vm, "sr policy: %s", (char *) pt->name);
-    }
-  vlib_cli_output (vm, "-------");
-
-  return;
-}
-
-/**
- * @brief CLI Parser for Display Segment Routing tunnel
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-show_sr_tunnel_fn (vlib_main_t * vm,
-		   unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  static ip6_sr_tunnel_t **tunnels;
-  ip6_sr_tunnel_t *t;
-  ip6_sr_main_t *sm = &sr_main;
-  int i;
-  uword *p = 0;
-  u8 *name = 0;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "name %s", &name))
-	{
-	  p = hash_get_mem (sm->tunnel_index_by_name, name);
-	  if (!p)
-	    vlib_cli_output (vm, "No SR tunnel with name: %s. Showing all.",
-			     name);
-	}
-      else
-	break;
-    }
-
-  vec_reset_length (tunnels);
-
-  if (!p)			/* Either name parm not passed or no tunnel with that name found, show all */
-    {
-      /* *INDENT-OFF* */
-  pool_foreach (t, sm->tunnels,
-  ({
-    vec_add1 (tunnels, t);
-  }));
-  /* *INDENT-ON* */
-    }
-  else				/* Just show the one tunnel by name */
-    vec_add1 (tunnels, &sm->tunnels[p[0]]);
-
-  if (vec_len (tunnels) == 0)
-    vlib_cli_output (vm, "No SR tunnels configured");
-
-  for (i = 0; i < vec_len (tunnels); i++)
-    {
-      t = tunnels[i];
-      ip6_sr_tunnel_display (vm, t);
-    }
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
-    .path = "show sr tunnel",
-    .short_help = "show sr tunnel [name ]",
-    .function = show_sr_tunnel_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Add or Delete a Segment Routing policy
- *
- * @param a ip6_sr_add_del_policy_args_t *
- *
- * @return retval int
- */
-int
-ip6_sr_add_del_policy (ip6_sr_add_del_policy_args_t * a)
-{
-  ip6_sr_main_t *sm = &sr_main;
-  uword *p;
-  ip6_sr_tunnel_t *t = 0;
-  ip6_sr_policy_t *policy;
-  u32 *tunnel_indices = 0;
-  int i;
-
-
-
-  if (a->is_del)
-    {
-      p = hash_get_mem (sm->policy_index_by_policy_name, a->name);
-      if (!p)
-	return -6;		/* policy name not found */
-
-      policy = pool_elt_at_index (sm->policies, p[0]);
-
-      vec_foreach_index (i, policy->tunnel_indices)
-      {
-	t = pool_elt_at_index (sm->tunnels, policy->tunnel_indices[i]);
-	t->policy_index = ~0;
-      }
-      hash_unset_mem (sm->policy_index_by_policy_name, a->name);
-      pool_put (sm->policies, policy);
-      return 0;
-    }
-
-
-  if (!vec_len (a->tunnel_names))
-    return -3;			/*tunnel name is required case */
-
-  vec_reset_length (tunnel_indices);
-  /* Check tunnel names, add tunnel_index to policy */
-  for (i = 0; i < vec_len (a->tunnel_names); i++)
-    {
-      p = hash_get_mem (sm->tunnel_index_by_name, a->tunnel_names[i]);
-      if (!p)
-	return -4;		/* tunnel name not found case */
-
-      t = pool_elt_at_index (sm->tunnels, p[0]);
-      /*
-         No need to check t==0. -3 condition above ensures name
-       */
-      if (t->policy_index != ~0)
-	return -5;		/* tunnel name already associated with a policy */
-
-      /* Add to tunnel indicies */
-      vec_add1 (tunnel_indices, p[0]);
-    }
-
-  /* Add policy to ip6_sr_main_t */
-  pool_get (sm->policies, policy);
-  policy->name = a->name;
-  policy->tunnel_indices = tunnel_indices;
-  hash_set_mem (sm->policy_index_by_policy_name, policy->name,
-		policy - sm->policies);
-
-  /* Yes, this could be construed as overkill but the last thing you should do is set
-     the policy_index on the tunnel after everything is set in ip6_sr_main_t.
-     If this is deemed overly cautious, could set this in the vec_len(tunnel_names) loop.
-   */
-  for (i = 0; i < vec_len (policy->tunnel_indices); i++)
-    {
-      t = pool_elt_at_index (sm->tunnels, policy->tunnel_indices[i]);
-      t->policy_index = policy - sm->policies;
-    }
-
-  return 0;
-}
-
-/**
- * @brief CLI Parser for Add or Delete a Segment Routing policy
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-sr_add_del_policy_command_fn (vlib_main_t * vm,
-			      unformat_input_t * input,
-			      vlib_cli_command_t * cmd)
-{
-  int is_del = 0;
-  u8 **tunnel_names = 0;
-  u8 *tunnel_name = 0;
-  u8 *name = 0;
-  ip6_sr_add_del_policy_args_t _a, *a = &_a;
-  int rv;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "del"))
-	is_del = 1;
-      else if (unformat (input, "name %s", &name))
-	;
-      else if (unformat (input, "tunnel %s", &tunnel_name))
-	{
-	  if (tunnel_name)
-	    {
-	      vec_add1 (tunnel_names, tunnel_name);
-	      tunnel_name = 0;
-	    }
-	}
-      else
-	break;
-    }
-
-  if (!name)
-    return clib_error_return (0, "name of SR policy required");
-
-
-  memset (a, 0, sizeof (*a));
-
-  a->is_del = is_del;
-  a->name = name;
-  a->tunnel_names = tunnel_names;
-
-  rv = ip6_sr_add_del_policy (a);
-
-  vec_free (tunnel_names);
-
-  switch (rv)
-    {
-    case 0:
-      break;
-
-    case -3:
-      return clib_error_return (0,
-				"tunnel name to associate to SR policy is required");
-
-    case -4:
-      return clib_error_return (0, "tunnel name not found");
-
-    case -5:
-      return clib_error_return (0, "tunnel already associated with policy");
-
-    case -6:
-      return clib_error_return (0, "policy name %s not found", name);
-
-    case -7:
-      return clib_error_return (0, "TODO: deleting policy name %s", name);
-
-    default:
-      return clib_error_return (0, "BUG: ip6_sr_add_del_policy returns %d",
-				rv);
-
-    }
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (sr_policy_command, static) = {
-    .path = "sr policy",
-    .short_help =
-    "sr policy [del] name  tunnel  [tunnel ]*",
-    .function = sr_add_del_policy_command_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief CLI Parser for Displaying Segment Routing policy
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-show_sr_policy_fn (vlib_main_t * vm,
-		   unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  static ip6_sr_policy_t **policies;
-  ip6_sr_policy_t *policy;
-  ip6_sr_tunnel_t *t;
-  ip6_sr_main_t *sm = &sr_main;
-  int i, j;
-  uword *p = 0;
-  u8 *name = 0;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "name %s", &name))
-	{
-	  p = hash_get_mem (sm->policy_index_by_policy_name, name);
-	  if (!p)
-	    vlib_cli_output (vm,
-			     "policy with name %s not found. Showing all.",
-			     name);
-	}
-      else
-	break;
-    }
-
-  vec_reset_length (policies);
-
-  if (!p)			/* Either name parm not passed or no policy with that name found, show all */
-    {
-      /* *INDENT-OFF* */
-  pool_foreach (policy, sm->policies,
-  ({
-    vec_add1 (policies, policy);
-  }));
-  /* *INDENT-ON* */
-    }
-  else				/* Just show the one policy by name and a summary of tunnel names */
-    {
-      policy = pool_elt_at_index (sm->policies, p[0]);
-      vec_add1 (policies, policy);
-    }
-
-  if (vec_len (policies) == 0)
-    vlib_cli_output (vm, "No SR policies configured");
-
-  for (i = 0; i < vec_len (policies); i++)
-    {
-      policy = policies[i];
-
-      if (policy->name)
-	vlib_cli_output (vm, "SR policy name: %s", (char *) policy->name);
-      for (j = 0; j < vec_len (policy->tunnel_indices); j++)
-	{
-	  t = pool_elt_at_index (sm->tunnels, policy->tunnel_indices[j]);
-	  ip6_sr_tunnel_display (vm, t);
-	}
-    }
-
-  return 0;
-
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (show_sr_policy_command, static) = {
-    .path = "show sr policy",
-    .short_help = "show sr policy [name ]",
-    .function = show_sr_policy_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Add or Delete a mapping of IP6 multicast address
- * to Segment Routing policy.
- *
- * @param a ip6_sr_add_del_multicastmap_args_t *
- *
- * @return retval int
- */
-int
-ip6_sr_add_del_multicastmap (ip6_sr_add_del_multicastmap_args_t * a)
-{
-  uword *p;
-  ip6_sr_tunnel_t *t;
-  ip6_sr_main_t *sm = &sr_main;
-  ip6_sr_policy_t *pt;
-  index_t rep;
-  u32 ii;
-
-  if (a->is_del)
-    {
-      /* clean up the adjacency */
-      p =
-	hash_get_mem (sm->policy_index_by_multicast_address,
-		      a->multicast_address);
-    }
-  else
-    {
-      /* Get our policy by policy_name */
-      p = hash_get_mem (sm->policy_index_by_policy_name, a->policy_name);
-
-    }
-  if (!p)
-    return -1;
-
-  pt = pool_elt_at_index (sm->policies, p[0]);
-
-  /*
-     Get the first tunnel associated with policy populate the fib adjacency.
-     From there, since this tunnel will have it's policy_index != ~0 it will
-     be the trigger in the dual_loop to pull up the policy and make a copy-rewrite
-     for each tunnel in the policy
-   */
-
-  t = pool_elt_at_index (sm->tunnels, pt->tunnel_indices[0]);
-
-  /*
-   * Stick the tunnel index into the rewrite header.
-   *
-   * Unfortunately, inserting an SR header according to the various
-   * RFC's requires parsing through the ip6 header, perhaps consing a
-   * buffer onto the head of the vlib_buffer_t, etc. We don't use the
-   * normal reverse bcopy rewrite code.
-   *
-   * We don't handle ugly RFC-related cases yet, but I'm sure PL will complain
-   * at some point...
-   */
-
-  /*
-   * Construct an mFIB entry for the multicast address,
-   * using the rx/tx fib from the first tunnel.
-   * There is no RPF information for this address (I need to discuss this with
-   * Pablo), so for now accept from anywhere...
-   */
-  /* *INDENT-OFF* */
-  mfib_prefix_t pfx = {
-    .fp_proto = FIB_PROTOCOL_IP6,
-    .fp_len = 128,
-    .fp_grp_addr = {
-      .ip6 = *a->multicast_address,
-    }
-  };
-  /* *INDENT-ON* */
-
-  if (a->is_del)
-    mfib_table_entry_delete (t->rx_fib_index, &pfx, MFIB_SOURCE_SRv6);
-  else
-    {
-      /*
-       * Construct a replicate DPO that will replicate received packets over
-       * each tunnel in the policy
-       */
-      dpo_id_t dpo = DPO_INVALID;
-
-      rep = replicate_create (vec_len (pt->tunnel_indices), DPO_PROTO_IP6);
-
-      vec_foreach_index (ii, pt->tunnel_indices)
-      {
-	dpo_set (&dpo, sr_dpo_type, DPO_PROTO_IP6, pt->tunnel_indices[ii]);
-
-	replicate_set_bucket (rep, ii, &dpo);
-      }
-
-      mfib_table_entry_special_add (t->rx_fib_index,
-				    &pfx,
-				    MFIB_SOURCE_SRv6,
-				    MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF, rep);
-
-      dpo_reset (&dpo);
-    }
-
-  u8 *mcast_copy = 0;
-  mcast_copy = vec_new (ip6_address_t, 1);
-  memcpy (mcast_copy, a->multicast_address, sizeof (ip6_address_t));
-
-  if (a->is_del)
-    {
-      hash_unset_mem (sm->policy_index_by_multicast_address, mcast_copy);
-      vec_free (mcast_copy);
-    }
-  else
-    {
-      hash_set_mem (sm->policy_index_by_multicast_address, mcast_copy,
-		    pt - sm->policies);
-    }
-
-  return 0;
-}
-
-/**
- * @brief CLI Parser for Adding or Delete a mapping of IP6 multicast address
- * to Segment Routing policy.
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-sr_add_del_multicast_map_command_fn (vlib_main_t * vm,
-				     unformat_input_t * input,
-				     vlib_cli_command_t * cmd)
-{
-  int is_del = 0;
-  ip6_address_t multicast_address;
-  u8 *policy_name = 0;
-  int multicast_address_set = 0;
-  ip6_sr_add_del_multicastmap_args_t _a, *a = &_a;
-  int rv;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "del"))
-	is_del = 1;
-      else
-	if (unformat
-	    (input, "address %U", unformat_ip6_address, &multicast_address))
-	multicast_address_set = 1;
-      else if (unformat (input, "sr-policy %s", &policy_name))
-	;
-      else
-	break;
-    }
-
-  if (!is_del && !policy_name)
-    return clib_error_return (0, "name of sr policy required");
-
-  if (!multicast_address_set)
-    return clib_error_return (0, "multicast address required");
-
-  memset (a, 0, sizeof (*a));
-
-  a->is_del = is_del;
-  a->multicast_address = &multicast_address;
-  a->policy_name = policy_name;
-
-  rv = ip6_sr_add_del_multicastmap (a);
-
-  switch (rv)
-    {
-    case 0:
-      break;
-    case -1:
-      return clib_error_return (0, "no policy with name: %s", policy_name);
-
-    case -2:
-      return clib_error_return (0, "multicast map someting ");
-
-    case -3:
-      return clib_error_return (0,
-				"tunnel name to associate to SR policy is required");
-
-    case -7:
-      return clib_error_return (0, "TODO: deleting policy name %s",
-				policy_name);
-
-    default:
-      return clib_error_return (0, "BUG: ip6_sr_add_del_policy returns %d",
-				rv);
-
-    }
-  return 0;
-
-}
-
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (sr_multicast_map_command, static) = {
-    .path = "sr multicast-map",
-    .short_help =
-    "sr multicast-map address  sr-policy  [del]",
-    .function = sr_add_del_multicast_map_command_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief CLI Parser for Displaying a mapping of IP6 multicast address
- * to Segment Routing policy.
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-show_sr_multicast_map_fn (vlib_main_t * vm,
-			  unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  ip6_sr_main_t *sm = &sr_main;
-  u8 *key = 0;
-  u32 value;
-  ip6_address_t multicast_address;
-  ip6_sr_policy_t *pt;
-
-  /* pull all entries from the hash table into vector for display */
-
-  /* *INDENT-OFF* */
-  hash_foreach_mem (key, value, sm->policy_index_by_multicast_address,
-  ({
-    if (!key)
-	vlib_cli_output (vm, "no multicast maps configured");
-    else
-      {
-	multicast_address = *((ip6_address_t *)key);
-	pt = pool_elt_at_index (sm->policies, value);
-	if (pt)
-	  {
-	    vlib_cli_output (vm, "address: %U policy: %s",
-			     format_ip6_address, &multicast_address,
-			     pt->name);
-	  }
-	else
-	  vlib_cli_output (vm, "BUG: policy not found for address: %U with policy index %d",
-			     format_ip6_address, &multicast_address,
-			     value);
-
-      }
-
-  }));
-  /* *INDENT-ON* */
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (show_sr_multicast_map_command, static) = {
-    .path = "show sr multicast-map",
-    .short_help = "show sr multicast-map",
-    .function = show_sr_multicast_map_fn,
-};
-/* *INDENT-ON* */
-
-
-#define foreach_sr_fix_dst_addr_next            \
-_(DROP, "error-drop")
-
-/**
- * @brief Struct for valid next-nodes for SR fix destination address node
- */
-typedef enum
-{
-#define _(s,n) SR_FIX_DST_ADDR_NEXT_##s,
-  foreach_sr_fix_dst_addr_next
-#undef _
-    SR_FIX_DST_ADDR_N_NEXT,
-} sr_fix_dst_addr_next_t;
-
-/**
- * @brief Error strings for SR Fix Destination rewrite
- */
-static char *sr_fix_dst_error_strings[] = {
-#define sr_fix_dst_error(n,s) s,
-#include "sr_fix_dst_error.def"
-#undef sr_fix_dst_error
-};
-
-/**
- * @brief Struct for errors for SR Fix Destination rewrite
- */
-typedef enum
-{
-#define sr_fix_dst_error(n,s) SR_FIX_DST_ERROR_##n,
-#include "sr_fix_dst_error.def"
-#undef sr_fix_dst_error
-  SR_FIX_DST_N_ERROR,
-} sr_fix_dst_error_t;
-
-/**
- * @brief Information for fix address trace
- */
-typedef struct
-{
-  ip6_address_t src, dst;
-  u32 next_index;
-  u32 adj_index;
-  u8 sr[256];
-} sr_fix_addr_trace_t;
-
-/**
- * @brief Formatter for fix address trace
- */
-u8 *
-format_sr_fix_addr_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  sr_fix_addr_trace_t *t = va_arg (*args, sr_fix_addr_trace_t *);
-  vnet_hw_interface_t *hi = 0;
-  ip_adjacency_t *adj;
-  ip6_main_t *im = &ip6_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
-  vnet_main_t *vnm = vnet_get_main ();
-
-  if (t->adj_index != ~0)
-    {
-      adj = ip_get_adjacency (lm, t->adj_index);
-      hi = vnet_get_sup_hw_interface (vnm, adj->rewrite_header.sw_if_index);
-    }
-
-  s = format (s, "SR-FIX_ADDR: next %s ip6 src %U dst %U\n",
-	      (t->next_index == SR_FIX_DST_ADDR_NEXT_DROP)
-	      ? "drop" : "output",
-	      format_ip6_address, &t->src, format_ip6_address, &t->dst);
-  if (t->next_index != SR_FIX_DST_ADDR_NEXT_DROP)
-    {
-      s =
-	format (s, "%U\n", format_ip6_sr_header, t->sr, 1 /* print_hmac */ );
-      s =
-	format (s, "   output via %s",
-		hi ? (char *) (hi->name) : "Invalid adj");
-    }
-  return s;
-}
-
-/**
- * @brief Fix SR destination address - dual-loop
- *
- * @node sr-fix-dst-addr
- * @param vm vlib_main_t *
- * @param node vlib_node_runtime_t *
- * @param from_frame vlib_frame_t *
- *
- * @return from_frame->n_vectors uword
- */
-static uword
-sr_fix_dst_addr (vlib_main_t * vm,
-		 vlib_node_runtime_t * node, vlib_frame_t * from_frame)
-{
-  u32 n_left_from, next_index, *from, *to_next;
-  ip6_main_t *im = &ip6_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-#if 0
-      while (0 && n_left_from >= 4 && n_left_to_next >= 2)
-	{
-	  u32 bi0, bi1;
-	  __attribute__ ((unused)) vlib_buffer_t *b0, *b1;
-	  u32 next0 = SR_FIX_DST_ADDR_NEXT_DROP;
-	  u32 next1 = SR_FIX_DST_ADDR_NEXT_DROP;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t *p2, *p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-	  }
-
-	  bi0 = from[0];
-	  bi1 = from[1];
-	  to_next[0] = bi0;
-	  to_next[1] = bi1;
-	  from += 2;
-	  to_next += 2;
-	  n_left_to_next -= 2;
-	  n_left_from -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-
-	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, bi1, next0, next1);
-	}
-#endif
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  ip6_header_t *ip0;
-	  ip_adjacency_t *adj0;
-	  ip6_sr_header_t *sr0;
-	  u32 next0 = SR_FIX_DST_ADDR_NEXT_DROP;
-	  ip6_address_t *new_dst0;
-	  ethernet_header_t *eh0;
-
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  adj0 =
-	    ip_get_adjacency (lm, vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
-	  next0 = adj0->if_address_index;
-
-	  /* We should be pointing at an Ethernet header... */
-	  eh0 = vlib_buffer_get_current (b0);
-	  ip0 = (ip6_header_t *) (eh0 + 1);
-	  sr0 = (ip6_sr_header_t *) (ip0 + 1);
-
-	  /* We'd better find an SR header... */
-	  if (PREDICT_FALSE (ip0->protocol != IPPROTO_IPV6_ROUTE))
-	    {
-	      b0->error = node->errors[SR_FIX_DST_ERROR_NO_SR_HEADER];
-	      goto do_trace0;
-	    }
-	  else
-	    {
-	      /*
-	       * We get here from sr_rewrite or sr_local, with
-	       * sr->segments_left pointing at the (copy of the original) dst
-	       * address. Use it, then increment sr0->segments_left.
-	       */
-
-	      /* Out of segments? Turf the packet */
-	      if (PREDICT_FALSE (sr0->segments_left == 0))
-		{
-		  b0->error = node->errors[SR_FIX_DST_ERROR_NO_MORE_SEGMENTS];
-		  goto do_trace0;
-		}
-
-	      /*
-	       * Rewrite the packet with the original dst address
-	       * We assume that the last segment (in processing order) contains
-	       * the original dst address. The list is reversed, so sr0->segments
-	       * contains the original dst address.
-	       */
-	      new_dst0 = sr0->segments;
-	      ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
-	      ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
-	    }
-
-	do_trace0:
-
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_fix_addr_trace_t *t = vlib_add_trace (vm, node,
-						       b0, sizeof (*t));
-	      t->next_index = next0;
-	      t->adj_index = ~0;
-
-	      if (next0 != SR_FIX_DST_ADDR_NEXT_DROP)
-		{
-		  t->adj_index = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-		  clib_memcpy (t->src.as_u8, ip0->src_address.as_u8,
-			       sizeof (t->src.as_u8));
-		  clib_memcpy (t->dst.as_u8, ip0->dst_address.as_u8,
-			       sizeof (t->dst.as_u8));
-		  clib_memcpy (t->sr, sr0, sizeof (t->sr));
-		}
-	    }
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  return from_frame->n_vectors;
-}
-
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (sr_fix_dst_addr_node) = {
-  .function = sr_fix_dst_addr,
-  .name = "sr-fix-dst-addr",
-  /* Takes a vector of packets. */
-  .vector_size = sizeof (u32),
-  .format_trace = format_sr_fix_addr_trace,
-  .format_buffer = format_ip6_sr_header_with_length,
-
-  .runtime_data_bytes = 0,
-
-  .n_errors = SR_FIX_DST_N_ERROR,
-  .error_strings = sr_fix_dst_error_strings,
-
-  .n_next_nodes = SR_FIX_DST_ADDR_N_NEXT,
-  .next_nodes = {
-#define _(s,n) [SR_FIX_DST_ADDR_NEXT_##s] = n,
-    foreach_sr_fix_dst_addr_next
-#undef _
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (sr_fix_dst_addr_node, sr_fix_dst_addr)
-/* *INDENT-ON* */
-
-static clib_error_t *
-sr_init (vlib_main_t * vm)
-{
-  ip6_sr_main_t *sm = &sr_main;
-  clib_error_t *error = 0;
-  vlib_node_t *ip6_lookup_node, *ip6_rewrite_node;
-
-  if ((error = vlib_call_init_function (vm, ip_main_init)))
-    return error;
-
-  if ((error = vlib_call_init_function (vm, ip6_lookup_init)))
-    return error;
-
-  sm->vlib_main = vm;
-  sm->vnet_main = vnet_get_main ();
-
-  vec_validate (sm->hmac_keys, 0);
-  sm->hmac_keys[0].shared_secret = (u8 *) 0xdeadbeef;
-
-  sm->tunnel_index_by_key =
-    hash_create_mem (0, sizeof (ip6_sr_tunnel_key_t), sizeof (uword));
-
-  sm->tunnel_index_by_name = hash_create_string (0, sizeof (uword));
-
-  sm->policy_index_by_policy_name = hash_create_string (0, sizeof (uword));
-
-  sm->policy_index_by_multicast_address =
-    hash_create_mem (0, sizeof (ip6_address_t), sizeof (uword));
-
-  sm->hmac_key_by_shared_secret = hash_create_string (0, sizeof (uword));
-
-  ip6_register_protocol (IPPROTO_IPV6_ROUTE, sr_local_node.index);
-
-  ip6_lookup_node = vlib_get_node_by_name (vm, (u8 *) "ip6-lookup");
-  ASSERT (ip6_lookup_node);
-
-  ip6_rewrite_node = vlib_get_node_by_name (vm, (u8 *) "ip6-rewrite");
-  ASSERT (ip6_rewrite_node);
-
-  /* Add a disposition to ip6_rewrite for the sr dst address hack node */
-  sm->ip6_rewrite_sr_next_index =
-    vlib_node_add_next (vm, ip6_rewrite_node->index,
-			sr_fix_dst_addr_node.index);
-
-  OpenSSL_add_all_digests ();
-
-  sm->md = (void *) EVP_get_digestbyname ("sha1");
-  sm->hmac_ctx = clib_mem_alloc (sizeof (HMAC_CTX));
-
-  sr_dpo_type = dpo_register_new_type (&sr_dpo_vft, sr_nodes);
-  sr_fib_node_type = fib_node_register_new_type (&sr_fib_vft);
-
-  return error;
-}
-
-VLIB_INIT_FUNCTION (sr_init);
-
-/**
- * @brief Definition of next-nodes for SR local
- */
-#define foreach_sr_local_next                   \
-  _ (ERROR, "error-drop")                       \
-  _ (IP6_LOOKUP, "ip6-lookup")
-
-/**
- * @brief Struct for definition of next-nodes for SR local
- */
-typedef enum
-{
-#define _(s,n) SR_LOCAL_NEXT_##s,
-  foreach_sr_local_next
-#undef _
-    SR_LOCAL_N_NEXT,
-} sr_local_next_t;
-
-/**
- * @brief Struct for packet trace of SR local
- */
-typedef struct
-{
-  u8 next_index;
-  u8 sr_valid;
-  ip6_address_t src, dst;
-  u16 length;
-  u8 sr[256];
-} sr_local_trace_t;
-
-/**
- * @brief Definition of SR local error-strings
- */
-static char *sr_local_error_strings[] = {
-#define sr_error(n,s) s,
-#include "sr_error.def"
-#undef sr_error
-};
-
-/**
- * @brief Struct for definition of SR local error-strings
- */
-typedef enum
-{
-#define sr_error(n,s) SR_LOCAL_ERROR_##n,
-#include "sr_error.def"
-#undef sr_error
-  SR_LOCAL_N_ERROR,
-} sr_local_error_t;
-
-/**
- * @brief Format SR local trace
- *
- * @param s u8 *
- * @param args va_list *
- *
- * @return s u8 *
- */
-u8 *
-format_sr_local_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  sr_local_trace_t *t = va_arg (*args, sr_local_trace_t *);
-
-  s = format (s, "SR-LOCAL: src %U dst %U len %u next_index %d",
-	      format_ip6_address, &t->src,
-	      format_ip6_address, &t->dst, t->length, t->next_index);
-  if (t->sr_valid)
-    s =
-      format (s, "\n  %U", format_ip6_sr_header, t->sr, 1 /* print_hmac */ );
-  else
-    s = format (s, "\n  popped SR header");
-
-  return s;
-}
-
-
-/* $$$$ fixme: smp, don't copy data, cache input, output (maybe) */
-/**
- * @brief Validate the SR HMAC
- *
- * @param sm ip6_sr_main_t *
- * @param ip ip6_header_t *
- * @param sr ip6_sr_header_t *
- *
- * @return retval int
- */
-static int
-sr_validate_hmac (ip6_sr_main_t * sm, ip6_header_t * ip, ip6_sr_header_t * sr)
-{
-  u32 key_index;
-  static u8 *keybuf;
-  u8 *copy_target;
-  int first_segment;
-  ip6_address_t *addrp;
-  int i;
-  ip6_sr_hmac_key_t *hmac_key;
-  static u8 *signature;
-  u32 sig_len;
-
-  key_index = sr->hmac_key;
-
-  /* No signature? Pass... */
-  if (key_index == 0)
-    return 0;
-
-  /* We don't know about this key? Fail... */
-  if (key_index >= vec_len (sm->hmac_keys))
-    return 1;
-
-  vec_validate (signature, SHA256_DIGEST_LENGTH - 1);
-
-  hmac_key = sm->hmac_keys + key_index;
-
-  vec_reset_length (keybuf);
-
-  /* pkt ip6 src address */
-  vec_add2 (keybuf, copy_target, sizeof (ip6_address_t));
-  clib_memcpy (copy_target, ip->src_address.as_u8, sizeof (ip6_address_t));
-
-  /* last segment */
-  vec_add2 (keybuf, copy_target, 1);
-  copy_target[0] = sr->first_segment;
-
-  /* octet w/ bit 0 = "clean" flag */
-  vec_add2 (keybuf, copy_target, 1);
-  copy_target[0]
-    = (sr->flags & clib_host_to_net_u16 (IP6_SR_HEADER_FLAG_CLEANUP))
-    ? 0x80 : 0;
-
-  /* hmac key id */
-  vec_add2 (keybuf, copy_target, 1);
-  copy_target[0] = sr->hmac_key;
-
-  first_segment = sr->first_segment;
-
-  addrp = sr->segments;
-
-  /* segments */
-  for (i = 0; i <= first_segment; i++)
-    {
-      vec_add2 (keybuf, copy_target, sizeof (ip6_address_t));
-      clib_memcpy (copy_target, addrp->as_u8, sizeof (ip6_address_t));
-      addrp++;
-    }
-
-  if (sm->is_debug)
-    clib_warning ("verify key index %d keybuf: %U", key_index,
-		  format_hex_bytes, keybuf, vec_len (keybuf));
-
-  /* shared secret */
-
-  /* SHA1 is shorter than SHA-256 */
-  memset (signature, 0, vec_len (signature));
-
-  HMAC_CTX_init (sm->hmac_ctx);
-  if (!HMAC_Init (sm->hmac_ctx, hmac_key->shared_secret,
-		  vec_len (hmac_key->shared_secret), sm->md))
-    clib_warning ("barf1");
-  if (!HMAC_Update (sm->hmac_ctx, keybuf, vec_len (keybuf)))
-    clib_warning ("barf2");
-  if (!HMAC_Final (sm->hmac_ctx, signature, &sig_len))
-    clib_warning ("barf3");
-  HMAC_CTX_cleanup (sm->hmac_ctx);
-
-  if (sm->is_debug)
-    clib_warning ("computed signature len %d, value %U", sig_len,
-		  format_hex_bytes, signature, vec_len (signature));
-
-  /* Point at the SHA signature in the packet */
-  addrp++;
-  if (sm->is_debug)
-    clib_warning ("read signature %U", format_hex_bytes, addrp,
-		  SHA256_DIGEST_LENGTH);
-
-  return memcmp (signature, addrp, SHA256_DIGEST_LENGTH);
-}
-
-/**
- * @brief SR local node
- * @node sr-local
- *
- * @param vm vlib_main_t *
- * @param node vlib_node_runtime_t *
- * @param from_frame vlib_frame_t *
- *
- * @return from_frame->n_vectors uword
- */
-static uword
-sr_local (vlib_main_t * vm,
-	  vlib_node_runtime_t * node, vlib_frame_t * from_frame)
-{
-  u32 n_left_from, next_index, *from, *to_next;
-  ip6_sr_main_t *sm = &sr_main;
-  u32 (*sr_local_cb) (vlib_main_t *, vlib_node_runtime_t *,
-		      vlib_buffer_t *, ip6_header_t *, ip6_sr_header_t *);
-  sr_local_cb = sm->sr_local_cb;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-	  u32 bi0, bi1;
-	  vlib_buffer_t *b0, *b1;
-	  ip6_header_t *ip0, *ip1;
-	  ip6_sr_header_t *sr0, *sr1;
-	  ip6_address_t *new_dst0, *new_dst1;
-	  u32 next0 = SR_LOCAL_NEXT_IP6_LOOKUP;
-	  u32 next1 = SR_LOCAL_NEXT_IP6_LOOKUP;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t *p2, *p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
-	    CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
-	  }
-
-	  bi0 = from[0];
-	  bi1 = from[1];
-	  to_next[0] = bi0;
-	  to_next[1] = bi1;
-	  from += 2;
-	  to_next += 2;
-	  n_left_to_next -= 2;
-	  n_left_from -= 2;
-
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  ip0 = vlib_buffer_get_current (b0);
-	  sr0 = (ip6_sr_header_t *) (ip0 + 1);
-	  if (PREDICT_FALSE
-	      (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-	    {
-	      ip6_hop_by_hop_ext_t *ext_hdr =
-		(ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-	      sr0 =
-		(ip6_sr_header_t *) ip6_ext_next_header ((ip6_ext_header_t *)
-							 ext_hdr);
-	    }
-
-	  if (PREDICT_FALSE (sr0->type != ROUTING_HEADER_TYPE_SR))
-	    {
-	      next0 = SR_LOCAL_NEXT_ERROR;
-	      b0->error =
-		node->errors[SR_LOCAL_ERROR_BAD_ROUTING_HEADER_TYPE];
-	      goto do_trace0;
-	    }
-
-	  /* Out of segments? Turf the packet */
-	  if (PREDICT_FALSE (sr0->segments_left == 0))
-	    {
-	      next0 = SR_LOCAL_NEXT_ERROR;
-	      b0->error = node->errors[SR_LOCAL_ERROR_NO_MORE_SEGMENTS];
-	      goto do_trace0;
-	    }
-
-	  if (PREDICT_FALSE (sm->validate_hmac))
-	    {
-	      if (sr_validate_hmac (sm, ip0, sr0))
-		{
-		  next0 = SR_LOCAL_NEXT_ERROR;
-		  b0->error = node->errors[SR_LOCAL_ERROR_HMAC_INVALID];
-		  goto do_trace0;
-		}
-	    }
-
-	  next0 = sr_local_cb ? sr_local_cb (vm, node, b0, ip0, sr0) : next0;
-
-	  /*
-	   * To suppress rewrite, return ~SR_LOCAL_NEXT_xxx
-	   */
-	  if (PREDICT_FALSE (next0 & 0x80000000))
-	    {
-	      next0 ^= 0xFFFFFFFF;
-	      if (PREDICT_FALSE (next0 == SR_LOCAL_NEXT_ERROR))
-		b0->error = node->errors[SR_LOCAL_ERROR_APP_CALLBACK];
-	    }
-	  else
-	    {
-	      u32 segment_index0;
-
-	      segment_index0 = sr0->segments_left - 1;
-
-	      /* Rewrite the packet */
-	      new_dst0 = (ip6_address_t *) (sr0->segments + segment_index0);
-	      ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
-	      ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
-
-	      if (PREDICT_TRUE (sr0->segments_left > 0))
-		sr0->segments_left -= 1;
-	    }
-
-	  /* End of the path. Clean up the SR header, or not */
-	  if (PREDICT_FALSE
-	      (sr0->segments_left == 0 &&
-	       (sr0->flags &
-		clib_host_to_net_u16 (IP6_SR_HEADER_FLAG_CLEANUP))))
-	    {
-	      u64 *copy_dst0, *copy_src0;
-	      u16 new_l0;
-	      u32 copy_len_u64s0 = 0;
-	      int i;
-
-	      /*
-	       * Copy the ip6 header right by the (real) length of the
-	       * sr header.
-	       */
-	      if (PREDICT_FALSE
-		  (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-		{
-		  ip6_hop_by_hop_ext_t *ext_hdr =
-		    (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-		  copy_len_u64s0 =
-		    (((ip6_ext_header_t *) ext_hdr)->n_data_u64s) + 1;
-		  ext_hdr->next_hdr = sr0->protocol;
-		}
-	      else
-		{
-		  ip0->protocol = sr0->protocol;
-		}
-	      vlib_buffer_advance (b0, (sr0->length + 1) * 8);
-
-	      new_l0 = clib_net_to_host_u16 (ip0->payload_length) -
-		(sr0->length + 1) * 8;
-	      ip0->payload_length = clib_host_to_net_u16 (new_l0);
-
-	      copy_src0 = (u64 *) ip0;
-	      copy_dst0 = copy_src0 + (sr0->length + 1);
-
-	      copy_dst0[4 + copy_len_u64s0] = copy_src0[4 + copy_len_u64s0];
-	      copy_dst0[3 + copy_len_u64s0] = copy_src0[3 + copy_len_u64s0];
-	      copy_dst0[2 + copy_len_u64s0] = copy_src0[2 + copy_len_u64s0];
-	      copy_dst0[1 + copy_len_u64s0] = copy_src0[1 + copy_len_u64s0];
-	      copy_dst0[0 + copy_len_u64s0] = copy_src0[0 + copy_len_u64s0];
-
-	      for (i = copy_len_u64s0 - 1; i >= 0; i--)
-		{
-		  copy_dst0[i] = copy_src0[i];
-		}
-
-	      sr0 = 0;
-	    }
-
-	do_trace0:
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_local_trace_t *tr = vlib_add_trace (vm, node,
-						     b0, sizeof (*tr));
-	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
-			   sizeof (tr->src.as_u8));
-	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
-			   sizeof (tr->dst.as_u8));
-	      tr->length = vlib_buffer_length_in_chain (vm, b0);
-	      tr->next_index = next0;
-	      tr->sr_valid = sr0 != 0;
-	      if (tr->sr_valid)
-		clib_memcpy (tr->sr, sr0, sizeof (tr->sr));
-	    }
-
-	  b1 = vlib_get_buffer (vm, bi1);
-	  ip1 = vlib_buffer_get_current (b1);
-	  sr1 = (ip6_sr_header_t *) (ip1 + 1);
-	  if (PREDICT_FALSE
-	      (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-	    {
-
-	      ip6_hop_by_hop_ext_t *ext_hdr =
-		(ip6_hop_by_hop_ext_t *) ip6_next_header (ip1);
-	      sr1 =
-		(ip6_sr_header_t *) ip6_ext_next_header ((ip6_ext_header_t *)
-							 ext_hdr);
-	    }
-
-	  if (PREDICT_FALSE (sr1->type != ROUTING_HEADER_TYPE_SR))
-	    {
-	      next1 = SR_LOCAL_NEXT_ERROR;
-	      b1->error =
-		node->errors[SR_LOCAL_ERROR_BAD_ROUTING_HEADER_TYPE];
-	      goto do_trace1;
-	    }
-
-	  /* Out of segments? Turf the packet */
-	  if (PREDICT_FALSE (sr1->segments_left == 0))
-	    {
-	      next1 = SR_LOCAL_NEXT_ERROR;
-	      b1->error = node->errors[SR_LOCAL_ERROR_NO_MORE_SEGMENTS];
-	      goto do_trace1;
-	    }
-
-	  if (PREDICT_FALSE (sm->validate_hmac))
-	    {
-	      if (sr_validate_hmac (sm, ip1, sr1))
-		{
-		  next1 = SR_LOCAL_NEXT_ERROR;
-		  b1->error = node->errors[SR_LOCAL_ERROR_HMAC_INVALID];
-		  goto do_trace1;
-		}
-	    }
-
-	  next1 = sr_local_cb ? sr_local_cb (vm, node, b1, ip1, sr1) : next1;
-
-	  /*
-	   * To suppress rewrite, return ~SR_LOCAL_NEXT_xxx
-	   */
-	  if (PREDICT_FALSE (next1 & 0x80000000))
-	    {
-	      next1 ^= 0xFFFFFFFF;
-	      if (PREDICT_FALSE (next1 == SR_LOCAL_NEXT_ERROR))
-		b1->error = node->errors[SR_LOCAL_ERROR_APP_CALLBACK];
-	    }
-	  else
-	    {
-	      u32 segment_index1;
-
-	      segment_index1 = sr1->segments_left - 1;
-
-	      /* Rewrite the packet */
-	      new_dst1 = (ip6_address_t *) (sr1->segments + segment_index1);
-	      ip1->dst_address.as_u64[0] = new_dst1->as_u64[0];
-	      ip1->dst_address.as_u64[1] = new_dst1->as_u64[1];
-
-	      if (PREDICT_TRUE (sr1->segments_left > 0))
-		sr1->segments_left -= 1;
-	    }
-
-	  /* End of the path. Clean up the SR header, or not */
-	  if (PREDICT_FALSE
-	      (sr1->segments_left == 0 &&
-	       (sr1->flags &
-		clib_host_to_net_u16 (IP6_SR_HEADER_FLAG_CLEANUP))))
-	    {
-	      u64 *copy_dst1, *copy_src1;
-	      u16 new_l1;
-	      u32 copy_len_u64s1 = 0;
-	      int i;
-
-	      /*
-	       * Copy the ip6 header right by the (real) length of the
-	       * sr header.
-	       */
-	      if (PREDICT_FALSE
-		  (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-		{
-		  ip6_hop_by_hop_ext_t *ext_hdr =
-		    (ip6_hop_by_hop_ext_t *) ip6_next_header (ip1);
-		  copy_len_u64s1 =
-		    (((ip6_ext_header_t *) ext_hdr)->n_data_u64s) + 1;
-		  ext_hdr->next_hdr = sr1->protocol;
-		}
-	      else
-		{
-		  ip1->protocol = sr1->protocol;
-		}
-	      vlib_buffer_advance (b1, (sr1->length + 1) * 8);
-
-	      new_l1 = clib_net_to_host_u16 (ip1->payload_length) -
-		(sr1->length + 1) * 8;
-	      ip1->payload_length = clib_host_to_net_u16 (new_l1);
-
-	      copy_src1 = (u64 *) ip1;
-	      copy_dst1 = copy_src1 + (sr1->length + 1);
-
-	      copy_dst1[4 + copy_len_u64s1] = copy_src1[4 + copy_len_u64s1];
-	      copy_dst1[3 + copy_len_u64s1] = copy_src1[3 + copy_len_u64s1];
-	      copy_dst1[2 + copy_len_u64s1] = copy_src1[2 + copy_len_u64s1];
-	      copy_dst1[1 + copy_len_u64s1] = copy_src1[1 + copy_len_u64s1];
-	      copy_dst1[0 + copy_len_u64s1] = copy_src1[0 + copy_len_u64s1];
-
-	      for (i = copy_len_u64s1 - 1; i >= 0; i--)
-		{
-		  copy_dst1[i] = copy_src1[i];
-		}
-
-	      sr1 = 0;
-	    }
-
-	do_trace1:
-	  if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_local_trace_t *tr = vlib_add_trace (vm, node,
-						     b1, sizeof (*tr));
-	      clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
-			   sizeof (tr->src.as_u8));
-	      clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
-			   sizeof (tr->dst.as_u8));
-	      tr->length = vlib_buffer_length_in_chain (vm, b1);
-	      tr->next_index = next1;
-	      tr->sr_valid = sr1 != 0;
-	      if (tr->sr_valid)
-		clib_memcpy (tr->sr, sr1, sizeof (tr->sr));
-	    }
-
-	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, bi1, next0, next1);
-	}
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  ip6_header_t *ip0 = 0;
-	  ip6_sr_header_t *sr0;
-	  ip6_address_t *new_dst0;
-	  u32 next0 = SR_LOCAL_NEXT_IP6_LOOKUP;
-
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  ip0 = vlib_buffer_get_current (b0);
-	  sr0 = (ip6_sr_header_t *) (ip0 + 1);
-
-	  if (PREDICT_FALSE
-	      (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-	    {
-	      ip6_hop_by_hop_ext_t *ext_hdr =
-		(ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-	      sr0 =
-		(ip6_sr_header_t *) ip6_ext_next_header ((ip6_ext_header_t *)
-							 ext_hdr);
-	    }
-	  if (PREDICT_FALSE (sr0->type != ROUTING_HEADER_TYPE_SR))
-	    {
-	      next0 = SR_LOCAL_NEXT_ERROR;
-	      b0->error =
-		node->errors[SR_LOCAL_ERROR_BAD_ROUTING_HEADER_TYPE];
-	      goto do_trace;
-	    }
-
-	  /* Out of segments? Turf the packet */
-	  if (PREDICT_FALSE (sr0->segments_left == 0))
-	    {
-	      next0 = SR_LOCAL_NEXT_ERROR;
-	      b0->error = node->errors[SR_LOCAL_ERROR_NO_MORE_SEGMENTS];
-	      goto do_trace;
-	    }
-
-	  if (PREDICT_FALSE (sm->validate_hmac))
-	    {
-	      if (sr_validate_hmac (sm, ip0, sr0))
-		{
-		  next0 = SR_LOCAL_NEXT_ERROR;
-		  b0->error = node->errors[SR_LOCAL_ERROR_HMAC_INVALID];
-		  goto do_trace;
-		}
-	    }
-
-	  next0 = sr_local_cb ? sr_local_cb (vm, node, b0, ip0, sr0) : next0;
-
-	  /*
-	   * To suppress rewrite, return ~SR_LOCAL_NEXT_xxx
-	   */
-	  if (PREDICT_FALSE (next0 & 0x80000000))
-	    {
-	      next0 ^= 0xFFFFFFFF;
-	      if (PREDICT_FALSE (next0 == SR_LOCAL_NEXT_ERROR))
-		b0->error = node->errors[SR_LOCAL_ERROR_APP_CALLBACK];
-	    }
-	  else
-	    {
-	      u32 segment_index0;
-
-	      segment_index0 = sr0->segments_left - 1;
-
-	      /* Rewrite the packet */
-	      new_dst0 = (ip6_address_t *) (sr0->segments + segment_index0);
-	      ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
-	      ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
-
-	      if (PREDICT_TRUE (sr0->segments_left > 0))
-		sr0->segments_left -= 1;
-	    }
-
-	  /* End of the path. Clean up the SR header, or not */
-	  if (PREDICT_FALSE
-	      (sr0->segments_left == 0 &&
-	       (sr0->flags &
-		clib_host_to_net_u16 (IP6_SR_HEADER_FLAG_CLEANUP))))
-	    {
-	      u64 *copy_dst0, *copy_src0;
-	      u16 new_l0;
-	      u32 copy_len_u64s0 = 0;
-	      int i;
-
-	      /*
-	       * Copy the ip6 header right by the (real) length of the
-	       * sr header.
-	       */
-	      if (PREDICT_FALSE
-		  (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-		{
-		  ip6_hop_by_hop_ext_t *ext_hdr =
-		    (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-		  copy_len_u64s0 =
-		    (((ip6_ext_header_t *) ext_hdr)->n_data_u64s) + 1;
-		  ext_hdr->next_hdr = sr0->protocol;
-		}
-	      else
-		{
-		  ip0->protocol = sr0->protocol;
-		}
-
-	      vlib_buffer_advance (b0, (sr0->length + 1) * 8);
-
-	      new_l0 = clib_net_to_host_u16 (ip0->payload_length) -
-		(sr0->length + 1) * 8;
-	      ip0->payload_length = clib_host_to_net_u16 (new_l0);
-
-	      copy_src0 = (u64 *) ip0;
-	      copy_dst0 = copy_src0 + (sr0->length + 1);
-	      copy_dst0[4 + copy_len_u64s0] = copy_src0[4 + copy_len_u64s0];
-	      copy_dst0[3 + copy_len_u64s0] = copy_src0[3 + copy_len_u64s0];
-	      copy_dst0[2 + copy_len_u64s0] = copy_src0[2 + copy_len_u64s0];
-	      copy_dst0[1 + copy_len_u64s0] = copy_src0[1 + copy_len_u64s0];
-	      copy_dst0[0 + copy_len_u64s0] = copy_src0[0 + copy_len_u64s0];
-
-	      for (i = copy_len_u64s0 - 1; i >= 0; i--)
-		{
-		  copy_dst0[i] = copy_src0[i];
-		}
-
-	      sr0 = 0;
-	    }
-
-	do_trace:
-	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      sr_local_trace_t *tr = vlib_add_trace (vm, node,
-						     b0, sizeof (*tr));
-	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
-			   sizeof (tr->src.as_u8));
-	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
-			   sizeof (tr->dst.as_u8));
-	      tr->length = vlib_buffer_length_in_chain (vm, b0);
-	      tr->next_index = next0;
-	      tr->sr_valid = sr0 != 0;
-	      if (tr->sr_valid)
-		clib_memcpy (tr->sr, sr0, sizeof (tr->sr));
-	    }
-
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  vlib_node_increment_counter (vm, sr_local_node.index,
-			       SR_LOCAL_ERROR_PKTS_PROCESSED,
-			       from_frame->n_vectors);
-  return from_frame->n_vectors;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (sr_local_node, static) = {
-  .function = sr_local,
-  .name = "sr-local",
-  /* Takes a vector of packets. */
-  .vector_size = sizeof (u32),
-  .format_trace = format_sr_local_trace,
-
-  .runtime_data_bytes = 0,
-
-  .n_errors = SR_LOCAL_N_ERROR,
-  .error_strings = sr_local_error_strings,
-
-  .n_next_nodes = SR_LOCAL_N_NEXT,
-  .next_nodes = {
-#define _(s,n) [SR_LOCAL_NEXT_##s] = n,
-    foreach_sr_local_next
-#undef _
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (sr_local_node, sr_local)
-/* *INDENT-ON* */
-
-ip6_sr_main_t *
-sr_get_main (vlib_main_t * vm)
-{
-  vlib_call_init_function (vm, sr_init);
-  ASSERT (sr_local_node.index);
-  return &sr_main;
-}
-
-/**
- * @brief CLI parser for SR fix destination rewrite node
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-set_ip6_sr_rewrite_fn (vlib_main_t * vm,
-		       unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  fib_prefix_t pfx = {
-    .fp_proto = FIB_PROTOCOL_IP6,
-    .fp_len = 128,
-  };
-  u32 fib_index = 0;
-  u32 fib_id = 0;
-  u32 adj_index;
-  ip_adjacency_t *adj;
-  vnet_hw_interface_t *hi;
-  u32 sw_if_index;
-  ip6_sr_main_t *sm = &sr_main;
-  vnet_main_t *vnm = vnet_get_main ();
-  fib_node_index_t fei;
-
-  if (!unformat (input, "%U", unformat_ip6_address, &pfx.fp_addr.ip6))
-    return clib_error_return (0, "ip6 address missing in '%U'",
-			      format_unformat_error, input);
-
-  if (unformat (input, "rx-table-id %d", &fib_id))
-    {
-      fib_index = fib_table_id_find_fib_index (FIB_PROTOCOL_IP6, fib_id);
-      if (fib_index == ~0)
-	return clib_error_return (0, "fib-id %d not found", fib_id);
-    }
-
-  fei = fib_table_lookup_exact_match (fib_index, &pfx);
-
-  if (FIB_NODE_INDEX_INVALID == fei)
-    return clib_error_return (0, "no match for %U",
-			      format_ip6_address, &pfx.fp_addr.ip6);
-
-  adj_index = fib_entry_get_adj_for_source (fei, FIB_SOURCE_SR);
-
-  if (ADJ_INDEX_INVALID == adj_index)
-    return clib_error_return (0, "%U not SR sourced",
-			      format_ip6_address, &pfx.fp_addr.ip6);
-
-  adj = adj_get (adj_index);
-
-  if (adj->lookup_next_index != IP_LOOKUP_NEXT_REWRITE)
-    return clib_error_return (0, "%U unresolved (not a rewrite adj)",
-			      format_ip6_address, &pfx.fp_addr.ip6);
-
-  adj->rewrite_header.next_index = sm->ip6_rewrite_sr_next_index;
-
-  sw_if_index = adj->rewrite_header.sw_if_index;
-  hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
-  adj->rewrite_header.node_index = sr_fix_dst_addr_node.index;
-
-  /* $$$$$ hack... steal the interface address index */
-  adj->if_address_index =
-    vlib_node_add_next (vm, sr_fix_dst_addr_node.index,
-			hi->output_node_index);
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (set_ip6_sr_rewrite, static) = {
-    .path = "set ip6 sr rewrite",
-    .short_help = "set ip6 sr rewrite  [fib-id ]",
-    .function = set_ip6_sr_rewrite_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Register a callback routine to set next0 in sr_local
- *
- * @param cb void *
- */
-void
-vnet_register_sr_app_callback (void *cb)
-{
-  ip6_sr_main_t *sm = &sr_main;
-
-  sm->sr_local_cb = cb;
-}
-
-/**
- * @brief Test routine for validation of HMAC
- */
-static clib_error_t *
-test_sr_hmac_validate_fn (vlib_main_t * vm,
-			  unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  ip6_sr_main_t *sm = &sr_main;
-
-  if (unformat (input, "validate on"))
-    sm->validate_hmac = 1;
-  else if (unformat (input, "chunk-offset off"))
-    sm->validate_hmac = 0;
-  else
-    return clib_error_return (0, "expected validate on|off in '%U'",
-			      format_unformat_error, input);
-
-  vlib_cli_output (vm, "hmac signature validation %s",
-		   sm->validate_hmac ? "on" : "off");
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (test_sr_hmac_validate, static) = {
-    .path = "test sr hmac",
-    .short_help = "test sr hmac validate [on|off]",
-    .function = test_sr_hmac_validate_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Add or Delete HMAC key
- *
- * @param sm ip6_sr_main_t *
- * @param key_id u32
- * @param shared_secret u8 *
- * @param is_del u8
- *
- * @return retval i32
- */
-// $$$ fixme shouldn't return i32
-i32
-sr_hmac_add_del_key (ip6_sr_main_t * sm, u32 key_id, u8 * shared_secret,
-		     u8 is_del)
-{
-  u32 index;
-  ip6_sr_hmac_key_t *key;
-
-  if (is_del == 0)
-    {
-      /* Specific key in use? Fail. */
-      if (key_id && vec_len (sm->hmac_keys) > key_id
-	  && sm->hmac_keys[key_id].shared_secret)
-	return -2;
-
-      index = key_id;
-      key = find_or_add_shared_secret (sm, shared_secret, &index);
-      ASSERT (index == key_id);
-      return 0;
-    }
-
-  /* delete */
-
-  if (key_id)			/* delete by key ID */
-    {
-      if (vec_len (sm->hmac_keys) <= key_id)
-	return -3;
-
-      key = sm->hmac_keys + key_id;
-
-      hash_unset_mem (sm->hmac_key_by_shared_secret, key->shared_secret);
-      vec_free (key->shared_secret);
-      return 0;
-    }
-
-  index = 0;
-  key = find_or_add_shared_secret (sm, shared_secret, &index);
-  hash_unset_mem (sm->hmac_key_by_shared_secret, key->shared_secret);
-  vec_free (key->shared_secret);
-  return 0;
-}
-
-
-static clib_error_t *
-sr_hmac_add_del_key_fn (vlib_main_t * vm,
-			unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  ip6_sr_main_t *sm = &sr_main;
-  u8 is_del = 0;
-  u32 key_id = 0;
-  u8 key_id_set = 0;
-  u8 *shared_secret = 0;
-  i32 rv;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "del"))
-	is_del = 1;
-      else if (unformat (input, "id %d", &key_id))
-	key_id_set = 1;
-      else if (unformat (input, "key %s", &shared_secret))
-	{
-	  /* Do not include the trailing NULL byte. Guaranteed interop issue */
-	  _vec_len (shared_secret) -= 1;
-	}
-      else
-	break;
-    }
-
-  if (is_del == 0 && shared_secret == 0)
-    return clib_error_return (0, "shared secret must be set to add a key");
-
-  if (shared_secret == 0 && key_id_set == 0)
-    return clib_error_return (0, "shared secret and key id both unset");
-
-  rv = sr_hmac_add_del_key (sm, key_id, shared_secret, is_del);
-
-  vec_free (shared_secret);
-
-  switch (rv)
-    {
-    case 0:
-      break;
-
-    default:
-      return clib_error_return (0, "sr_hmac_add_del_key returned %d", rv);
-    }
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (sr_hmac, static) = {
-    .path = "sr hmac",
-    .short_help = "sr hmac [del] id  key ",
-    .function = sr_hmac_add_del_key_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief CLI parser for show HMAC key shared secrets
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-show_sr_hmac_fn (vlib_main_t * vm,
-		 unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  ip6_sr_main_t *sm = &sr_main;
-  int i;
-
-  for (i = 1; i < vec_len (sm->hmac_keys); i++)
-    {
-      if (sm->hmac_keys[i].shared_secret)
-	vlib_cli_output (vm, "[%d]: %v", i, sm->hmac_keys[i].shared_secret);
-    }
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (show_sr_hmac, static) = {
-    .path = "show sr hmac",
-    .short_help = "show sr hmac",
-    .function = show_sr_hmac_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Test for SR debug flag
- *
- * @param vm vlib_main_t *
- * @param input unformat_input_t *
- * @param cmd vlib_cli_command_t *
- *
- * @return error clib_error_t *
- */
-static clib_error_t *
-test_sr_debug_fn (vlib_main_t * vm,
-		  unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  ip6_sr_main_t *sm = &sr_main;
-
-  if (unformat (input, "on"))
-    sm->is_debug = 1;
-  else if (unformat (input, "off"))
-    sm->is_debug = 0;
-  else
-    return clib_error_return (0, "expected on|off in '%U'",
-			      format_unformat_error, input);
-
-  vlib_cli_output (vm, "debug trace now %s", sm->is_debug ? "on" : "off");
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (test_sr_debug, static) = {
-    .path = "test sr debug",
-    .short_help = "test sr debug on|off",
-    .function = test_sr_debug_fn,
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
+* fd.io coding-style-patch-verification: ON
+*
+* Local Variables:
+* eval: (c-set-style "gnu")
+* End:
+*/
diff --git a/src/vnet/sr/sr.h b/src/vnet/sr/sr.h
old mode 100644
new mode 100755
index 3c50b735..eb781e4b
--- a/src/vnet/sr/sr.h
+++ b/src/vnet/sr/sr.h
@@ -12,256 +12,283 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 /**
  * @file
- * @brief Segment Routing header
+ * @brief Segment Routing data structures definitions
+ *
  */
+
 #ifndef included_vnet_sr_h
 #define included_vnet_sr_h
 
 #include 
 #include 
 #include 
+#include 
 
-#include 
 #include 
 #include 
 
-#include 
-#include 
-#include 
-#include 
+#define IPv6_DEFAULT_HEADER_LENGTH 40
+#define IPv6_DEFAULT_HOP_LIMIT 64
+#define IPv6_DEFAULT_MAX_MASK_WIDTH 128
 
-/**
- *    @brief Segment Route tunnel key
- */
-typedef struct
-{
-  ip6_address_t src;
-  ip6_address_t dst;
-} ip6_sr_tunnel_key_t;
+#define SR_BEHAVIOR_END 1
+#define SR_BEHAVIOR_X 2
+#define SR_BEHAVIOR_D_FIRST 3	/* Unused. Separator in between regular and D */
+#define SR_BEHAVIOR_DX2 4
+#define SR_BEHAVIOR_DX6 5
+#define SR_BEHAVIOR_DX4 6
+#define SR_BEHAVIOR_DT6 7
+#define SR_BEHAVIOR_DT4 8
+#define SR_BEHAVIOR_LAST 9	/* Must always be the last one */
+
+#define SR_STEER_L2 2
+#define SR_STEER_IPV4 4
+#define SR_STEER_IPV6 6
+
+#define SR_FUNCTION_SIZE 4
+#define SR_ARGUMENT_SIZE 4
+
+#define SR_SEGMENT_LIST_WEIGHT_DEFAULT 1
 
 /**
- * @brief Segment Route tunnel
+ * @brief SR Segment List (SID list)
  */
 typedef struct
 {
-  /** src, dst address */
-  ip6_sr_tunnel_key_t key;
-
-  /** Pptional tunnel name */
-  u8 *name;
+  ip6_address_t *segments;		/**< SIDs (key) */
 
-  /** Mask width for FIB entry */
-  u32 dst_mask_width;
+  u32 weight;						/**< SID list weight (wECMP / UCMP) */
 
-  /** First hop, to save 1 elt in the segment list */
-  ip6_address_t first_hop;
+  u8 *rewrite;					/**< Precomputed rewrite header */
+  u8 *rewrite_bsid;				/**< Precomputed rewrite header for bindingSID */
 
-  /** RX Fib index */
-  u32 rx_fib_index;
-  /** TX Fib index */
-  u32 tx_fib_index;
+  dpo_id_t bsid_dpo;				/**< DPO for Encaps/Insert for BSID */
+  dpo_id_t ip6_dpo;				/**< DPO for Encaps/Insert IPv6 */
+  dpo_id_t ip4_dpo;				/**< DPO for Encaps IPv6 */
+} ip6_sr_sl_t;
 
-  /** The actual ip6 SR header */
-  u8 *rewrite;
+/* SR policy types */
+#define SR_POLICY_TYPE_DEFAULT 0
+#define SR_POLICY_TYPE_SPRAY 1
+/**
+ * @brief SR Policy
+ */
+typedef struct
+{
+  u32 *segments_lists;		/**< SID lists indexes (vector) */
 
-  /** Indicates that this tunnel is part of a policy comprising
-     of multiple tunnels. If == ~0 tunnel is not part of a policy */
-  u32 policy_index;
+  ip6_address_t bsid;			/**< BindingSID (key) */
 
-  /**
-   * The FIB node graph linkage
-   */
-  fib_node_t node;
+  u8 type;					/**< Type (default is 0) */
 
-  /**
-   * The FIB entry index for the first hop. We track this so we
-   * don't need an extra lookup for it in the data plane
-   */
-  fib_node_index_t fib_entry_index;
+  /* SR Policy specific DPO                                                                               */
+  /* IF Type = DEFAULT Then Load Balancer DPO among SID lists     */
+  /* IF Type = SPRAY then Spray DPO with all SID lists                    */
+  dpo_id_t bsid_dpo;			/**< SR Policy specific DPO - BSID */
+  dpo_id_t ip4_dpo;			/**< SR Policy specific DPO - IPv6 */
+  dpo_id_t ip6_dpo;			/**< SR Policy specific DPO - IPv4 */
 
-  /**
-   * This tunnel's sibling index in the children of the FIB entry
-   */
-  u32 sibling_index;
+  u32 fib_table;			/**< FIB table */
 
-  /**
-   * The DPO contributed by the first-hop FIB entry.
-   */
-  dpo_id_t first_hop_dpo;
-} ip6_sr_tunnel_t;
+  u8 is_encap;				/**< Mode (0 is SRH insert, 1 Encaps) */
+} ip6_sr_policy_t;
 
 /**
- * @brief Shared secret for keyed-hash message authentication code (HMAC).
+ * @brief SR LocalSID
  */
 typedef struct
 {
-  u8 *shared_secret;
-} ip6_sr_hmac_key_t;
+  ip6_address_t localsid;		/**< LocalSID IPv6 address */
 
-/**
- * @brief Args required for add/del tunnel.
- *
- * Else we end up passing a LOT of parameters around.
- */
-typedef struct
-{
-  /** Key (header imposition case) */
-  ip6_address_t *src_address;
-  ip6_address_t *dst_address;
-  u32 dst_mask_width;
-  u32 rx_table_id;
-  u32 tx_table_id;
+  char end_psp;					/**< Combined with End.PSP? */
 
-  /** optional name argument - for referencing SR tunnel/policy by name */
-  u8 *name;
+  u16 behavior;					/**< Behavior associated to this localsid */
 
-  /** optional policy name */
-  u8 *policy_name;
+  union
+  {
+    u32 sw_if_index;				/**< xconnect only */
+    u32 vrf_index;				/**< vrf only */
+  };
 
-  /** segment list, when inserting an ip6 SR header */
-  ip6_address_t *segments;
+  u32 fib_table;				/**< FIB table where localsid is registered */
 
-  /**
-   * "Tag" list, aka segments inserted at the end of the list,
-   * past last_seg
-   */
-  ip6_address_t *tags;
+  u32 vlan_index;				/**< VLAN tag (not an index) */
 
-  /** Shared secret => generate SHA-256 HMAC security fields */
-  u8 *shared_secret;
+  ip46_address_t next_hop;		/**< Next_hop for xconnect usage only */
 
-  /** Flags, e.g. cleanup, policy-list flags */
-  u16 flags_net_byte_order;
+  u32 nh_adj;						/**< Next_adj for xconnect usage only */
 
-  /** Delete the tunnnel? */
-  u8 is_del;
-} ip6_sr_add_del_tunnel_args_t;
+  void *plugin_mem;				/**< Memory to be used by the plugin callback functions */
+} ip6_sr_localsid_t;
+
+typedef int (sr_plugin_callback_t) (ip6_sr_localsid_t * localsid);
 
 /**
- * @brief Args for creating a policy.
- *
- * Typically used for multicast replication.
- * ie a multicast address can be associated with a policy,
- * then replicated across a number of unicast SR tunnels.
+ * @brief SR LocalSID behavior registration
  */
 typedef struct
 {
-  /** policy name */
-  u8 *name;
+  u16 sr_localsid_function_number;			/**< SR LocalSID plugin function (>SR_BEHAVIOR_LAST) */
 
-  /** tunnel names */
-  u8 **tunnel_names;
+  u8 *function_name;							/**< Function name. (key). */
 
-  /** Delete the policy? */
-  u8 is_del;
-} ip6_sr_add_del_policy_args_t;
+  u8 *keyword_str;							/**< Behavior keyword (i.e. End.X) */
+
+  u8 *def_str;								/**< Behavior definition (i.e. Endpoint with cross-connect) */
+
+  u8 *params_str;							/**< Behavior parameters (i.e.  ) */
+
+  dpo_type_t dpo;							/**< DPO type registration */
+
+  format_function_t *ls_format;				/**< LocalSID format function */
+
+  unformat_function_t *ls_unformat;			/**< LocalSID unformat function */
+
+  sr_plugin_callback_t *creation;			/**< Function within plugin that will be called after localsid creation*/
+
+  sr_plugin_callback_t *removal;			/**< Function within plugin that will be called before localsid removal */
+} sr_localsid_fn_registration_t;
 
 /**
- * @brief Segment Routing policy.
+ * @brief Steering db key
  *
- * Typically used for multicast replication.
- * ie a multicast address can be associated with a policy,
- * then replicated across a number of unicast SR tunnels.
+ * L3 is IPv4/IPv6 + mask
+ * L2 is sf_if_index + vlan
  */
 typedef struct
 {
-  /** name of policy */
-  u8 *name;
-
-  /** vector to SR tunnel index */
-  u32 *tunnel_indices;
+  union
+  {
+    struct
+    {
+      ip46_address_t prefix;			/**< IP address of the prefix */
+      u32 mask_width;					/**< Mask width of the prefix */
+      u32 fib_table;					/**< VRF of the prefix */
+    } l3;
+    struct
+    {
+      u32 sw_if_index;					/**< Incoming software interface */
+    } l2;
+  };
+  u8 traffic_type;					/**< Traffic type (IPv4, IPv6, L2) */
+} sr_steering_key_t;
 
-} ip6_sr_policy_t;
+typedef struct
+{
+  sr_steering_key_t classify;		/**< Traffic classification */
+  u32 sr_policy;					/**< SR Policy index */
+} ip6_sr_steering_policy_t;
 
 /**
- * @brief Args for mapping of multicast address to policy name.
- *
- * Typically used for multicast replication.
- * ie a multicast address can be associated with a policy,
- * then replicated across a number of unicast SR tunnels.
+ * @brief Segment Routing main datastructure
  */
 typedef struct
 {
-  /** multicast IP6 address */
-  ip6_address_t *multicast_address;
+  /* ip6-lookup next index for imposition FIB entries */
+  u32 ip6_lookup_sr_next_index;
 
-  /** name of policy to map to */
-  u8 *policy_name;
+  /* ip6-replicate next index for multicast tunnel */
+  u32 ip6_lookup_sr_spray_index;
 
-  /** Delete the mapping */
-  u8 is_del;
+  /* IP4-lookup -> SR rewrite next index */
+  u32 ip4_lookup_sr_policy_rewrite_encaps_index;
+  u32 ip4_lookup_sr_policy_rewrite_insert_index;
 
-} ip6_sr_add_del_multicastmap_args_t;
+  /* IP6-lookup -> SR rewrite next index */
+  u32 ip6_lookup_sr_policy_rewrite_encaps_index;
+  u32 ip6_lookup_sr_policy_rewrite_insert_index;
 
-/**
- * @brief Segment Routing state.
- */
-typedef struct
-{
-  /** pool of tunnel instances, sr entry only */
-  ip6_sr_tunnel_t *tunnels;
+  /* L2-input -> SR rewrite next index */
+  u32 l2_sr_policy_rewrite_index;
+
+  /* IP6-lookup -> SR LocalSID (SR End processing) index */
+  u32 ip6_lookup_sr_localsid_index;
 
-  /** find an sr "tunnel" by its outer-IP src/dst */
-  uword *tunnel_index_by_key;
+  /* SR SID lists */
+  ip6_sr_sl_t *sid_lists;
 
-  /** find an sr "tunnel" by its name */
-  uword *tunnel_index_by_name;
+  /* SR policies */
+  ip6_sr_policy_t *sr_policies;
 
-  /** policy pool */
-  ip6_sr_policy_t *policies;
+  /* Find an SR policy by its BindingSID */
+  ip6_address_t *sr_policy_index_by_key;
 
-  /** find a policy by name */
-  uword *policy_index_by_policy_name;
+  /* Pool of SR localsid instances */
+  ip6_sr_localsid_t *localsids;
 
-  /** multicast address to policy mapping */
-  uword *policy_index_by_multicast_address;
+  /* Find a SR localsid instance based on its functionID */
+  ip6_address_t *localsids_index_by_key;
 
-  /** hmac key id by shared secret */
-  uword *hmac_key_by_shared_secret;
+  /* Pool of SR steer policies instances */
+  ip6_sr_steering_policy_t *steer_policies;
 
-  /** ip6-rewrite next index for reinstalling the original dst address */
-  u32 ip6_rewrite_sr_next_index;
+  /* Find a steer policy based on its classifier */
+  sr_steering_key_t *steer_policies_index_by_key;
 
-  /** application API callback */
-  void *sr_local_cb;
+  /* L2 steering ifaces - sr_policies */
+  u32 *sw_iface_sr_policies;
 
-  /** validate hmac keys */
-  u8 validate_hmac;
+  /* Spray DPO */
+  dpo_type_t sr_pr_spray_dpo_type;
 
-  /** pool of hmac keys */
-  ip6_sr_hmac_key_t *hmac_keys;
+  /* Plugin functions */
+  sr_localsid_fn_registration_t *plugin_functions;
 
-  /** Openssl var */
-  EVP_MD *md;
-  /** Openssl var */
-  HMAC_CTX *hmac_ctx;
+  /* Find plugin function by name */
+  uword *plugin_functions_by_key;
 
-  /** enable debug spew */
-  u8 is_debug;
+  /* Counters */
+  vlib_combined_counter_main_t sr_ls_valid_counters;
+  vlib_combined_counter_main_t sr_ls_invalid_counters;
 
-  /** convenience */
+  /* SR Policies FIBs */
+  u32 fib_table_ip6;
+  u32 fib_table_ip4;
+
+  /* convenience */
   vlib_main_t *vlib_main;
-  /** convenience */
   vnet_main_t *vnet_main;
 } ip6_sr_main_t;
 
 ip6_sr_main_t sr_main;
 
-format_function_t format_ip6_sr_header;
-format_function_t format_ip6_sr_header_with_length;
-
-vlib_node_registration_t ip6_sr_input_node;
-
-int ip6_sr_add_del_tunnel (ip6_sr_add_del_tunnel_args_t * a);
-int ip6_sr_add_del_policy (ip6_sr_add_del_policy_args_t * a);
-int ip6_sr_add_del_multicastmap (ip6_sr_add_del_multicastmap_args_t * a);
-
-void vnet_register_sr_app_callback (void *cb);
-
-void sr_fix_hmac (ip6_sr_main_t * sm, ip6_header_t * ip,
-		  ip6_sr_header_t * sr);
+extern vlib_node_registration_t sr_policy_rewrite_encaps_node;
+extern vlib_node_registration_t sr_policy_rewrite_insert_node;
+extern vlib_node_registration_t sr_localsid_node;
+extern vlib_node_registration_t sr_localsid_d_node;
+
+void sr_dpo_lock (dpo_id_t * dpo);
+void sr_dpo_unlock (dpo_id_t * dpo);
+
+int sr_localsid_register_function (vlib_main_t * vm, u8 * fn_name,
+				   u8 * keyword_str, u8 * def_str,
+				   u8 * params_str, dpo_type_t * dpo,
+				   format_function_t * ls_format,
+				   unformat_function_t * ls_unformat,
+				   sr_plugin_callback_t * creation_fn,
+				   sr_plugin_callback_t * removal_fn);
+
+int
+sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments,
+	       u32 weight, u8 behavior, u32 fib_table, u8 is_encap);
+int
+sr_policy_mod (ip6_address_t * bsid, u32 index, u32 fib_table,
+	       u8 operation, ip6_address_t * segments, u32 sl_index,
+	       u32 weight);
+int sr_policy_del (ip6_address_t * bsid, u32 index);
+
+int sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
+		     char end_psp, u8 behavior, u32 sw_if_index,
+		     u32 vlan_index, u32 fib_table, ip46_address_t * nh_addr,
+		     void *ls_plugin_mem);
+
+int
+sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
+		    u32 table_id, ip46_address_t * prefix, u32 mask_width,
+		    u32 sw_if_index, u8 traffic_type);
 
 #endif /* included_vnet_sr_h */
 
diff --git a/src/vnet/sr/sr_api.c b/src/vnet/sr/sr_api.c
index bab0fc84..f4e1c346 100644
--- a/src/vnet/sr/sr_api.c
+++ b/src/vnet/sr/sr_api.c
@@ -18,6 +18,7 @@
  */
 
 #include 
+#include 
 #include 
 
 #include 
@@ -43,159 +44,129 @@
 #include 
 
 #define foreach_vpe_api_msg                             \
-_(SR_MULTICAST_MAP_ADD_DEL, sr_multicast_map_add_del)
-
-static void vl_api_sr_tunnel_add_del_t_handler
-  (vl_api_sr_tunnel_add_del_t * mp)
+_(SR_LOCALSID_ADD_DEL, sr_localsid_add_del)             \
+_(SR_POLICY_DEL, sr_policy_del)                         \
+_(SR_STEERING_ADD_DEL, sr_steering_add_del)
+//_(SR_LOCALSIDS, sr_localsids_dump)
+//_(SR_LOCALSID_BEHAVIORS, sr_localsid_behaviors_dump)
+
+static void vl_api_sr_localsid_add_del_t_handler
+  (vl_api_sr_localsid_add_del_t * mp)
 {
-#if IP6SR == 0
-  clib_warning ("unimplemented");
-#else
-  ip6_sr_add_del_tunnel_args_t _a, *a = &_a;
+  vl_api_sr_localsid_add_del_reply_t *rmp;
   int rv = 0;
-  vl_api_sr_tunnel_add_del_reply_t *rmp;
-  ip6_address_t *segments = 0, *seg;
-  ip6_address_t *tags = 0, *tag;
-  ip6_address_t *this_address;
-  int i;
+/*
+ * int sr_cli_localsid (char is_del, ip6_address_t *localsid_addr,
+ *  char end_psp, u8 behavior, u32 sw_if_index, u32 vlan_index, u32 fib_table,
+ *  ip46_address_t *nh_addr, void *ls_plugin_mem)
+ */
+  rv = sr_cli_localsid (mp->is_del,
+			(ip6_address_t *) & mp->localsid_addr,
+			mp->end_psp,
+			mp->behavior,
+			ntohl (mp->sw_if_index),
+			ntohl (mp->vlan_index),
+			ntohl (mp->fib_table),
+			(ip46_address_t *) & mp->nh_addr, NULL);
+
+  REPLY_MACRO (VL_API_SR_LOCALSID_ADD_DEL_REPLY);
+}
 
-  if (mp->n_segments == 0)
-    {
-      rv = -11;
-      goto out;
-    }
+static void
+vl_api_sr_policy_add_t_handler (vl_api_sr_policy_add_t * mp)
+{
+  vl_api_sr_policy_add_reply_t *rmp;
+  ip6_address_t *segments = 0, *seg;
+  ip6_address_t *this_address = (ip6_address_t *) mp->segments;
 
-  memset (a, 0, sizeof (*a));
-  a->src_address = (ip6_address_t *) & mp->src_address;
-  a->dst_address = (ip6_address_t *) & mp->dst_address;
-  a->dst_mask_width = mp->dst_mask_width;
-  a->flags_net_byte_order = mp->flags_net_byte_order;
-  a->is_del = (mp->is_add == 0);
-  a->rx_table_id = ntohl (mp->outer_vrf_id);
-  a->tx_table_id = ntohl (mp->inner_vrf_id);
-
-  a->name = format (0, "%s", mp->name);
-  if (!(vec_len (a->name)))
-    a->name = 0;
-
-  a->policy_name = format (0, "%s", mp->policy_name);
-  if (!(vec_len (a->policy_name)))
-    a->policy_name = 0;
-
-  /* Yank segments and tags out of the API message */
-  this_address = (ip6_address_t *) mp->segs_and_tags;
+  int i;
   for (i = 0; i < mp->n_segments; i++)
     {
       vec_add2 (segments, seg, 1);
       clib_memcpy (seg->as_u8, this_address->as_u8, sizeof (*this_address));
       this_address++;
     }
-  for (i = 0; i < mp->n_tags; i++)
-    {
-      vec_add2 (tags, tag, 1);
-      clib_memcpy (tag->as_u8, this_address->as_u8, sizeof (*this_address));
-      this_address++;
-    }
 
-  a->segments = segments;
-  a->tags = tags;
-
-  rv = ip6_sr_add_del_tunnel (a);
-
-out:
+/*
+ * sr_policy_add (ip6_address_t *bsid, ip6_address_t *segments,
+ *                u32 weight, u8 behavior, u32 fib_table, u8 is_encap)
+ */
+  int rv = 0;
+  rv = sr_policy_add ((ip6_address_t *) & mp->bsid_addr,
+		      segments,
+		      ntohl (mp->weight),
+		      mp->type, ntohl (mp->fib_table), mp->is_encap);
 
-  REPLY_MACRO (VL_API_SR_TUNNEL_ADD_DEL_REPLY);
-#endif
+  REPLY_MACRO (VL_API_SR_POLICY_ADD_REPLY);
 }
 
-static void vl_api_sr_policy_add_del_t_handler
-  (vl_api_sr_policy_add_del_t * mp)
+static void
+vl_api_sr_policy_mod_t_handler (vl_api_sr_policy_mod_t * mp)
 {
-#if IP6SR == 0
-  clib_warning ("unimplemented");
-#else
-  ip6_sr_add_del_policy_args_t _a, *a = &_a;
-  int rv = 0;
-  vl_api_sr_policy_add_del_reply_t *rmp;
-  int i;
-
-  memset (a, 0, sizeof (*a));
-  a->is_del = (mp->is_add == 0);
-
-  a->name = format (0, "%s", mp->name);
-  if (!(vec_len (a->name)))
-    {
-      rv = VNET_API_ERROR_NO_SUCH_NODE2;
-      goto out;
-    }
-
-  if (!(mp->tunnel_names[0]))
-    {
-      rv = VNET_API_ERROR_NO_SUCH_NODE2;
-      goto out;
-    }
+  vl_api_sr_policy_mod_reply_t *rmp;
 
-  // start deserializing tunnel_names
-  int num_tunnels = mp->tunnel_names[0];	//number of tunnels
-  u8 *deser_tun_names = mp->tunnel_names;
-  deser_tun_names += 1;		//moving along
-
-  u8 *tun_name = 0;
-  int tun_name_len = 0;
+  ip6_address_t *segments = 0, *seg;
+  ip6_address_t *this_address = (ip6_address_t *) mp->segments;
 
-  for (i = 0; i < num_tunnels; i++)
+  int i;
+  for (i = 0; i < mp->n_segments; i++)
     {
-      tun_name_len = *deser_tun_names;
-      deser_tun_names += 1;
-      vec_resize (tun_name, tun_name_len);
-      memcpy (tun_name, deser_tun_names, tun_name_len);
-      vec_add1 (a->tunnel_names, tun_name);
-      deser_tun_names += tun_name_len;
-      tun_name = 0;
+      vec_add2 (segments, seg, 1);
+      clib_memcpy (seg->as_u8, this_address->as_u8, sizeof (*this_address));
+      this_address++;
     }
 
-  rv = ip6_sr_add_del_policy (a);
-
-out:
+  int rv = 0;
+/*
+ * int
+ * sr_policy_mod(ip6_address_t *bsid, u32 index, u32 fib_table,
+ *               u8 operation, ip6_address_t *segments, u32 sl_index,
+ *               u32 weight, u8 is_encap)
+ */
+  rv = sr_policy_mod ((ip6_address_t *) & mp->bsid_addr,
+		      ntohl (mp->sr_policy_index),
+		      ntohl (mp->fib_table),
+		      mp->operation,
+		      segments, ntohl (mp->sl_index), ntohl (mp->weight));
 
-  REPLY_MACRO (VL_API_SR_POLICY_ADD_DEL_REPLY);
-#endif
+  REPLY_MACRO (VL_API_SR_POLICY_MOD_REPLY);
 }
 
-static void vl_api_sr_multicast_map_add_del_t_handler
-  (vl_api_sr_multicast_map_add_del_t * mp)
+static void
+vl_api_sr_policy_del_t_handler (vl_api_sr_policy_del_t * mp)
 {
-#if IP6SR == 0
-  clib_warning ("unimplemented");
-#else
-  ip6_sr_add_del_multicastmap_args_t _a, *a = &_a;
+  vl_api_sr_policy_del_reply_t *rmp;
   int rv = 0;
-  vl_api_sr_multicast_map_add_del_reply_t *rmp;
-
-  memset (a, 0, sizeof (*a));
-  a->is_del = (mp->is_add == 0);
-
-  a->multicast_address = (ip6_address_t *) & mp->multicast_address;
-  a->policy_name = format (0, "%s", mp->policy_name);
-
-  if (a->multicast_address == 0)
-    {
-      rv = -1;
-      goto out;
-    }
-
-  if (!(a->policy_name))
-    {
-      rv = -2;
-      goto out;
-    }
-
-  rv = ip6_sr_add_del_multicastmap (a);
+/*
+ * int
+ * sr_policy_del (ip6_address_t *bsid, u32 index)
+ */
+  rv = sr_policy_del ((ip6_address_t *) & mp->bsid_addr,
+		      ntohl (mp->sr_policy_index));
 
-out:
+  REPLY_MACRO (VL_API_SR_POLICY_DEL_REPLY);
+}
 
-  REPLY_MACRO (VL_API_SR_MULTICAST_MAP_ADD_DEL_REPLY);
-#endif
+static void vl_api_sr_steering_add_del_t_handler
+  (vl_api_sr_steering_add_del_t * mp)
+{
+  vl_api_sr_steering_add_del_reply_t *rmp;
+  int rv = 0;
+/*
+ * int
+ * sr_steering_policy(int is_del, ip6_address_t *bsid, u32 sr_policy_index,
+ *  u32 table_id, ip46_address_t *prefix, u32 mask_width, u32 sw_if_index,
+ *  u8 traffic_type)
+ */
+  rv = sr_steering_policy (mp->is_del,
+			   (ip6_address_t *) & mp->bsid_addr,
+			   ntohl (mp->sr_policy_index),
+			   ntohl (mp->table_id),
+			   (ip46_address_t *) & mp->prefix_addr,
+			   ntohl (mp->mask_width),
+			   ntohl (mp->sw_if_index), mp->traffic_type);
+
+  REPLY_MACRO (VL_API_SR_STEERING_ADD_DEL_REPLY);
 }
 
 /*
@@ -233,27 +204,26 @@ sr_api_hookup (vlib_main_t * vm)
 #undef _
 
   /*
-   * Manually register the sr tunnel add del msg, so we trace
+   * Manually register the sr policy add msg, so we trace
    * enough bytes to capture a typical segment list
    */
-  vl_msg_api_set_handlers (VL_API_SR_TUNNEL_ADD_DEL,
-			   "sr_tunnel_add_del",
-			   vl_api_sr_tunnel_add_del_t_handler,
+  vl_msg_api_set_handlers (VL_API_SR_POLICY_ADD,
+			   "sr_policy_add",
+			   vl_api_sr_policy_add_t_handler,
 			   vl_noop_handler,
-			   vl_api_sr_tunnel_add_del_t_endian,
-			   vl_api_sr_tunnel_add_del_t_print, 256, 1);
-
+			   vl_api_sr_policy_add_t_endian,
+			   vl_api_sr_policy_add_t_print, 256, 1);
 
   /*
-   * Manually register the sr policy add del msg, so we trace
-   * enough bytes to capture a typical tunnel name list
+   * Manually register the sr policy mod msg, so we trace
+   * enough bytes to capture a typical segment list
    */
-  vl_msg_api_set_handlers (VL_API_SR_POLICY_ADD_DEL,
-			   "sr_policy_add_del",
-			   vl_api_sr_policy_add_del_t_handler,
+  vl_msg_api_set_handlers (VL_API_SR_POLICY_MOD,
+			   "sr_policy_mod",
+			   vl_api_sr_policy_mod_t_handler,
 			   vl_noop_handler,
-			   vl_api_sr_policy_add_del_t_endian,
-			   vl_api_sr_policy_add_del_t_print, 256, 1);
+			   vl_api_sr_policy_mod_t_endian,
+			   vl_api_sr_policy_mod_t_print, 256, 1);
 
   /*
    * Set up the (msg_name, crc, message-id) table
diff --git a/src/vnet/sr/sr_doc.md b/src/vnet/sr/sr_doc.md
new file mode 100644
index 00000000..a7220630
--- /dev/null
+++ b/src/vnet/sr/sr_doc.md
@@ -0,0 +1,161 @@
+# SRv6: Segment Routing for IPv6    {#sr_doc}
+
+This is a memo intended to contain documentation of the VPP SRv6 implementation
+Everything that is not directly obvious should come here.
+For any feedback on content that should be explained please mailto:pcamaril@cisco.com
+
+## Segment Routing
+
+Segment routing is a network technology focused on addressing the limitations of existing IP and Multiprotocol Label Switching (MPLS) networks in terms of simplicity, scale, and ease of operation. It is a foundation for application engineered routing as it prepares the networks for new business models where applications can control the network behavior.
+
+Segment routing seeks the right balance between distributed intelligence and centralized optimization and programming. It was built for the software-defined networking (SDN) era.
+
+Segment routing enhances packet forwarding behavior by enabling a network to transport unicast packets through a specific forwarding path, different from the normal path that a packet usually takes (IGP shortest path or BGP best path). This capability benefits many use cases, and one can build those specific paths based on application requirements.
+
+Segment routing uses the source routing paradigm. A node, usually a router but also a switch, a trusted server, or a virtual forwarder running on a hypervisor, steers a packet through an ordered list of instructions, called segments. A segment can represent any instruction, topological or service-based. A segment can have a local semantic to a segment-routing node or global within a segment-routing network. Segment routing allows an operator to enforce a flow through any topological path and service chain while maintaining per-flow state only at the ingress node to the segment-routing network. Segment routing also supports equal-cost multipath (ECMP) by design.
+
+Segment routing can operate with either an MPLS or an IPv6 data plane. All the currently available MPLS services, such as Layer 3 VPN (L3VPN), L2VPN (Virtual Private Wire Service [VPWS], Virtual Private LAN Services [VPLS], Ethernet VPN [E-VPN], and Provider Backbone Bridging Ethernet VPN [PBB-EVPN]), can run on top of a segment-routing transport network.
+
+**The implementation of Segment Routing in VPP only covers the IPv6 data plane (SRv6).**
+
+## Segment Routing terminology
+
+* Segment Routing Header (SRH): IPv6 routing extension header of type 'Segment Routing'. (draft-ietf-6man-segment-routing-header-05)
+* SegmentID (SID): is an IPv6 address.
+* Segment List (SL) (SID List): is the sequence of SIDs that the packet will traverse.
+* SR Policy: defines the SRH that will be applied to a packet. A packet steered into an SR policy may either receive the SRH by IPv6 header encapsulation (as recommended in draft-ietf-6man-rfc2460bis) or it could be inserted within an existing IPv6 header. An SR policy is uniquely identified by its Binding SID and associated with a weighted set of Segment Lists. In case several SID lists are defined, traffic steered into the policy is unevenly load-balanced among them according to their respective weights.
+* Local SID: is a SID associated with a processing function on the local node, which may go from advancing to the next SID in the SRH, to complex user-defined behaviors. When a FIB lookup, either in the main FIB or in a specific VRF, returns a match on a local SID, the associated function is performed.
+* BindingSID: a BindingSID is a SID (only one) associated one-one with an SR Policy. If a packet arrives with an IPv6 DA corresponding to a BindingSID, then the SR policy will be applied to such packet.
+
+## Creating an SR LocalSID
+
+A local SID is associated to a Segment Routing behavior -or function- on the current node.
+
+The most basic behavior is called END. It simply activates the next SID in the current packet, by decrementing the Segments Left value and updating the IPv6 DA.
+
+A local END SID is instantiated using the following CLI:
+
+    sr localsid (del) address XX::YY behavior end
+
+This creates a new entry in the main FIB for IPv6 address XX::YY. All packets whose IPv6 DA matches this FIB entry are redirected to the sr-localsid node, where they are processed as described above.
+
+Other examples of local SIDs are the following:
+
+    sr localsid (del) address XX::YY behavior end (psp)
+    sr localsid (del) address XX::YY behavior end.x GE0/1/0 2001::a (psp)
+    sr localsid (del) address XX::YY behavior end.dx6 GE0/1/0 2001::a
+    sr localsid (del) address XX::YY behavior end.dx4 GE0/1/0 10.0.0.1
+    sr localsid (del) address XX::YY behavior end.dx2 GigabitE0/11/0
+    sr localsid (del) address XX::YY behavior end.dt6 5
+    sr localsid (del) address XX::YY behavior end.dt6 5
+
+Note that all of these behaviors match the specifications in **TODO REF NET PGM**. Please refer to this document for a detailed description of each behavior.
+
+Help on the available local SID behaviors and their usage can be obtained with:
+    
+    help sr localsid
+
+Alternatively they can be obtained using.
+
+    show sr localsids behavior
+
+The difference in between those two commands is that the first one will only display the SR LocalSID behaviors that are built-in VPP, while the latter will display those behaviors plus the ones added with the SR LocalSID Development Framework.
+
+
+VPP keeps a 'My LocalSID Table' where it stores all the SR local SIDs instantiated as well as their parameters. Every time a new local SID is instantiated, a new entry is added to this table. In addition, counters for correctly and incorrectly processed traffic are maintained for each local SID. The counters store both the number of packets and bytes.
+
+The contents of the 'My LocalSID Table' is shown with:
+
+    vpp# show sr localsid
+    SRv6 - My LocalSID Table:
+    =========================
+            Address:        c3::1
+            Behavior:       DX6 (Endpoint with decapsulation and IPv6 cross-connect)
+            Iface:          GigabitEthernet0/5/0
+            Next hop:       b:c3::b
+            Good traffic:   [51277 packets : 5332808 bytes]
+            Bad traffic:    [0 packets : 0 bytes]
+    --------------------
+
+The traffic counters can be reset with:
+
+    vpp# clear sr localsid counters
+
+## Creating a SR Policy
+
+An SR Policy is defined by a Binding SID and a weighted set of Segment Lists.
+
+A new SR policy is created with a first SID list using:
+
+    sr policy add bsid 2001::1 next A1:: next B1:: next C1:: (weight 5) (fib-table 3)
+
+* The weight parameter is only used if more than one SID list is associated with the policy.
+* The fib-table parameter specifies in which table (VRF) the Binding SID is to be installed.
+
+An SR policy is deleted with:
+
+    sr policy del bsid 2001::1
+    sr policy del index 1
+
+The existing SR policies are listed with:
+
+    show sr policies
+
+### Adding/Removing SID Lists from an SR policy
+
+An additional SID list is associated with an existing SR policy with:
+
+    sr policy mod bsid 2001::1 add sl next A2:: next B2:: next C2:: (weight 3)
+    sr policy mod index 3      add sl next A2:: next B2:: next C2:: (weight 3)
+
+Conversely, a SID list can be removed from an SR policy with:
+
+    sr policy mod bsid 2001::1 del sl index 1
+    sr policy mod index 3      del sl index 1
+
+Note that this cannot be used to remove the last SID list of a policy.
+
+The weight of a SID list can also be modified with:
+
+    sr policy mod bsid 2001::1 mod sl index 1 weight 4
+    sr policy mod index 3      mod sl index 1 weight 4
+
+### SR Policies: Spray policies
+
+Spray policies are a specific type of SR policies where the packet is replicated on all the SID lists, rather than load-balanced among them.
+
+SID list weights are ignored with this type of policies.
+
+A Spray policy is instantiated by appending the keyword **spray** to a regular SR policy command, as in:
+
+    sr policy add bsid 2001::1 next A1:: next B1:: next C1:: spray
+
+Spray policies are used for removing multicast state from a network core domain, and instead send a linear unicast copy to every access node. The last SID in each list accesses the multicast tree within the access node.  
+
+### Encapsulation SR policies
+
+In case the user decides to create an SR policy an IPv6 Source Address must be specified for the encapsulated traffic. In order to do so the user might use the following command:
+    
+    set sr encaps source addr XXXX::YYYY
+
+## Steering packets into a SR Policy 
+
+To steer packets in Transit into an SR policy (T.Insert, T.Encaps and T.Encaps.L2 behaviors), the user needs to create an 'sr steering policy'.
+
+    sr steer l3 2001::/64 via sr policy index 1
+    sr steer l3 2001::/64 via sr policy bsid cafe::1
+    sr steer l3 2001::/64 via sr policy bsid cafe::1 fib-table 3
+    sr steer l3 10.0.0.0/16 via sr policy bsid cafe::1
+    sr steer l2 TenGE0/1/0 via sr policy bsid cafe::1
+
+Disclaimer: The T.Encaps.L2 will steer L2 frames into an SR Policy. Notice that creating an SR steering policy for L2 frames will actually automatically *puts the interface into promiscous mode*.
+
+## SR LocalSID development framework
+
+One of the * 'key' * concepts about SRv6 is regarding network programmability. This is why an SRv6 LocalSID is associated with an specific function. 
+
+However, the trully way to enable network programmability is allowing any developer **easily** create his own SRv6 LocalSID function. That is the reason why we have added some API calls such that any developer can code his own SRv6 LocalSID behaviors as plugins an add them to the running SRv6 code.
+
+The principle is that the developer only codes the behavior -the graph node-. However all the FIB handling, SR LocalSID instantiation and so on are done by the VPP SRv6 code.
+
+For more information please refer to the documentation *SRv6 Sample SR LocalSID plugin*.
diff --git a/src/vnet/sr/sr_error.def b/src/vnet/sr/sr_error.def
deleted file mode 100644
index 62d021fd..00000000
--- a/src/vnet/sr/sr_error.def
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-sr_error (NONE, "no error")
-sr_error (BAD_ROUTING_HEADER_TYPE, "bad routing header type (not 4)")
-sr_error (NO_MORE_SEGMENTS, "out of SR segment drops")
-sr_error (PKTS_PROCESSED, "SR packets processed")
-sr_error (APP_CALLBACK, "SR application callback errors")
-sr_error (HMAC_INVALID, "SR packets with invalid HMAC signatures")
diff --git a/src/vnet/sr/sr_fix_dst_error.def b/src/vnet/sr/sr_fix_dst_error.def
deleted file mode 100644
index 48fe7af6..00000000
--- a/src/vnet/sr/sr_fix_dst_error.def
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-sr_fix_dst_error (NONE, "no error")
-sr_fix_dst_error (NO_SR_HEADER, "no SR header present")
-sr_fix_dst_error (NO_MORE_SEGMENTS, "no more SR segments")
diff --git a/src/vnet/sr/sr_localsid.c b/src/vnet/sr/sr_localsid.c
new file mode 100755
index 00000000..407491ce
--- /dev/null
+++ b/src/vnet/sr/sr_localsid.c
@@ -0,0 +1,1478 @@
+/*
+ * sr_localsid.c: ipv6 segment routing Endpoint behaviors
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ * @brief Processing of packets with a SRH
+ *
+ * CLI to define new Segment Routing End processing functions.
+ * Graph node to support such functions.
+ *
+ * Each function associates an SRv6 segment (IPv6 address) with an specific
+ * Segment Routing function.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+/**
+ * @brief Dynamically added SR localsid DPO type
+ */
+static dpo_type_t sr_localsid_dpo_type;
+static dpo_type_t sr_localsid_d_dpo_type;
+
+/**
+ * @brief SR localsid add/del
+ *
+ * Function to add or delete SR LocalSIDs.
+ *
+ * @param is_del Boolean of whether its a delete instruction
+ * @param localsid_addr IPv6 address of the localsid
+ * @param is_decap Boolean of whether decapsulation is allowed in this function
+ * @param behavior Type of behavior (function) for this localsid
+ * @param sw_if_index Only for L2/L3 xconnect. OIF. In VRF variant the fib_table.
+ * @param vlan_index Only for L2 xconnect. Outgoing VLAN tag.
+ * @param fib_table  FIB table in which we should install the localsid entry
+ * @param nh_addr Next Hop IPv4/IPv6 address. Only for L2/L3 xconnect.
+ *
+ * @return 0 on success, error otherwise.
+ */
+int
+sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
+		 char end_psp, u8 behavior, u32 sw_if_index, u32 vlan_index,
+		 u32 fib_table, ip46_address_t * nh_addr, void *ls_plugin_mem)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  uword *p;
+  int rv;
+
+  ip6_sr_localsid_t *ls = 0;
+  ip6_address_t *key_copy;
+
+  dpo_id_t dpo = DPO_INVALID;
+
+  /* Search for the item */
+  p = hash_get_mem (sm->localsids_index_by_key, localsid_addr);
+
+  if (p)
+    {
+      if (is_del)
+	{
+	  hash_pair_t *hp;
+	  /* Retrieve localsid */
+	  ls = pool_elt_at_index (sm->localsids, p[0]);
+	  /* Delete FIB entry */
+	  fib_prefix_t pfx = {
+	    .fp_proto = FIB_PROTOCOL_IP6,
+	    .fp_len = 128,
+	    .fp_addr = {
+			.ip6 = *localsid_addr,
+			}
+	  };
+
+	  fib_table_entry_delete (fib_table_id_find_fib_index
+				  (FIB_PROTOCOL_IP6, fib_table), &pfx,
+				  FIB_SOURCE_SR);
+
+	  /* In case it is a Xconnect iface remove the (OIF, NHOP) adj */
+	  if (ls->behavior == SR_BEHAVIOR_X || ls->behavior == SR_BEHAVIOR_DX6
+	      || ls->behavior == SR_BEHAVIOR_DX4)
+	    adj_unlock (ls->nh_adj);
+
+	  if (ls->behavior >= SR_BEHAVIOR_LAST)
+	    {
+	      sr_localsid_fn_registration_t *plugin = 0;
+	      plugin = pool_elt_at_index (sm->plugin_functions,
+					  ls->behavior - SR_BEHAVIOR_LAST);
+
+	      /* Callback plugin removal function */
+	      rv = plugin->removal (ls);
+	    }
+
+	  /* Delete localsid registry */
+	  pool_put (sm->localsids, ls);
+	  hp = hash_get_pair (sm->localsids_index_by_key, localsid_addr);
+	  key_copy = (void *) (hp->key);
+	  hash_unset_mem (sm->localsids_index_by_key, localsid_addr);
+	  vec_free (key_copy);
+	  return 1;
+	}
+      else			/* create with function already existing; complain */
+	return -1;
+    }
+  else
+    /* delete; localsid does not exist; complain */
+  if (is_del)
+    return -2;
+
+  /* Check whether there exists a FIB entry with such address */
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP6,
+    .fp_len = 128,
+  };
+
+  pfx.fp_addr.as_u64[0] = localsid_addr->as_u64[0];
+  pfx.fp_addr.as_u64[1] = localsid_addr->as_u64[1];
+
+  /* Lookup the FIB index associated to the table id provided */
+  u32 fib_index = fib_table_id_find_fib_index (FIB_PROTOCOL_IP6, fib_table);
+  if (fib_index == ~0)
+    return -3;
+
+  /* Lookup the localsid in such FIB table */
+  fib_node_index_t fei = fib_table_lookup_exact_match (fib_index, &pfx);
+  if (FIB_NODE_INDEX_INVALID != fei)
+    return -4;			//There is an entry for such address (the localsid addr)
+
+  /* Create a new localsid registry */
+  pool_get (sm->localsids, ls);
+  memset (ls, 0, sizeof (*ls));
+
+  clib_memcpy (&ls->localsid, localsid_addr, sizeof (ip6_address_t));
+  ls->end_psp = end_psp;
+  ls->behavior = behavior;
+  ls->nh_adj = (u32) ~ 0;
+  ls->fib_table = fib_table;
+  switch (behavior)
+    {
+    case SR_BEHAVIOR_END:
+      break;
+    case SR_BEHAVIOR_X:
+      ls->sw_if_index = sw_if_index;
+      clib_memcpy (&ls->next_hop.ip6, &nh_addr->ip6, sizeof (ip6_address_t));
+      break;
+    case SR_BEHAVIOR_DX4:
+      ls->sw_if_index = sw_if_index;
+      clib_memcpy (&ls->next_hop.ip4, &nh_addr->ip4, sizeof (ip4_address_t));
+      break;
+    case SR_BEHAVIOR_DX6:
+      ls->sw_if_index = sw_if_index;
+      clib_memcpy (&ls->next_hop.ip6, &nh_addr->ip6, sizeof (ip6_address_t));
+      break;
+    case SR_BEHAVIOR_DT6:
+      ls->vrf_index = sw_if_index;
+      break;
+    case SR_BEHAVIOR_DX2:
+      ls->sw_if_index = sw_if_index;
+      ls->vlan_index = vlan_index;
+      break;
+    }
+
+  /* Figure out the adjacency magic for Xconnect variants */
+  if (ls->behavior == SR_BEHAVIOR_X || ls->behavior == SR_BEHAVIOR_DX4
+      || ls->behavior == SR_BEHAVIOR_DX6)
+    {
+      adj_index_t nh_adj_index = ADJ_INDEX_INVALID;
+
+      /* Retrieve the adjacency corresponding to the (OIF, next_hop) */
+      if (ls->behavior == SR_BEHAVIOR_DX6 || ls->behavior == SR_BEHAVIOR_X)
+	nh_adj_index = adj_nbr_add_or_lock (FIB_PROTOCOL_IP6, VNET_LINK_IP6,
+					    nh_addr, sw_if_index);
+
+      else if (ls->behavior == SR_BEHAVIOR_DX4)
+	nh_adj_index = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4, VNET_LINK_IP4,
+					    nh_addr, sw_if_index);
+
+      /* Check for ADJ creation error. If so panic */
+      if (nh_adj_index == ADJ_INDEX_INVALID)
+	{
+	  pool_put (sm->localsids, ls);
+	  return -5;
+	}
+
+      ls->nh_adj = nh_adj_index;
+    }
+
+  /* Set DPO */
+  if (ls->behavior == SR_BEHAVIOR_END || ls->behavior == SR_BEHAVIOR_X)
+    dpo_set (&dpo, sr_localsid_dpo_type, DPO_PROTO_IP6, ls - sm->localsids);
+  else if (ls->behavior > SR_BEHAVIOR_D_FIRST
+	   && ls->behavior < SR_BEHAVIOR_LAST)
+    dpo_set (&dpo, sr_localsid_d_dpo_type, DPO_PROTO_IP6, ls - sm->localsids);
+  else if (ls->behavior >= SR_BEHAVIOR_LAST)
+    {
+      sr_localsid_fn_registration_t *plugin = 0;
+      plugin = pool_elt_at_index (sm->plugin_functions,
+				  ls->behavior - SR_BEHAVIOR_LAST);
+      /* Copy the unformat memory result */
+      ls->plugin_mem = ls_plugin_mem;
+      /* Callback plugin creation function */
+      rv = plugin->creation (ls);
+      if (rv)
+	{
+	  pool_put (sm->localsids, ls);
+	  return -6;
+	}
+      dpo_set (&dpo, plugin->dpo, DPO_PROTO_IP6, ls - sm->localsids);
+    }
+
+  /* Set hash key for searching localsid by address */
+  key_copy = vec_new (ip6_address_t, 1);
+  clib_memcpy (key_copy, localsid_addr, sizeof (ip6_address_t));
+  hash_set_mem (sm->localsids_index_by_key, key_copy, ls - sm->localsids);
+
+  fib_table_entry_special_dpo_add (fib_index, &pfx, FIB_SOURCE_SR,
+				   FIB_ENTRY_FLAG_EXCLUSIVE, &dpo);
+  dpo_reset (&dpo);
+
+  /* Set counter to zero */
+  vlib_validate_combined_counter (&(sm->sr_ls_valid_counters),
+				  ls - sm->localsids);
+  vlib_validate_combined_counter (&(sm->sr_ls_invalid_counters),
+				  ls - sm->localsids);
+
+  vlib_zero_combined_counter (&(sm->sr_ls_valid_counters),
+			      ls - sm->localsids);
+  vlib_zero_combined_counter (&(sm->sr_ls_invalid_counters),
+			      ls - sm->localsids);
+
+  return 0;
+}
+
+/**
+ * @brief SR LocalSID CLI function.
+ *
+ * @see sr_cli_localsid
+ */
+static clib_error_t *
+sr_cli_localsid_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ip6_sr_main_t *sm = &sr_main;
+  u32 sw_if_index = (u32) ~ 0, vlan_index = (u32) ~ 0, fib_index = 0;
+  int is_del = 0;
+  int end_psp = 0;
+  ip6_address_t resulting_address;
+  ip46_address_t next_hop;
+  char address_set = 0;
+  char behavior = 0;
+  void *ls_plugin_mem = 0;
+
+  int rv;
+
+  memset (&resulting_address, 0, sizeof (ip6_address_t));
+  ip46_address_reset (&next_hop);
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "del"))
+	is_del = 1;
+      else if (!address_set
+	       && unformat (input, "address %U", unformat_ip6_address,
+			    &resulting_address))
+	address_set = 1;
+      else if (!address_set
+	       && unformat (input, "addr %U", unformat_ip6_address,
+			    &resulting_address))
+	address_set = 1;
+      else if (unformat (input, "fib-table %u", &fib_index));
+      else if (vlan_index == (u32) ~ 0
+	       && unformat (input, "vlan %u", &vlan_index));
+      else if (!behavior && unformat (input, "behavior"))
+	{
+	  if (unformat (input, "end.x %U %U",
+			unformat_vnet_sw_interface, vnm, &sw_if_index,
+			unformat_ip6_address, &next_hop.ip6))
+	    behavior = SR_BEHAVIOR_X;
+	  else if (unformat (input, "end.dx6 %U %U",
+			     unformat_vnet_sw_interface, vnm, &sw_if_index,
+			     unformat_ip6_address, &next_hop.ip6))
+	    behavior = SR_BEHAVIOR_DX6;
+	  else if (unformat (input, "end.dx4 %U %U",
+			     unformat_vnet_sw_interface, vnm, &sw_if_index,
+			     unformat_ip4_address, &next_hop.ip4))
+	    behavior = SR_BEHAVIOR_DX4;
+	  else if (unformat (input, "end.dx2 %U",
+			     unformat_vnet_sw_interface, vnm, &sw_if_index))
+	    behavior = SR_BEHAVIOR_DX2;
+	  else if (unformat (input, "end.dt6 %u", &sw_if_index))
+	    behavior = SR_BEHAVIOR_DT6;
+	  else if (unformat (input, "end.dt4 %u", &sw_if_index))
+	    behavior = SR_BEHAVIOR_DT4;
+	  else
+	    {
+	      /* Loop over all the plugin behavior format functions */
+	      sr_localsid_fn_registration_t *plugin = 0, **vec_plugins = 0;
+	      sr_localsid_fn_registration_t **plugin_it = 0;
+
+	      /* Create a vector out of the plugin pool as recommended */
+        /* *INDENT-OFF* */
+        pool_foreach (plugin, sm->plugin_functions,
+        {
+          vec_add1 (vec_plugins, plugin);
+        });
+        /* *INDENT-ON* */
+
+	      vec_foreach (plugin_it, vec_plugins)
+	      {
+		if (unformat
+		    (input, "%U", (*plugin_it)->ls_unformat, &ls_plugin_mem))
+		  {
+		    behavior = (*plugin_it)->sr_localsid_function_number;
+		    break;
+		  }
+	      }
+	    }
+
+	  if (!behavior)
+	    {
+	      if (unformat (input, "end"))
+		behavior = SR_BEHAVIOR_END;
+	      else
+		break;
+	    }
+	}
+      else if (!end_psp && unformat (input, "psp"))
+	end_psp = 1;
+      else
+	break;
+    }
+
+  if (!behavior && end_psp)
+    behavior = SR_BEHAVIOR_END;
+
+  if (!address_set)
+    return clib_error_return (0,
+			      "Error: SRv6 LocalSID address is mandatory.");
+  if (!is_del && !behavior)
+    return clib_error_return (0,
+			      "Error: SRv6 LocalSID behavior is mandatory.");
+  if (vlan_index != (u32) ~ 0)
+    return clib_error_return (0,
+			      "Error: SRv6 End.DX2 with rewrite VLAN tag not supported by now.");
+  if (end_psp && !(behavior == SR_BEHAVIOR_END || behavior == SR_BEHAVIOR_X))
+    return clib_error_return (0,
+			      "Error: SRv6 PSP only compatible with End and End.X");
+
+  rv = sr_cli_localsid (is_del, &resulting_address, end_psp, behavior,
+			sw_if_index, vlan_index, fib_index, &next_hop,
+			ls_plugin_mem);
+
+  switch (rv)
+    {
+    case 0:
+      break;
+    case 1:
+      return 0;
+    case -1:
+      return clib_error_return (0,
+				"Identical localsid already exists. Requested localsid not created.");
+    case -2:
+      return clib_error_return (0,
+				"The requested localsid could not be deleted. SR localsid not found");
+    case -3:
+      return clib_error_return (0, "FIB table %u does not exist", fib_index);
+    case -4:
+      return clib_error_return (0, "There is already one FIB entry for the"
+				"requested localsid non segment routing related");
+    case -5:
+      return clib_error_return (0,
+				"Could not create ARP/ND entry for such next_hop. Internal error.");
+    case -6:
+      return clib_error_return (0,
+				"Error on the plugin based localsid creation.");
+    default:
+      return clib_error_return (0, "BUG: sr localsid returns %d", rv);
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (sr_localsid_command, static) = {
+  .path = "sr localsid",
+  .short_help = "sr localsid (del) address XX:XX::YY:YY"
+      "(fib-table 8) behavior STRING",
+  .long_help =
+    "Create SR LocalSID and binds it to a particular behavior\n"
+    "Arguments:\n"
+    "\tlocalSID IPv6_addr(128b)   LocalSID IPv6 address\n"
+    "\t(fib-table X)              Optional. VRF where to install SRv6 localsid\n"
+    "\tbehavior STRING            Specifies the behavior\n"
+    "\n\tBehaviors:\n"
+    "\tEnd\t-> Endpoint.\n"
+    "\tEnd.X\t-> Endpoint with decapsulation and Layer-3 cross-connect.\n"
+    "\t\tParameters: ' '\n"
+    "\tEnd.DX2\t-> Endpoint with decapsulation and Layer-2 cross-connect.\n"
+    "\t\tParameters: ''\n"
+    "\tEnd.DX6\t-> Endpoint with decapsulation and IPv6 cross-connect.\n"
+    "\t\tParameters: ' '\n"
+    "\tEnd.DX4\t-> Endpoint with decapsulation and IPv4 cross-connect.\n"
+    "\t\tParameters: ' '\n"
+    "\tEnd.DT6\t-> Endpoint with decapsulation and specific IPv6 table lookup.\n"
+    "\t\tParameters: ''\n"
+    "\tEnd.DT4\t-> Endpoint with decapsulation and specific IPv4 table lookup.\n"
+    "\t\tParameters: ''\n",
+  .function = sr_cli_localsid_command_fn,
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief CLI function to 'show' all SR LocalSIDs on console.
+ */
+static clib_error_t *
+show_sr_localsid_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			     vlib_cli_command_t * cmd)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_sr_localsid_t **localsid_list = 0;
+  ip6_sr_localsid_t *ls;
+  int i;
+
+  vlib_cli_output (vm, "SRv6 - My LocalSID Table:");
+  vlib_cli_output (vm, "=========================");
+  /* *INDENT-OFF* */
+  pool_foreach (ls, sm->localsids, ({ vec_add1 (localsid_list, ls); }));
+  /* *INDENT-ON* */
+  for (i = 0; i < vec_len (localsid_list); i++)
+    {
+      ls = localsid_list[i];
+      switch (ls->behavior)
+	{
+	case SR_BEHAVIOR_END:
+	  vlib_cli_output (vm, "\tAddress: \t%U\n\tBehavior: \tEnd",
+			   format_ip6_address, &ls->localsid);
+	  break;
+	case SR_BEHAVIOR_X:
+	  vlib_cli_output (vm,
+			   "\tAddress: \t%U\n\tBehavior: \tX (Endpoint with Layer-3 cross-connect)"
+			   "\n\tIface:  \t%U\n\tNext hop: \t%U",
+			   format_ip6_address, &ls->localsid,
+			   format_vnet_sw_if_index_name, vnm, ls->sw_if_index,
+			   format_ip6_address, &ls->next_hop.ip6);
+	  break;
+	case SR_BEHAVIOR_DX4:
+	  vlib_cli_output (vm,
+			   "\tAddress: \t%U\n\tBehavior: \tDX4 (Endpoint with decapsulation and IPv4 cross-connect)"
+			   "\n\tIface:  \t%U\n\tNext hop: \t%U",
+			   format_ip6_address, &ls->localsid,
+			   format_vnet_sw_if_index_name, vnm, ls->sw_if_index,
+			   format_ip4_address, &ls->next_hop.ip4);
+	  break;
+	case SR_BEHAVIOR_DX6:
+	  vlib_cli_output (vm,
+			   "\tAddress: \t%U\n\tBehavior: \tDX6 (Endpoint with decapsulation and IPv6 cross-connect)"
+			   "\n\tIface:  \t%U\n\tNext hop: \t%U",
+			   format_ip6_address, &ls->localsid,
+			   format_vnet_sw_if_index_name, vnm, ls->sw_if_index,
+			   format_ip6_address, &ls->next_hop.ip6);
+	  break;
+	case SR_BEHAVIOR_DX2:
+	  if (ls->vlan_index == (u32) ~ 0)
+	    vlib_cli_output (vm,
+			     "\tAddress: \t%U\n\tBehavior: \tDX2 (Endpoint with decapulation and Layer-2 cross-connect)"
+			     "\n\tIface:  \t%U", format_ip6_address,
+			     &ls->localsid, format_vnet_sw_if_index_name, vnm,
+			     ls->sw_if_index);
+	  else
+	    vlib_cli_output (vm,
+			     "Unsupported yet. (DX2 with egress VLAN rewrite)");
+	  break;
+	case SR_BEHAVIOR_DT6:
+	  vlib_cli_output (vm,
+			   "\tAddress: \t%U\n\tBehavior: \tDT6 (Endpoint with decapsulation and specific IPv6 table lookup)"
+			   "\n\tTable: %u", format_ip6_address, &ls->localsid,
+			   ls->fib_table);
+	  break;
+	case SR_BEHAVIOR_DT4:
+	  vlib_cli_output (vm,
+			   "\tAddress: \t%U\n\tBehavior: \tDT4 (Endpoint with decapsulation and specific IPv4 table lookup)"
+			   "\n\tTable: \t%u", format_ip6_address,
+			   &ls->localsid, ls->fib_table);
+	  break;
+	default:
+	  if (ls->behavior >= SR_BEHAVIOR_LAST)
+	    {
+	      sr_localsid_fn_registration_t *plugin =
+		pool_elt_at_index (sm->plugin_functions,
+				   ls->behavior - SR_BEHAVIOR_LAST);
+
+	      vlib_cli_output (vm, "\tAddress: \t%U\n"
+			       "\tBehavior: \t%s (%s)\n\t%U",
+			       format_ip6_address, &ls->localsid,
+			       plugin->keyword_str, plugin->def_str,
+			       plugin->ls_format, ls->plugin_mem);
+	    }
+	  else
+	    //Should never get here...
+	    vlib_cli_output (vm, "Internal error");
+	  break;
+	}
+      if (ls->end_psp)
+	vlib_cli_output (vm, "\tPSP: \tTrue\n");
+
+      /* Print counters */
+      vlib_counter_t valid, invalid;
+      vlib_get_combined_counter (&(sm->sr_ls_valid_counters), i, &valid);
+      vlib_get_combined_counter (&(sm->sr_ls_invalid_counters), i, &invalid);
+      vlib_cli_output (vm, "\tGood traffic: \t[%Ld packets : %Ld bytes]\n",
+		       valid.packets, valid.bytes);
+      vlib_cli_output (vm, "\tBad traffic:  \t[%Ld packets : %Ld bytes]\n",
+		       invalid.packets, invalid.bytes);
+      vlib_cli_output (vm, "--------------------");
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_sr_localsid_command, static) = {
+  .path = "show sr localsids",
+  .short_help = "show sr localsids",
+  .function = show_sr_localsid_command_fn,
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Function to 'clear' ALL SR localsid counters
+ */
+static clib_error_t *
+clear_sr_localsid_counters_command_fn (vlib_main_t * vm,
+				       unformat_input_t * input,
+				       vlib_cli_command_t * cmd)
+{
+  ip6_sr_main_t *sm = &sr_main;
+
+  vlib_clear_combined_counters (&(sm->sr_ls_valid_counters));
+  vlib_clear_combined_counters (&(sm->sr_ls_invalid_counters));
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (clear_sr_localsid_counters_command, static) = {
+  .path = "clear sr localsid counters",
+  .short_help = "clear sr localsid counters",
+  .function = clear_sr_localsid_counters_command_fn,
+};
+/* *INDENT-ON* */
+
+/************************ SR LocalSID graphs node ****************************/
+/**
+ * @brief SR localsid node trace
+ */
+typedef struct
+{
+  u32 localsid_index;
+  ip6_address_t src, out_dst;
+  u8 sr[256];
+  u8 num_segments;
+  u8 segments_left;
+  //With SRv6 header update include flags here.
+} sr_localsid_trace_t;
+
+#define foreach_sr_localsid_error                                   \
+_(NO_INNER_HEADER, "(SR-Error) No inner IP header")                 \
+_(NO_MORE_SEGMENTS, "(SR-Error) No more segments")                  \
+_(NO_SRH, "(SR-Error) No SR header")                                \
+_(NO_PSP, "(SR-Error) PSP Not available (segments left > 0)")       \
+_(NOT_LS, "(SR-Error) Decaps not available (segments left > 0)")    \
+_(L2, "(SR-Error) SRv6 decapsulated a L2 frame without dest")
+
+typedef enum
+{
+#define _(sym,str) SR_LOCALSID_ERROR_##sym,
+  foreach_sr_localsid_error
+#undef _
+    SR_LOCALSID_N_ERROR,
+} sr_localsid_error_t;
+
+static char *sr_localsid_error_strings[] = {
+#define _(sym,string) string,
+  foreach_sr_localsid_error
+#undef _
+};
+
+#define foreach_sr_localsid_next        \
+_(ERROR, "error-drop")                  \
+_(IP6_LOOKUP, "ip6-lookup")             \
+_(IP4_LOOKUP, "ip4-lookup")             \
+_(IP6_REWRITE, "ip6-rewrite")           \
+_(IP4_REWRITE, "ip4-rewrite")           \
+_(INTERFACE_OUTPUT, "interface-output")
+
+typedef enum
+{
+#define _(s,n) SR_LOCALSID_NEXT_##s,
+  foreach_sr_localsid_next
+#undef _
+    SR_LOCALSID_N_NEXT,
+} sr_localsid_next_t;
+
+/**
+ * @brief SR LocalSID graph node trace function
+ *
+ * @see sr_localsid
+ */
+u8 *
+format_sr_localsid_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  ip6_sr_main_t *sm = &sr_main;
+  sr_localsid_trace_t *t = va_arg (*args, sr_localsid_trace_t *);
+
+  ip6_sr_localsid_t *ls =
+    pool_elt_at_index (sm->localsids, t->localsid_index);
+
+  s =
+    format (s, "SR-LOCALSID:\n\tLocalsid: %U\n", format_ip6_address,
+	    &ls->localsid);
+  switch (ls->behavior)
+    {
+    case SR_BEHAVIOR_END:
+      s = format (s, "\tBehavior: End\n");
+      break;
+    case SR_BEHAVIOR_DX6:
+      s = format (s, "\tBehavior: Decapsulation with IPv6 L3 xconnect\n");
+      break;
+    case SR_BEHAVIOR_DX4:
+      s = format (s, "\tBehavior: Decapsulation with IPv4 L3 xconnect\n");
+      break;
+    case SR_BEHAVIOR_X:
+      s = format (s, "\tBehavior: IPv6 L3 xconnect\n");
+      break;
+    case SR_BEHAVIOR_DT6:
+      s = format (s, "\tBehavior: Decapsulation with IPv6 Table lookup\n");
+      break;
+    case SR_BEHAVIOR_DT4:
+      s = format (s, "\tBehavior: Decapsulation with IPv4 Table lookup\n");
+      break;
+    case SR_BEHAVIOR_DX2:
+      s = format (s, "\tBehavior: Decapsulation with L2 xconnect\n");
+      break;
+    default:
+      s = format (s, "\tBehavior: defined in plugin\n");	//TODO
+      break;
+    }
+  if (t->num_segments != 0xFF)
+    {
+      if (t->num_segments > 0)
+	{
+	  s = format (s, "\tSegments left: %d\n", t->num_segments);
+	  s = format (s, "\tSID list: [in ietf order]");
+	  int i = 0;
+	  for (i = 0; i < t->num_segments; i++)
+	    {
+	      s = format (s, "\n\t-> %U", format_ip6_address,
+			  (ip6_address_t *) & t->sr[i *
+						    sizeof (ip6_address_t)]);
+	    }
+	}
+    }
+  return s;
+}
+
+/**
+ * @brief Function doing End processing.
+ */
+static_always_inline void
+end_srh_processing (vlib_node_runtime_t * node,
+		    vlib_buffer_t * b0,
+		    ip6_header_t * ip0,
+		    ip6_sr_header_t * sr0,
+		    ip6_sr_localsid_t * ls0, u32 * next0)
+{
+  ip6_address_t *new_dst0;
+
+  if (PREDICT_TRUE (sr0->type == ROUTING_HEADER_TYPE_SR))
+    {
+      if (PREDICT_TRUE (sr0->segments_left != 0))
+	{
+	  sr0->segments_left -= 1;
+	  new_dst0 = (ip6_address_t *) (sr0->segments);
+	  new_dst0 += sr0->segments_left;
+	  ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
+	  ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
+
+	  if (ls0->behavior == SR_BEHAVIOR_X)
+	    {
+	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ls0->nh_adj;
+	      *next0 = SR_LOCALSID_NEXT_IP6_REWRITE;
+	    }
+	}
+      else
+	{
+	  *next0 = SR_LOCALSID_NEXT_ERROR;
+	  b0->error = node->errors[SR_LOCALSID_ERROR_NO_MORE_SEGMENTS];
+	}
+    }
+  else
+    {
+      /* Error. Routing header of type != SR */
+      *next0 = SR_LOCALSID_NEXT_ERROR;
+      b0->error = node->errors[SR_LOCALSID_ERROR_NO_SRH];
+    }
+}
+
+/*
+ * @brief Function doing SRH processing for D* variants
+ */
+//FixME. I must crosscheck that next_proto matches the localsid
+static_always_inline void
+end_decaps_srh_processing (vlib_node_runtime_t * node,
+			   vlib_buffer_t * b0,
+			   ip6_header_t * ip0,
+			   ip6_sr_header_t * sr0,
+			   ip6_sr_localsid_t * ls0, u32 * next0)
+{
+  /* Compute the size of the IPv6 header with all Ext. headers */
+  u8 next_proto;
+  ip6_ext_header_t *next_ext_header;
+  u16 total_size = 0;
+
+  next_proto = ip0->protocol;
+  next_ext_header = (void *) (ip0 + 1);
+  total_size = sizeof (ip6_header_t);
+  while (ip6_ext_hdr (next_proto))
+    {
+      total_size += ip6_ext_header_len (next_ext_header);
+      next_proto = next_ext_header->next_hdr;
+      next_ext_header = ip6_ext_next_header (next_ext_header);
+    }
+
+  /* Ensure this is the last segment. Otherwise drop. */
+  if (sr0 && sr0->segments_left != 0)
+    {
+      *next0 = SR_LOCALSID_NEXT_ERROR;
+      b0->error = node->errors[SR_LOCALSID_ERROR_NOT_LS];
+      return;
+    }
+
+  switch (next_proto)
+    {
+    case IP_PROTOCOL_IPV6:
+      /* Encap-End IPv6. Pop outer IPv6 header. */
+      if (ls0->behavior == SR_BEHAVIOR_DX6)
+	{
+	  vlib_buffer_advance (b0, total_size);
+	  vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ls0->nh_adj;
+	  *next0 = SR_LOCALSID_NEXT_IP6_REWRITE;
+	  return;
+	}
+      else if (ls0->behavior == SR_BEHAVIOR_DT6)
+	{
+	  vlib_buffer_advance (b0, total_size);
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ls0->fib_table;
+	  return;
+	}
+      break;
+    case IP_PROTOCOL_IP_IN_IP:
+      /* Encap-End IPv4. Pop outer IPv6 header */
+      if (ls0->behavior == SR_BEHAVIOR_DX4)
+	{
+	  vlib_buffer_advance (b0, total_size);
+	  vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ls0->nh_adj;
+	  *next0 = SR_LOCALSID_NEXT_IP4_REWRITE;
+	  return;
+	}
+      else if (ls0->behavior == SR_BEHAVIOR_DT4)
+	{
+	  vlib_buffer_advance (b0, total_size);
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ls0->fib_table;
+	  *next0 = SR_LOCALSID_NEXT_IP4_LOOKUP;
+	  return;
+	}
+      break;
+    case IP_PROTOCOL_IP6_NONXT:
+      /* L2 encaps */
+      if (ls0->behavior == SR_BEHAVIOR_DX2)
+	{
+	  vlib_buffer_advance (b0, total_size);
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ls0->sw_if_index;
+	  *next0 = SR_LOCALSID_NEXT_INTERFACE_OUTPUT;
+	  return;
+	}
+      break;
+    }
+  *next0 = SR_LOCALSID_NEXT_ERROR;
+  b0->error = node->errors[SR_LOCALSID_ERROR_NO_INNER_HEADER];
+  return;
+}
+
+/**
+ * @brief Function doing End processing with PSP
+ */
+static_always_inline void
+end_psp_srh_processing (vlib_node_runtime_t * node,
+			vlib_buffer_t * b0,
+			ip6_header_t * ip0,
+			ip6_ext_header_t * prev0,
+			ip6_sr_header_t * sr0,
+			ip6_sr_localsid_t * ls0, u32 * next0)
+{
+  u32 new_l0, sr_len;
+
+  if (PREDICT_TRUE (sr0->type == ROUTING_HEADER_TYPE_SR))
+    {
+      if (PREDICT_TRUE (sr0->segments_left == 1))
+	{
+	  ip0->dst_address.as_u64[0] = sr0->segments->as_u64[0];
+	  ip0->dst_address.as_u64[1] = sr0->segments->as_u64[1];
+
+	  /* Remove the SRH taking care of the rest of IPv6 ext header */
+	  if (prev0)
+	    prev0->next_hdr = sr0->protocol;
+	  else
+	    ip0->protocol = sr0->protocol;
+
+	  sr_len = ip6_ext_header_len (sr0);
+	  vlib_buffer_advance (b0, sr_len);
+	  new_l0 = clib_net_to_host_u16 (ip0->payload_length) - sr_len;
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+	  clib_memcpy ((void *) ip0 + sr_len, ip0,
+		       (void *) sr0 - (void *) ip0);
+
+	  if (ls0->behavior == SR_BEHAVIOR_X)
+	    {
+	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ls0->nh_adj;
+	      *next0 = SR_LOCALSID_NEXT_IP6_REWRITE;
+	    }
+	  return;
+	}
+    }
+  /* Error. Routing header of type != SR */
+  *next0 = SR_LOCALSID_NEXT_ERROR;
+  b0->error = node->errors[SR_LOCALSID_ERROR_NO_PSP];
+}
+
+/**
+ * @brief SR LocalSID graph node. Supports all default SR Endpoint variants
+ */
+static uword
+sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		  vlib_frame_t * from_frame)
+{
+  u32 n_left_from, next_index, *from, *to_next;
+  ip6_sr_main_t *sm = &sr_main;
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+  next_index = node->cached_next_index;
+  u32 cpu_index = os_get_cpu_number ();
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_ext_header_t *prev0, *prev1, *prev2, *prev3;
+	  ip6_sr_header_t *sr0, *sr1, *sr2, *sr3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+	  ip6_sr_localsid_t *ls0, *ls1, *ls2, *ls3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+4 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  ls0 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ls1 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ls2 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ls3 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  ip6_ext_header_find_t (ip0, prev0, sr0, IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip1, prev1, sr1, IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip2, prev2, sr2, IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip3, prev3, sr3, IP_PROTOCOL_IPV6_ROUTE);
+
+	  end_decaps_srh_processing (node, b0, ip0, sr0, ls0, &next0);
+	  end_decaps_srh_processing (node, b1, ip1, sr1, ls1, &next1);
+	  end_decaps_srh_processing (node, b2, ip2, sr2, ls2, &next2);
+	  end_decaps_srh_processing (node, b3, ip3, sr3, ls3, &next3);
+
+	  //TODO: trace.
+
+	  vlib_increment_combined_counter
+	    (((next0 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b0));
+
+	  vlib_increment_combined_counter
+	    (((next1 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls1 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b1));
+
+	  vlib_increment_combined_counter
+	    (((next2 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls2 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b2));
+
+	  vlib_increment_combined_counter
+	    (((next3 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls3 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b3));
+
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0;
+	  ip6_ext_header_t *prev0;
+	  ip6_sr_header_t *sr0;
+	  u32 next0 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+	  ip6_sr_localsid_t *ls0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  /* Lookup the SR End behavior based on IP DA (adj) */
+	  ls0 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  /* Find SRH as well as previous header */
+	  ip6_ext_header_find_t (ip0, prev0, sr0, IP_PROTOCOL_IPV6_ROUTE);
+
+	  /* SRH processing and End variants */
+	  end_decaps_srh_processing (node, b0, ip0, sr0, ls0, &next0);
+
+	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_localsid_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      tr->num_segments = 0;
+	      tr->localsid_index = ls0 - sm->localsids;
+
+	      if (ip0 == vlib_buffer_get_current (b0))
+		{
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->out_dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->out_dst.as_u8));
+		  if (ip0->protocol == IP_PROTOCOL_IPV6_ROUTE
+		      && sr0->type == ROUTING_HEADER_TYPE_SR)
+		    {
+		      clib_memcpy (tr->sr, sr0->segments, sr0->length * 8);
+		      tr->num_segments =
+			sr0->length * 8 / sizeof (ip6_address_t);
+		      tr->segments_left = sr0->segments_left;
+		    }
+		}
+	      else
+		tr->num_segments = 0xFF;
+	    }
+
+	  /* Increase the counters */
+	  vlib_increment_combined_counter
+	    (((next0 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b0));
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_localsid_d_node) = {
+  .function = sr_localsid_d_fn,
+  .name = "sr-localsid-d",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_localsid_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_LOCALSID_N_ERROR,
+  .error_strings = sr_localsid_error_strings,
+  .n_next_nodes = SR_LOCALSID_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_LOCALSID_NEXT_##s] = n,
+    foreach_sr_localsid_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief SR LocalSID graph node. Supports all default SR Endpoint variants
+ */
+static uword
+sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		vlib_frame_t * from_frame)
+{
+  u32 n_left_from, next_index, *from, *to_next;
+  ip6_sr_main_t *sm = &sr_main;
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+  next_index = node->cached_next_index;
+  u32 cpu_index = os_get_cpu_number ();
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_sr_header_t *sr0, *sr1, *sr2, *sr3;
+	  ip6_ext_header_t *prev0, *prev1, *prev2, *prev3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+	  ip6_sr_localsid_t *ls0, *ls1, *ls2, *ls3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  ip6_ext_header_find_t (ip0, prev0, sr0, IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip1, prev1, sr1, IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip2, prev2, sr2, IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip3, prev3, sr3, IP_PROTOCOL_IPV6_ROUTE);
+
+	  ls0 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ls1 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ls2 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ls3 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  if (ls0->end_psp)
+	    end_psp_srh_processing (node, b0, ip0, prev0, sr0, ls0, &next0);
+	  else
+	    end_srh_processing (node, b0, ip0, sr0, ls0, &next0);
+
+	  if (ls1->end_psp)
+	    end_psp_srh_processing (node, b1, ip1, prev1, sr1, ls1, &next1);
+	  else
+	    end_srh_processing (node, b1, ip1, sr1, ls1, &next1);
+
+	  if (ls2->end_psp)
+	    end_psp_srh_processing (node, b2, ip2, prev2, sr2, ls2, &next2);
+	  else
+	    end_srh_processing (node, b2, ip2, sr2, ls2, &next2);
+
+	  if (ls3->end_psp)
+	    end_psp_srh_processing (node, b3, ip3, prev3, sr3, ls3, &next3);
+	  else
+	    end_srh_processing (node, b3, ip3, sr3, ls3, &next3);
+
+	  //TODO: proper trace.
+
+	  vlib_increment_combined_counter
+	    (((next0 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b0));
+
+	  vlib_increment_combined_counter
+	    (((next1 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls1 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b1));
+
+	  vlib_increment_combined_counter
+	    (((next2 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls2 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b2));
+
+	  vlib_increment_combined_counter
+	    (((next3 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls3 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b3));
+
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0;
+	  ip6_ext_header_t *prev0;
+	  ip6_sr_header_t *sr0;
+	  u32 next0 = SR_LOCALSID_NEXT_IP6_LOOKUP;
+	  ip6_sr_localsid_t *ls0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip6_ext_header_find_t (ip0, prev0, sr0, IP_PROTOCOL_IPV6_ROUTE);
+
+	  /* Lookup the SR End behavior based on IP DA (adj) */
+	  ls0 =
+	    pool_elt_at_index (sm->localsids,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  /* SRH processing */
+	  if (ls0->end_psp)
+	    end_psp_srh_processing (node, b0, ip0, prev0, sr0, ls0, &next0);
+	  else
+	    end_srh_processing (node, b0, ip0, sr0, ls0, &next0);
+
+	  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_localsid_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      tr->num_segments = 0;
+	      tr->localsid_index = ls0 - sm->localsids;
+
+	      if (ip0 == vlib_buffer_get_current (b0))
+		{
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->out_dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->out_dst.as_u8));
+		  if (ip0->protocol == IP_PROTOCOL_IPV6_ROUTE
+		      && sr0->type == ROUTING_HEADER_TYPE_SR)
+		    {
+		      clib_memcpy (tr->sr, sr0->segments, sr0->length * 8);
+		      tr->num_segments =
+			sr0->length * 8 / sizeof (ip6_address_t);
+		      tr->segments_left = sr0->segments_left;
+		    }
+		}
+	      else
+		{
+		  tr->num_segments = 0xFF;
+		}
+	    }
+
+	  vlib_increment_combined_counter
+	    (((next0 ==
+	       SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) :
+	      &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1,
+	     vlib_buffer_length_in_chain (vm, b0));
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_localsid_node) = {
+  .function = sr_localsid_fn,
+  .name = "sr-localsid",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_localsid_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_LOCALSID_N_ERROR,
+  .error_strings = sr_localsid_error_strings,
+  .n_next_nodes = SR_LOCALSID_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_LOCALSID_NEXT_##s] = n,
+    foreach_sr_localsid_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+static u8 *
+format_sr_dpo (u8 * s, va_list * args)
+{
+  index_t index = va_arg (*args, index_t);
+  CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
+
+  return (format (s, "SR: localsid_index:[%d]", index));
+}
+
+const static dpo_vft_t sr_loc_vft = {
+  .dv_lock = sr_dpo_lock,
+  .dv_unlock = sr_dpo_unlock,
+  .dv_format = format_sr_dpo,
+};
+
+const static char *const sr_loc_ip6_nodes[] = {
+  "sr-localsid",
+  NULL,
+};
+
+const static char *const *const sr_loc_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = sr_loc_ip6_nodes,
+};
+
+const static char *const sr_loc_d_ip6_nodes[] = {
+  "sr-localsid-d",
+  NULL,
+};
+
+const static char *const *const sr_loc_d_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = sr_loc_d_ip6_nodes,
+};
+
+
+/*************************** SR LocalSID plugins ******************************/
+/**
+ * @brief SR LocalSID plugin registry
+ */
+int
+sr_localsid_register_function (vlib_main_t * vm, u8 * fn_name,
+			       u8 * keyword_str, u8 * def_str,
+			       u8 * params_str, dpo_type_t * dpo,
+			       format_function_t * ls_format,
+			       unformat_function_t * ls_unformat,
+			       sr_plugin_callback_t * creation_fn,
+			       sr_plugin_callback_t * removal_fn)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  uword *p;
+
+  sr_localsid_fn_registration_t *plugin;
+
+  /* Did this function exist? If so update it */
+  p = hash_get_mem (sm->plugin_functions_by_key, fn_name);
+  if (p)
+    {
+      plugin = pool_elt_at_index (sm->plugin_functions, p[0]);
+    }
+  /* Else create a new one and set hash key */
+  else
+    {
+      pool_get (sm->plugin_functions, plugin);
+      hash_set_mem (sm->plugin_functions_by_key, fn_name,
+		    plugin - sm->plugin_functions);
+    }
+
+  memset (plugin, 0, sizeof (*plugin));
+
+  plugin->sr_localsid_function_number = (plugin - sm->plugin_functions);
+  plugin->sr_localsid_function_number += SR_BEHAVIOR_LAST;
+  plugin->ls_format = ls_format;
+  plugin->ls_unformat = ls_unformat;
+  plugin->creation = creation_fn;
+  plugin->removal = removal_fn;
+  clib_memcpy (&plugin->dpo, dpo, sizeof (dpo_type_t));
+  plugin->function_name = format (0, "%s%c", fn_name, 0);
+  plugin->keyword_str = format (0, "%s%c", keyword_str, 0);
+  plugin->def_str = format (0, "%s%c", def_str, 0);
+  plugin->params_str = format (0, "%s%c", params_str, 0);
+
+  return plugin->sr_localsid_function_number;
+}
+
+/**
+ * @brief CLI function to 'show' all available SR LocalSID behaviors
+ */
+static clib_error_t *
+show_sr_localsid_behaviors_command_fn (vlib_main_t * vm,
+				       unformat_input_t * input,
+				       vlib_cli_command_t * cmd)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  sr_localsid_fn_registration_t *plugin;
+  sr_localsid_fn_registration_t **plugins_vec = 0;
+  int i;
+
+  vlib_cli_output (vm,
+		   "SR LocalSIDs behaviors:\n-----------------------\n\n");
+
+  /* *INDENT-OFF* */
+  pool_foreach (plugin, sm->plugin_functions,
+    ({ vec_add1 (plugins_vec, plugin); }));
+  /* *INDENT-ON* */
+
+  /* Print static behaviors */
+  vlib_cli_output (vm, "Default behaviors:\n"
+		   "\tEnd\t-> Endpoint.\n"
+		   "\tEnd.X\t-> Endpoint with decapsulation and Layer-3 cross-connect.\n"
+		   "\t\tParameters: ' '\n"
+		   "\tEnd.DX2\t-> Endpoint with decapsulation and Layer-2 cross-connect.\n"
+		   "\t\tParameters: ''\n"
+		   "\tEnd.DX6\t-> Endpoint with decapsulation and IPv6 cross-connect.\n"
+		   "\t\tParameters: ' '\n"
+		   "\tEnd.DX4\t-> Endpoint with decapsulation and IPv4 cross-connect.\n"
+		   "\t\tParameters: ' '\n"
+		   "\tEnd.DT6\t-> Endpoint with decapsulation and specific IPv6 table lookup.\n"
+		   "\t\tParameters: ''\n"
+		   "\tEnd.DT4\t-> Endpoint with decapsulation and specific IPv4 table lookup.\n"
+		   "\t\tParameters: ''\n");
+  vlib_cli_output (vm, "Plugin behaviors:\n");
+  for (i = 0; i < vec_len (plugins_vec); i++)
+    {
+      plugin = plugins_vec[i];
+      vlib_cli_output (vm, "\t%s\t-> %s.\n", plugin->keyword_str,
+		       plugin->def_str);
+      vlib_cli_output (vm, "\t\tParameters: '%s'\n", plugin->params_str);
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_sr_localsid_behaviors_command, static) = {
+  .path = "show sr localsids behaviors",
+  .short_help = "show sr localsids behaviors",
+  .function = show_sr_localsid_behaviors_command_fn,
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief SR LocalSID initialization
+ */
+clib_error_t *
+sr_localsids_init (vlib_main_t * vm)
+{
+  /* Init memory for function keys */
+  ip6_sr_main_t *sm = &sr_main;
+  sm->localsids_index_by_key =
+    hash_create_mem (0, sizeof (ip6_address_t), sizeof (uword));
+  /* Init SR behaviors DPO type */
+  sr_localsid_dpo_type = dpo_register_new_type (&sr_loc_vft, sr_loc_nodes);
+  /* Init SR behaviors DPO type */
+  sr_localsid_d_dpo_type =
+    dpo_register_new_type (&sr_loc_vft, sr_loc_d_nodes);
+  /* Init memory for localsid plugins */
+  sm->plugin_functions_by_key = hash_create_string (0, sizeof (uword));
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (sr_localsids_init);
+/*
+* fd.io coding-style-patch-verification: ON
+*
+* Local Variables:
+* eval: (c-set-style "gnu")
+* End:
+*/
diff --git a/src/vnet/sr/sr_packet.h b/src/vnet/sr/sr_packet.h
old mode 100644
new mode 100755
index 179b94c2..7af4ad4d
--- a/src/vnet/sr/sr_packet.h
+++ b/src/vnet/sr/sr_packet.h
@@ -20,60 +20,36 @@
  * limitations under the License.
  */
 
-/**
- *  @file
- *  @brief The Segment Routing Header (SRH).
- *
- *  The Segment Routing Header (SRH) is defined in the diagram below.
- *
- *
- *     0                   1                   2                   3
- *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    | Next Header   |  Hdr Ext Len  | Routing Type  | Segments Left |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    | First Segment |             Flags             |  HMAC Key ID  |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |            Segment List[0] (128 bits ipv6 address)            |
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |                                                               |
- *                                  ...
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |            Segment List[n] (128 bits ipv6 address)            |
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |            Policy List[0] (optional)                          |
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |            Policy List[1] (optional)                          |
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |            Policy List[2] (optional)                          |
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |                                                               |
- *    |                                                               |
- *    |                                                               |
- *    |                       HMAC (256 bits)                         |
- *    |                        (optional)                             |
- *    |                                                               |
- *    |                                                               |
- *    |                                                               |
- *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/*
+ *   The Segment Routing Header (SRH) is defined as follows:
+ *
+ *    0                   1                   2                   3
+ *    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   | Next Header   |  Hdr Ext Len  | Routing Type  | Segments Left |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   | First Segment |     Flags     |           RESERVED            |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   |                                                               |
+ *   |            Segment List[0] (128 bits IPv6 address)            |
+ *   |                                                               |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   |                                                               |
+ *   |                                                               |
+ *                                 ...
+ *   |                                                               |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   |                                                               |
+ *   |            Segment List[n] (128 bits IPv6 address)            |
+ *   |                                                               |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   //                                                             //
+ *   //         Optional Type Length Value objects (variable)       //
+ *   //                                                             //
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *
  *   where:
  *
@@ -87,57 +63,39 @@
  *
  *   o  Segments Left.  Defined in [RFC2460], it contains the index, in
  *      the Segment List, of the next segment to inspect.  Segments Left
- *      is decremented at each segment and it is used as an index in the
- *      segment list.
- *
- *   o  First Segment: offset in the SRH, not including the first 8 octets
- *      and expressed in 16-octet units, pointing to the last element of
- *      the segment list, which is in fact the first segment of the
- *      segment routing path.
- *
- *   o  Flags: 16 bits of flags.  Following flags are defined:
- *
- *                              1
- *          0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
- *         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *         |C|P|R|R|    Policy Flags       |
- *         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *
- *         C-flag: Clean-up flag.  Set when the SRH has to be removed from
- *         the packet when packet reaches the last segment.
- *
- *         P-flag: Protected flag.  Set when the packet has been rerouted
- *         through FRR mechanism by a SR endpoint node.  See Section 6.3
- *         for more details.
- *
- *         R-flags.  Reserved and for future use.
- *
- *         Policy Flags.  Define the type of the IPv6 addresses encoded
- *         into the Policy List (see below).  The following have been
- *         defined:
+ *      is decremented at each segment.
  *
- *            Bits 4-6: determine the type of the first element after the
- *            segment list.
+ *   o  First Segment: contains the index, in the Segment List, of the
+ *      first segment of the path which is in fact the last element of the
+ *      Segment List.
  *
- *            Bits 7-9: determine the type of the second element.
+ *   o  Flags: 8 bits of flags.  Following flags are defined:
  *
- *            Bits 10-12: determine the type of the third element.
+ *         0 1 2 3 4 5 6 7
+ *        +-+-+-+-+-+-+-+-+
+ *        |U|P|O|A|H|  U  |
+ *        +-+-+-+-+-+-+-+-+
  *
- *            Bits 13-15: determine the type of the fourth element.
+ *        U: Unused and for future use.  SHOULD be unset on transmission
+ *        and MUST be ignored on receipt.
  *
- *         The following values are used for the type:
+ *        P-flag: Protected flag.  Set when the packet has been rerouted
+ *        through FRR mechanism by an SR endpoint node.
  *
- *            0x0: Not present.  If value is set to 0x0, it means the
- *            element represented by these bits is not present.
+ *        O-flag: OAM flag.  When set, it indicates that this packet is
+ *        an operations and management (OAM) packet.
  *
- *            0x1: SR Ingress.
+ *        A-flag: Alert flag.  If present, it means important Type Length
+ *        Value (TLV) objects are present.  See Section 3.1 for details
+ *        on TLVs objects.
  *
- *            0x2: SR Egress.
+ *        H-flag: HMAC flag.  If set, the HMAC TLV is present and is
+ *        encoded as the last TLV of the SRH.  In other words, the last
+ *        36 octets of the SRH represent the HMAC information.  See
+ *        Section 3.1.5 for details on the HMAC TLV.
  *
- *            0x3: Original Source Address.
- *
- *   o  HMAC Key ID and HMAC field, and their use are defined in
- *      [I-D.vyncke-6man-segment-routing-security].
+ *   o  RESERVED: SHOULD be unset on transmission and MUST be ignored on
+ *      receipt.
  *
  *   o  Segment List[n]: 128 bit IPv6 addresses representing the nth
  *      segment in the Segment List.  The Segment List is encoded starting
@@ -147,23 +105,8 @@
  *      contains the first segment of the path.  The index contained in
  *      "Segments Left" identifies the current active segment.
  *
- *   o  Policy List.  Optional addresses representing specific nodes in
- *      the SR path such as:
- *
- *         SR Ingress: a 128 bit generic identifier representing the
- *         ingress in the SR domain (i.e.: it needs not to be a valid IPv6
- *         address).
- *
- *         SR Egress: a 128 bit generic identifier representing the egress
- *         in the SR domain (i.e.: it needs not to be a valid IPv6
- *         address).
- *
- *         Original Source Address: IPv6 address originally present in the
- *         SA field of the packet.
+ *   o  Type Length Value (TLV) are described in Section 3.1.
  *
- *      The segments in the Policy List are encoded after the segment list
- *      and they are optional.  If none are in the SRH, all bits of the
- *      Policy List Flags MUST be set to 0x0.
  */
 
 #ifndef IPPROTO_IPV6_ROUTE
@@ -171,81 +114,46 @@
 #endif
 
 #define ROUTING_HEADER_TYPE_SR    4
-/**
-    @brief SR header struct.
-*/
+
 typedef struct
 {
-  /** Protocol for next header. */
+  /* Protocol for next header. */
   u8 protocol;
-
-  /**
+  /*
    * Length of routing header in 8 octet units,
    * not including the first 8 octets
    */
   u8 length;
 
-  /** Type of routing header; type 4 = segement routing */
+  /* Type of routing header; type 4 = segement routing */
   u8 type;
 
-  /** Next segment in the segment list */
+  /* Next segment in the segment list */
   u8 segments_left;
 
-  /**
-   * Policy list pointer: offset in the SRH of the policy
-   * list - in 16-octet units - not including the first 8 octets.
-   */
+  /* Pointer to the first segment in the header */
   u8 first_segment;
 
-  /** Flag bits */
-#define IP6_SR_HEADER_FLAG_CLEANUP    (0x8000)
-  /** Flag bits */
-#define IP6_SR_HEADER_FLAG_PROTECTED  (0x4000)
-  /** Flag bits */
-#define IP6_SR_HEADER_FLAG_RESERVED   (0x3000)
-  /** Flag bits */
-#define IP6_SR_HEADER_FLAG_PL_ELT_NOT_PRESENT (0x0)
-    /** Flag bits */
-#define IP6_SR_HEADER_FLAG_PL_ELT_INGRESS_PE (0x1)
-    /** Flag bits */
-#define IP6_SR_HEADER_FLAG_PL_ELT_EGRESS_PE (0x2)
-    /** Flag bits */
-#define IP6_SR_HEADER_FLAG_PL_ELT_ORIG_SRC_ADDR (0x3)
-  /** values 0x4 - 0x7 are reserved */
-  u16 flags;
-  u8 hmac_key;
+  /* Flag bits */
+#define IP6_SR_HEADER_FLAG_PROTECTED  (0x40)
+#define IP6_SR_HEADER_FLAG_OAM        (0x20)
+#define IP6_SR_HEADER_FLAG_ALERT      (0x10)
+#define IP6_SR_HEADER_FLAG_HMAC       (0x80)
+
+  /* values 0x0, 0x4 - 0x7 are reserved */
+  u8 flags;
+  u16 reserved;
 
-  /** The segment + policy list elts */
+  /* The segment elts */
   ip6_address_t segments[0];
 } __attribute__ ((packed)) ip6_sr_header_t;
 
-static inline int
-ip6_sr_policy_list_shift_from_index (int pl_index)
-{
-  return (-3 * pl_index) + 12;
-}
-
-/** pl_index is one-origined */
-static inline int
-ip6_sr_policy_list_flags (u16 flags_host_byte_order, int pl_index)
-{
-  int shift;
-
-  if (pl_index <= 0 || pl_index > 4)
-    return 0;
-
-  shift = (-3 * pl_index) + 12;
-  flags_host_byte_order >>= shift;
-
-  return (flags_host_byte_order & 7);
-}
+/*
+* fd.io coding-style-patch-verification: ON
+*
+* Local Variables:
+* eval: (c-set-style "gnu")
+* End:
+*/
 
 #endif /* included_vnet_sr_packet_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/sr/sr_policy_rewrite.c b/src/vnet/sr/sr_policy_rewrite.c
new file mode 100755
index 00000000..1f8bdca5
--- /dev/null
+++ b/src/vnet/sr/sr_policy_rewrite.c
@@ -0,0 +1,3253 @@
+/*
+ * sr_policy_rewrite.c: ipv6 sr policy creation
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ * @brief SR policy creation and application
+ *
+ * Create an SR policy.
+ * An SR policy can be either of 'default' type or 'spray' type
+ * An SR policy has attached a list of SID lists.
+ * In case the SR policy is a default one it will load balance among them.
+ * An SR policy has associated a BindingSID.
+ * In case any packet arrives with IPv6 DA == BindingSID then the SR policy
+ * associated to such bindingSID will be applied to such packet.
+ *
+ * SR policies can be applied either by using IPv6 encapsulation or
+ * SRH insertion. Both methods can be found on this file.
+ *
+ * Traffic input usually is IPv6 packets. However it is possible to have
+ * IPv4 packets or L2 frames. (that are encapsulated into IPv6 with SRH)
+ *
+ * This file provides the appropiates VPP graph nodes to do any of these
+ * methods.
+ *
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+/**
+ * @brief SR policy rewrite trace
+ */
+typedef struct
+{
+  ip6_address_t src, dst;
+} sr_policy_rewrite_trace_t;
+
+/* Graph arcs */
+#define foreach_sr_policy_rewrite_next     \
+_(IP6_LOOKUP, "ip6-lookup")         \
+_(ERROR, "error-drop")
+
+typedef enum
+{
+#define _(s,n) SR_POLICY_REWRITE_NEXT_##s,
+  foreach_sr_policy_rewrite_next
+#undef _
+    SR_POLICY_REWRITE_N_NEXT,
+} sr_policy_rewrite_next_t;
+
+/* SR rewrite errors */
+#define foreach_sr_policy_rewrite_error                     \
+_(INTERNAL_ERROR, "Segment Routing undefined error")        \
+_(BSID_ZERO, "BSID with SL = 0")                            \
+_(COUNTER_TOTAL, "SR steered IPv6 packets")                 \
+_(COUNTER_ENCAP, "SR: Encaps packets")                      \
+_(COUNTER_INSERT, "SR: SRH inserted packets")               \
+_(COUNTER_BSID, "SR: BindingSID steered packets")
+
+typedef enum
+{
+#define _(sym,str) SR_POLICY_REWRITE_ERROR_##sym,
+  foreach_sr_policy_rewrite_error
+#undef _
+    SR_POLICY_REWRITE_N_ERROR,
+} sr_policy_rewrite_error_t;
+
+static char *sr_policy_rewrite_error_strings[] = {
+#define _(sym,string) string,
+  foreach_sr_policy_rewrite_error
+#undef _
+};
+
+/**
+ * @brief Dynamically added SR SL DPO type
+ */
+static dpo_type_t sr_pr_encaps_dpo_type;
+static dpo_type_t sr_pr_insert_dpo_type;
+static dpo_type_t sr_pr_bsid_encaps_dpo_type;
+static dpo_type_t sr_pr_bsid_insert_dpo_type;
+
+/**
+ * @brief IPv6 SA for encapsulated packets
+ */
+static ip6_address_t sr_pr_encaps_src;
+
+/******************* SR rewrite set encaps IPv6 source addr *******************/
+/* Note:  This is temporal. We don't know whether to follow this path or
+          take the ip address of a loopback interface or even the OIF         */
+
+static clib_error_t *
+set_sr_src_command_fn (vlib_main_t * vm, unformat_input_t * input,
+		       vlib_cli_command_t * cmd)
+{
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (input, "addr %U", unformat_ip6_address, &sr_pr_encaps_src))
+	return 0;
+      else
+	return clib_error_return (0, "No address specified");
+    }
+  return clib_error_return (0, "No address specified");
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_sr_src_command, static) = {
+  .path = "set sr encaps source",
+  .short_help = "set sr encaps source addr ",
+  .function = set_sr_src_command_fn,
+};
+/* *INDENT-ON* */
+
+/*********************** SR rewrite string computation ************************/
+/**
+ * @brief SR rewrite string computation for IPv6 encapsulation (inline)
+ *
+ * @param sl is a vector of IPv6 addresses composing the Segment List
+ *
+ * @return precomputed rewrite string for encapsulation
+ */
+static inline u8 *
+compute_rewrite_encaps (ip6_address_t * sl)
+{
+  ip6_header_t *iph;
+  ip6_sr_header_t *srh;
+  ip6_address_t *addrp, *this_address;
+  u32 header_length = 0;
+  u8 *rs = NULL;
+
+  header_length = 0;
+  header_length += IPv6_DEFAULT_HEADER_LENGTH;
+  if (vec_len (sl) > 1)
+    {
+      header_length += sizeof (ip6_sr_header_t);
+      header_length += vec_len (sl) * sizeof (ip6_address_t);
+    }
+
+  vec_validate (rs, header_length - 1);
+
+  iph = (ip6_header_t *) rs;
+  iph->ip_version_traffic_class_and_flow_label =
+    clib_host_to_net_u32 (0 | ((6 & 0xF) << 28));
+  iph->src_address.as_u64[0] = sr_pr_encaps_src.as_u64[0];
+  iph->src_address.as_u64[1] = sr_pr_encaps_src.as_u64[1];
+  iph->payload_length = header_length - IPv6_DEFAULT_HEADER_LENGTH;
+  iph->protocol = IP_PROTOCOL_IPV6;
+  iph->hop_limit = IPv6_DEFAULT_HOP_LIMIT;
+
+  srh = (ip6_sr_header_t *) (iph + 1);
+  iph->protocol = IP_PROTOCOL_IPV6_ROUTE;
+  srh->protocol = IP_PROTOCOL_IPV6;
+  srh->type = ROUTING_HEADER_TYPE_SR;
+  srh->segments_left = vec_len (sl) - 1;
+  srh->first_segment = vec_len (sl) - 1;
+  srh->length = ((sizeof (ip6_sr_header_t) +
+		  (vec_len (sl) * sizeof (ip6_address_t))) / 8) - 1;
+  srh->flags = 0x00;
+  srh->reserved = 0x00;
+  addrp = srh->segments + vec_len (sl) - 1;
+  vec_foreach (this_address, sl)
+  {
+    clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t));
+    addrp--;
+  }
+  iph->dst_address.as_u64[0] = sl->as_u64[0];
+  iph->dst_address.as_u64[1] = sl->as_u64[1];
+  return rs;
+}
+
+/**
+ * @brief SR rewrite string computation for SRH insertion (inline)
+ *
+ * @param sl is a vector of IPv6 addresses composing the Segment List
+ *
+ * @return precomputed rewrite string for SRH insertion
+ */
+static inline u8 *
+compute_rewrite_insert (ip6_address_t * sl)
+{
+  ip6_sr_header_t *srh;
+  ip6_address_t *addrp, *this_address;
+  u32 header_length = 0;
+  u8 *rs = NULL;
+
+  header_length = 0;
+  header_length += sizeof (ip6_sr_header_t);
+  header_length += (vec_len (sl) + 1) * sizeof (ip6_address_t);
+
+  vec_validate (rs, header_length - 1);
+
+  srh = (ip6_sr_header_t *) rs;
+  srh->type = ROUTING_HEADER_TYPE_SR;
+  srh->segments_left = vec_len (sl);
+  srh->first_segment = vec_len (sl);
+  srh->length = ((sizeof (ip6_sr_header_t) +
+		  ((vec_len (sl) + 1) * sizeof (ip6_address_t))) / 8) - 1;
+  srh->flags = 0x00;
+  srh->reserved = 0x0000;
+  addrp = srh->segments + vec_len (sl);
+  vec_foreach (this_address, sl)
+  {
+    clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t));
+    addrp--;
+  }
+  return rs;
+}
+
+/**
+ * @brief SR rewrite string computation for SRH insertion with BSID (inline)
+ *
+ * @param sl is a vector of IPv6 addresses composing the Segment List
+ *
+ * @return precomputed rewrite string for SRH insertion with BSID
+ */
+static inline u8 *
+compute_rewrite_bsid (ip6_address_t * sl)
+{
+  ip6_sr_header_t *srh;
+  ip6_address_t *addrp, *this_address;
+  u32 header_length = 0;
+  u8 *rs = NULL;
+
+  header_length = 0;
+  header_length += sizeof (ip6_sr_header_t);
+  header_length += vec_len (sl) * sizeof (ip6_address_t);
+
+  vec_validate (rs, header_length - 1);
+
+  srh = (ip6_sr_header_t *) rs;
+  srh->type = ROUTING_HEADER_TYPE_SR;
+  srh->segments_left = vec_len (sl) - 1;
+  srh->first_segment = vec_len (sl) - 1;
+  srh->length = ((sizeof (ip6_sr_header_t) +
+		  (vec_len (sl) * sizeof (ip6_address_t))) / 8) - 1;
+  srh->flags = 0x00;
+  srh->reserved = 0x0000;
+  addrp = srh->segments + vec_len (sl) - 1;
+  vec_foreach (this_address, sl)
+  {
+    clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t));
+    addrp--;
+  }
+  return rs;
+}
+
+/***************************  SR LB helper functions **************************/
+/**
+ * @brief Creates a Segment List and adds it to an SR policy
+ *
+ * Creates a Segment List and adds it to the SR policy. Notice that the SL are
+ * not necessarily unique. Hence there might be two Segment List within the
+ * same SR Policy with exactly the same segments and same weight.
+ *
+ * @param sr_policy is the SR policy where the SL will be added
+ * @param sl is a vector of IPv6 addresses composing the Segment List
+ * @param weight is the weight of the SegmentList (for load-balancing purposes)
+ * @param is_encap represents the mode (SRH insertion vs Encapsulation)
+ *
+ * @return pointer to the just created segment list
+ */
+static inline ip6_sr_sl_t *
+create_sl (ip6_sr_policy_t * sr_policy, ip6_address_t * sl, u32 weight,
+	   u8 is_encap)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_sr_sl_t *segment_list;
+
+  pool_get (sm->sid_lists, segment_list);
+  memset (segment_list, 0, sizeof (*segment_list));
+
+  vec_add1 (sr_policy->segments_lists, segment_list - sm->sid_lists);
+
+  /* Fill in segment list */
+  segment_list->weight =
+    (weight != (u32) ~ 0 ? weight : SR_SEGMENT_LIST_WEIGHT_DEFAULT);
+  segment_list->segments = vec_dup (sl);
+
+  if (is_encap)
+    {
+      segment_list->rewrite = compute_rewrite_encaps (sl);
+      segment_list->rewrite_bsid = segment_list->rewrite;
+    }
+  else
+    {
+      segment_list->rewrite = compute_rewrite_insert (sl);
+      segment_list->rewrite_bsid = compute_rewrite_bsid (sl);
+    }
+
+  /* Create DPO */
+  dpo_reset (&segment_list->bsid_dpo);
+  dpo_reset (&segment_list->ip6_dpo);
+  dpo_reset (&segment_list->ip4_dpo);
+
+  if (is_encap)
+    {
+      dpo_set (&segment_list->ip6_dpo, sr_pr_encaps_dpo_type, DPO_PROTO_IP6,
+	       segment_list - sm->sid_lists);
+      dpo_set (&segment_list->ip4_dpo, sr_pr_encaps_dpo_type, DPO_PROTO_IP4,
+	       segment_list - sm->sid_lists);
+      dpo_set (&segment_list->bsid_dpo, sr_pr_bsid_encaps_dpo_type,
+	       DPO_PROTO_IP6, segment_list - sm->sid_lists);
+    }
+  else
+    {
+      dpo_set (&segment_list->ip6_dpo, sr_pr_insert_dpo_type, DPO_PROTO_IP6,
+	       segment_list - sm->sid_lists);
+      dpo_set (&segment_list->bsid_dpo, sr_pr_bsid_insert_dpo_type,
+	       DPO_PROTO_IP6, segment_list - sm->sid_lists);
+    }
+
+  return segment_list;
+}
+
+/**
+ * @brief Updates the Load Balancer after an SR Policy change
+ *
+ * @param sr_policy is the modified SR Policy
+ */
+static inline void
+update_lb (ip6_sr_policy_t * sr_policy)
+{
+  flow_hash_config_t fhc;
+  u32 *sl_index;
+  ip6_sr_sl_t *segment_list;
+  ip6_sr_main_t *sm = &sr_main;
+  load_balance_path_t path;
+  load_balance_path_t *ip4_path_vector = 0;
+  load_balance_path_t *ip6_path_vector = 0;
+  load_balance_path_t *b_path_vector = 0;
+
+  /* In case LB does not exist, create it */
+  if (!dpo_id_is_valid (&sr_policy->bsid_dpo))
+    {
+      fib_prefix_t pfx = {
+	.fp_proto = FIB_PROTOCOL_IP6,
+	.fp_len = 128,
+	.fp_addr = {
+		    .ip6 = sr_policy->bsid,
+		    }
+      };
+
+      /* Add FIB entry for BSID */
+      fhc = fib_table_get_flow_hash_config (sr_policy->fib_table,
+					    dpo_proto_to_fib (DPO_PROTO_IP6));
+
+      dpo_set (&sr_policy->bsid_dpo, DPO_LOAD_BALANCE, DPO_PROTO_IP6,
+	       load_balance_create (0, DPO_PROTO_IP6, fhc));
+
+      dpo_set (&sr_policy->ip6_dpo, DPO_LOAD_BALANCE, DPO_PROTO_IP6,
+	       load_balance_create (0, DPO_PROTO_IP6, fhc));
+
+      /* Update FIB entry's to point to the LB DPO in the main FIB and hidden one */
+      fib_table_entry_special_dpo_update (fib_table_id_find_fib_index
+					  (FIB_PROTOCOL_IP6,
+					   sr_policy->fib_table), &pfx,
+					  FIB_SOURCE_SR,
+					  FIB_ENTRY_FLAG_EXCLUSIVE,
+					  &sr_policy->bsid_dpo);
+
+      fib_table_entry_special_dpo_update (sm->fib_table_ip6,
+					  &pfx,
+					  FIB_SOURCE_SR,
+					  FIB_ENTRY_FLAG_EXCLUSIVE,
+					  &sr_policy->ip6_dpo);
+
+      if (sr_policy->is_encap)
+	{
+	  dpo_set (&sr_policy->ip4_dpo, DPO_LOAD_BALANCE, DPO_PROTO_IP4,
+		   load_balance_create (0, DPO_PROTO_IP4, fhc));
+
+	  fib_table_entry_special_dpo_update (sm->fib_table_ip4,
+					      &pfx,
+					      FIB_SOURCE_SR,
+					      FIB_ENTRY_FLAG_EXCLUSIVE,
+					      &sr_policy->ip4_dpo);
+	}
+
+    }
+
+  /* Create the LB path vector */
+  //path_vector = vec_new(load_balance_path_t, vec_len(sr_policy->segments_lists));
+  vec_foreach (sl_index, sr_policy->segments_lists)
+  {
+    segment_list = pool_elt_at_index (sm->sid_lists, *sl_index);
+    path.path_dpo = segment_list->bsid_dpo;
+    path.path_weight = segment_list->weight;
+    vec_add1 (b_path_vector, path);
+    path.path_dpo = segment_list->ip6_dpo;
+    vec_add1 (ip6_path_vector, path);
+    if (sr_policy->is_encap)
+      {
+	path.path_dpo = segment_list->ip4_dpo;
+	vec_add1 (ip4_path_vector, path);
+      }
+  }
+
+  /* Update LB multipath */
+  load_balance_multipath_update (&sr_policy->bsid_dpo, b_path_vector,
+				 LOAD_BALANCE_FLAG_NONE);
+  load_balance_multipath_update (&sr_policy->ip6_dpo, ip6_path_vector,
+				 LOAD_BALANCE_FLAG_NONE);
+  if (sr_policy->is_encap)
+    load_balance_multipath_update (&sr_policy->ip4_dpo, ip4_path_vector,
+				   LOAD_BALANCE_FLAG_NONE);
+
+  /* Cleanup */
+  vec_free (b_path_vector);
+  vec_free (ip6_path_vector);
+  vec_free (ip4_path_vector);
+
+}
+
+/**
+ * @brief Updates the Replicate DPO after an SR Policy change
+ *
+ * @param sr_policy is the modified SR Policy (type spray)
+ */
+static inline void
+update_replicate (ip6_sr_policy_t * sr_policy)
+{
+  u32 *sl_index;
+  ip6_sr_sl_t *segment_list;
+  ip6_sr_main_t *sm = &sr_main;
+  load_balance_path_t path;
+  load_balance_path_t *b_path_vector = 0;
+  load_balance_path_t *ip6_path_vector = 0;
+  load_balance_path_t *ip4_path_vector = 0;
+
+  /* In case LB does not exist, create it */
+  if (!dpo_id_is_valid (&sr_policy->bsid_dpo))
+    {
+      dpo_set (&sr_policy->bsid_dpo, DPO_REPLICATE,
+	       DPO_PROTO_IP6, replicate_create (0, DPO_PROTO_IP6));
+
+      dpo_set (&sr_policy->ip6_dpo, DPO_REPLICATE,
+	       DPO_PROTO_IP6, replicate_create (0, DPO_PROTO_IP6));
+
+      /* Update FIB entry's DPO to point to SR without LB */
+      fib_prefix_t pfx = {
+	.fp_proto = FIB_PROTOCOL_IP6,
+	.fp_len = 128,
+	.fp_addr = {
+		    .ip6 = sr_policy->bsid,
+		    }
+      };
+      fib_table_entry_special_dpo_update (fib_table_id_find_fib_index
+					  (FIB_PROTOCOL_IP6,
+					   sr_policy->fib_table), &pfx,
+					  FIB_SOURCE_SR,
+					  FIB_ENTRY_FLAG_EXCLUSIVE,
+					  &sr_policy->bsid_dpo);
+
+      fib_table_entry_special_dpo_update (sm->fib_table_ip6,
+					  &pfx,
+					  FIB_SOURCE_SR,
+					  FIB_ENTRY_FLAG_EXCLUSIVE,
+					  &sr_policy->ip6_dpo);
+
+      if (sr_policy->is_encap)
+	{
+	  dpo_set (&sr_policy->ip4_dpo, DPO_REPLICATE, DPO_PROTO_IP4,
+		   replicate_create (0, DPO_PROTO_IP4));
+
+	  fib_table_entry_special_dpo_update (sm->fib_table_ip4,
+					      &pfx,
+					      FIB_SOURCE_SR,
+					      FIB_ENTRY_FLAG_EXCLUSIVE,
+					      &sr_policy->ip4_dpo);
+	}
+
+    }
+
+  /* Create the replicate path vector */
+  path.path_weight = 1;
+  vec_foreach (sl_index, sr_policy->segments_lists)
+  {
+    segment_list = pool_elt_at_index (sm->sid_lists, *sl_index);
+    path.path_dpo = segment_list->bsid_dpo;
+    vec_add1 (b_path_vector, path);
+    path.path_dpo = segment_list->ip6_dpo;
+    vec_add1 (ip6_path_vector, path);
+    if (sr_policy->is_encap)
+      {
+	path.path_dpo = segment_list->ip4_dpo;
+	vec_add1 (ip4_path_vector, path);
+      }
+  }
+
+  /* Update replicate multipath */
+  replicate_multipath_update (&sr_policy->bsid_dpo, b_path_vector);
+  replicate_multipath_update (&sr_policy->ip6_dpo, ip6_path_vector);
+  if (sr_policy->is_encap)
+    replicate_multipath_update (&sr_policy->ip4_dpo, ip4_path_vector);
+
+  /* Cleanup */
+  vec_free (b_path_vector);
+  vec_free (ip6_path_vector);
+  vec_free (ip4_path_vector);
+}
+
+/******************************* SR rewrite API *******************************/
+/* Three functions for handling sr policies:
+ *   -> sr_policy_add
+ *   -> sr_policy_del
+ *   -> sr_policy_mod
+ * All of them are API. CLI function on sr_policy_command_fn                  */
+
+/**
+ * @brief Create a new SR policy
+ *
+ * @param bsid is the bindingSID of the SR Policy
+ * @param segments is a vector of IPv6 address composing the segment list
+ * @param weight is the weight of the sid list. optional.
+ * @param behavior is the behavior of the SR policy. (default//spray)
+ * @param fib_table is the VRF where to install the FIB entry for the BSID
+ * @param is_encap (bool) whether SR policy should behave as Encap/SRH Insertion
+ *
+ * @return 0 if correct, else error
+ */
+int
+sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments,
+	       u32 weight, u8 behavior, u32 fib_table, u8 is_encap)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_sr_policy_t *sr_policy = 0;
+  ip6_address_t *key_copy;
+  uword *p;
+
+  /* Search for existing keys (BSID) */
+  p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+  if (p)
+    {
+      /* Add SR policy that already exists; complain */
+      return -12;
+    }
+
+  /* Search collision in FIB entries */
+  /* Explanation: It might be possible that some other entity has already
+   * created a route for the BSID. This in theory is impossible, but in
+   * practise we could see it. Assert it and scream if needed */
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP6,
+    .fp_len = 128,
+    .fp_addr = {
+		.ip6 = *bsid,
+		}
+  };
+
+  /* Lookup the FIB index associated to the table selected */
+  u32 fib_index = fib_table_id_find_fib_index (FIB_PROTOCOL_IP6,
+					       (fib_table !=
+						(u32) ~ 0 ? fib_table : 0));
+  if (fib_index == ~0)
+    return -13;
+
+  /* Lookup whether there exists an entry for the BSID */
+  fib_node_index_t fei = fib_table_lookup_exact_match (fib_index, &pfx);
+  if (FIB_NODE_INDEX_INVALID != fei)
+    return -12;			//There is an entry for such lookup
+
+  /* Add an SR policy object */
+  pool_get (sm->sr_policies, sr_policy);
+  memset (sr_policy, 0, sizeof (*sr_policy));
+  clib_memcpy (&sr_policy->bsid, bsid, sizeof (ip6_address_t));
+  sr_policy->type = behavior;
+  sr_policy->fib_table = (fib_table != (u32) ~ 0 ? fib_table : 0);	//Is default FIB 0 ?
+  sr_policy->is_encap = is_encap;
+
+  /* Copy the key */
+  key_copy = vec_new (ip6_address_t, 1);
+  clib_memcpy (key_copy, bsid, sizeof (ip6_address_t));
+  hash_set_mem (sm->sr_policy_index_by_key, key_copy,
+		sr_policy - sm->sr_policies);
+
+  /* Create a segment list and add the index to the SR policy */
+  create_sl (sr_policy, segments, weight, is_encap);
+
+  /* If FIB doesnt exist, create them */
+  if (sm->fib_table_ip6 == (u32) ~ 0)
+    {
+      sm->fib_table_ip6 = fib_table_create_and_lock (FIB_PROTOCOL_IP6,
+						     "SRv6 steering of IP6 prefixes through BSIDs");
+      sm->fib_table_ip4 = fib_table_create_and_lock (FIB_PROTOCOL_IP6,
+						     "SRv6 steering of IP4 prefixes through BSIDs");
+      fib_table_flush (sm->fib_table_ip6, FIB_PROTOCOL_IP6,
+		       FIB_SOURCE_SPECIAL);
+      fib_table_flush (sm->fib_table_ip4, FIB_PROTOCOL_IP6,
+		       FIB_SOURCE_SPECIAL);
+    }
+
+  /* Create IPv6 FIB for the BindingSID attached to the DPO of the only SL */
+  if (sr_policy->type == SR_POLICY_TYPE_DEFAULT)
+    update_lb (sr_policy);
+  else if (sr_policy->type == SR_POLICY_TYPE_SPRAY)
+    update_replicate (sr_policy);
+  return 0;
+}
+
+/**
+ * @brief Delete a SR policy
+ *
+ * @param bsid is the bindingSID of the SR Policy
+ * @param index is the index of the SR policy
+ *
+ * @return 0 if correct, else error
+ */
+int
+sr_policy_del (ip6_address_t * bsid, u32 index)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_sr_policy_t *sr_policy = 0;
+  ip6_sr_sl_t *segment_list;
+  ip6_address_t *key_copy;
+  u32 *sl_index;
+  uword *p;
+
+  hash_pair_t *hp;
+  if (bsid)
+    {
+      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+      if (p)
+	sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
+      else
+	return -1;
+    }
+  else
+    {
+      sr_policy = pool_elt_at_index (sm->sr_policies, index);
+      if (!sr_policy)
+	return -1;
+    }
+
+  /* Remove BindingSID FIB entry */
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP6,
+    .fp_len = 128,
+    .fp_addr = {
+		.ip6 = sr_policy->bsid,
+		}
+    ,
+  };
+
+  fib_table_entry_special_remove (fib_table_id_find_fib_index
+				  (FIB_PROTOCOL_IP6, sr_policy->fib_table),
+				  &pfx, FIB_SOURCE_SR);
+
+  fib_table_entry_special_remove (sm->fib_table_ip6, &pfx, FIB_SOURCE_SR);
+
+  if (sr_policy->is_encap)
+    fib_table_entry_special_remove (sm->fib_table_ip4, &pfx, FIB_SOURCE_SR);
+
+  if (dpo_id_is_valid (&sr_policy->bsid_dpo))
+    {
+      dpo_reset (&sr_policy->bsid_dpo);
+      dpo_reset (&sr_policy->ip4_dpo);
+      dpo_reset (&sr_policy->ip6_dpo);
+    }
+
+  /* Clean SID Lists */
+  vec_foreach (sl_index, sr_policy->segments_lists)
+  {
+    segment_list = pool_elt_at_index (sm->sid_lists, *sl_index);
+    vec_free (segment_list->segments);
+    vec_free (segment_list->rewrite);
+    vec_free (segment_list->rewrite_bsid);
+    pool_put_index (sm->sid_lists, *sl_index);
+  }
+
+  /* Remove SR policy entry */
+  hp = hash_get_pair (sm->sr_policy_index_by_key, &sr_policy->bsid);
+  key_copy = (void *) (hp->key);
+  hash_unset_mem (sm->sr_policy_index_by_key, &sr_policy->bsid);
+  vec_free (key_copy);
+  pool_put (sm->sr_policies, sr_policy);
+
+  /* If FIB empty unlock it */
+  if (!pool_elts (sm->sr_policies))
+    {
+      fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6);
+      fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6);
+      sm->fib_table_ip6 = (u32) ~ 0;
+      sm->fib_table_ip4 = (u32) ~ 0;
+    }
+
+  return 0;
+}
+
+/**
+ * @brief Modify an existing SR policy
+ *
+ * The possible modifications are adding a new Segment List, modifying an
+ * existing Segment List (modify the weight only) and delete a given
+ * Segment List from the SR Policy.
+ *
+ * @param bsid is the bindingSID of the SR Policy
+ * @param index is the index of the SR policy
+ * @param fib_table is the VRF where to install the FIB entry for the BSID
+ * @param operation is the operation to perform (among the top ones)
+ * @param segments is a vector of IPv6 address composing the segment list
+ * @param sl_index is the index of the Segment List to modify/delete
+ * @param weight is the weight of the sid list. optional.
+ * @param is_encap Mode. Encapsulation or SRH insertion.
+ *
+ * @return 0 if correct, else error
+ */
+int
+sr_policy_mod (ip6_address_t * bsid, u32 index, u32 fib_table,
+	       u8 operation, ip6_address_t * segments, u32 sl_index,
+	       u32 weight)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_sr_policy_t *sr_policy = 0;
+  ip6_sr_sl_t *segment_list;
+  u32 *sl_index_iterate;
+  uword *p;
+
+  if (bsid)
+    {
+      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+      if (p)
+	sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
+      else
+	return -1;
+    }
+  else
+    {
+      sr_policy = pool_elt_at_index (sm->sr_policies, index);
+      if (!sr_policy)
+	return -1;
+    }
+
+  if (operation == 1)		/* Add SR List to an existing SR policy */
+    {
+      /* Create the new SL */
+      segment_list =
+	create_sl (sr_policy, segments, weight, sr_policy->is_encap);
+
+      /* Create a new LB DPO */
+      if (sr_policy->type == SR_POLICY_TYPE_DEFAULT)
+	update_lb (sr_policy);
+      else if (sr_policy->type == SR_POLICY_TYPE_SPRAY)
+	update_replicate (sr_policy);
+    }
+  else if (operation == 2)	/* Delete SR List from an existing SR policy */
+    {
+      /* Check that currently there are more than one SID list */
+      if (vec_len (sr_policy->segments_lists) == 1)
+	return -21;
+
+      /* Check that the SR list does exist and is assigned to the sr policy */
+      vec_foreach (sl_index_iterate, sr_policy->segments_lists)
+	if (*sl_index_iterate == sl_index)
+	break;
+
+      if (*sl_index_iterate != sl_index)
+	return -22;
+
+      /* Remove the lucky SR list that is being kicked out */
+      segment_list = pool_elt_at_index (sm->sid_lists, sl_index);
+      vec_free (segment_list->segments);
+      vec_free (segment_list->rewrite);
+      vec_free (segment_list->rewrite_bsid);
+      pool_put_index (sm->sid_lists, sl_index);
+      vec_del1 (sr_policy->segments_lists,
+		sl_index_iterate - sr_policy->segments_lists);
+
+      /* Create a new LB DPO */
+      if (sr_policy->type == SR_POLICY_TYPE_DEFAULT)
+	update_lb (sr_policy);
+      else if (sr_policy->type == SR_POLICY_TYPE_SPRAY)
+	update_replicate (sr_policy);
+    }
+  else if (operation == 3)	/* Modify the weight of an existing SR List */
+    {
+      /* Find the corresponding SL */
+      vec_foreach (sl_index_iterate, sr_policy->segments_lists)
+	if (*sl_index_iterate == sl_index)
+	break;
+
+      if (*sl_index_iterate != sl_index)
+	return -32;
+
+      /* Change the weight */
+      segment_list = pool_elt_at_index (sm->sid_lists, sl_index);
+      segment_list->weight = weight;
+
+      /* Update LB */
+      if (sr_policy->type == SR_POLICY_TYPE_DEFAULT)
+	update_lb (sr_policy);
+    }
+  else				/* Incorrect op. */
+    return -1;
+
+  return 0;
+}
+
+/**
+ * @brief CLI for 'sr policies' command family
+ */
+static clib_error_t *
+sr_policy_command_fn (vlib_main_t * vm, unformat_input_t * input,
+		      vlib_cli_command_t * cmd)
+{
+  int rv = -1;
+  char is_del = 0, is_add = 0, is_mod = 0;
+  char policy_set = 0;
+  ip6_address_t bsid, next_address;
+  u32 sr_policy_index = (u32) ~ 0, sl_index = (u32) ~ 0;
+  u32 weight = (u32) ~ 0, fib_table = (u32) ~ 0;
+  ip6_address_t *segments = 0, *this_seg;
+  u8 operation = 0;
+  char is_encap = 1;
+  char is_spray = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (!is_add && !is_mod && !is_del && unformat (input, "add"))
+	is_add = 1;
+      else if (!is_add && !is_mod && !is_del && unformat (input, "del"))
+	is_del = 1;
+      else if (!is_add && !is_mod && !is_del && unformat (input, "mod"))
+	is_mod = 1;
+      else if (!policy_set
+	       && unformat (input, "bsid %U", unformat_ip6_address, &bsid))
+	policy_set = 1;
+      else if (!is_add && !policy_set
+	       && unformat (input, "index %d", &sr_policy_index))
+	policy_set = 1;
+      else if (unformat (input, "weight %d", &weight));
+      else
+	if (unformat (input, "next %U", unformat_ip6_address, &next_address))
+	{
+	  vec_add2 (segments, this_seg, 1);
+	  clib_memcpy (this_seg->as_u8, next_address.as_u8,
+		       sizeof (*this_seg));
+	}
+      else if (unformat (input, "add sl"))
+	operation = 1;
+      else if (unformat (input, "del sl index %d", &sl_index))
+	operation = 2;
+      else if (unformat (input, "mod sl index %d", &sl_index))
+	operation = 3;
+      else if (fib_table == (u32) ~ 0
+	       && unformat (input, "fib-table %d", &fib_table));
+      else if (unformat (input, "encap"))
+	is_encap = 1;
+      else if (unformat (input, "insert"))
+	is_encap = 0;
+      else if (unformat (input, "spray"))
+	is_spray = 1;
+      else
+	break;
+    }
+
+  if (!is_add && !is_mod && !is_del)
+    return clib_error_return (0, "Incorrect CLI");
+
+  if (!policy_set)
+    return clib_error_return (0, "No SR policy BSID or index specified");
+
+  if (is_add)
+    {
+      if (vec_len (segments) == 0)
+	return clib_error_return (0, "No Segment List specified");
+      rv = sr_policy_add (&bsid, segments, weight,
+			  (is_spray ? SR_POLICY_TYPE_SPRAY :
+			   SR_POLICY_TYPE_DEFAULT), fib_table, is_encap);
+    }
+  else if (is_del)
+    rv = sr_policy_del ((sr_policy_index != (u32) ~ 0 ? NULL : &bsid),
+			sr_policy_index);
+  else if (is_mod)
+    {
+      if (!operation)
+	return clib_error_return (0, "No SL modification specified");
+      if (operation != 1 && sl_index == (u32) ~ 0)
+	return clib_error_return (0, "No Segment List index specified");
+      if (operation == 1 && vec_len (segments) == 0)
+	return clib_error_return (0, "No Segment List specified");
+      if (operation == 3 && weight == (u32) ~ 0)
+	return clib_error_return (0, "No new weight for the SL specified");
+      rv = sr_policy_mod ((sr_policy_index != (u32) ~ 0 ? NULL : &bsid),
+			  sr_policy_index, fib_table, operation, segments,
+			  sl_index, weight);
+    }
+
+  switch (rv)
+    {
+    case 0:
+      break;
+    case 1:
+      return 0;
+    case -12:
+      return clib_error_return (0,
+				"There is already a FIB entry for the BindingSID address.\n"
+				"The SR policy could not be created.");
+    case -13:
+      return clib_error_return (0, "The specified FIB table does not exist.");
+    case -21:
+      return clib_error_return (0,
+				"The selected SR policy only contains ONE segment list. "
+				"Please remove the SR policy instead");
+    case -22:
+      return clib_error_return (0,
+				"Could not delete the segment list. "
+				"It is not associated with that SR policy.");
+    case -32:
+      return clib_error_return (0,
+				"Could not modify the segment list. "
+				"The given SL is not associated with such SR policy.");
+    default:
+      return clib_error_return (0, "BUG: sr policy returns %d", rv);
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (sr_policy_command, static) = {
+  .path = "sr policy",
+  .short_help = "sr policy [add||del||mod] [bsid 2001::1||index 5] "
+    "next A:: next B:: next C:: (weight 1) (fib-table 2) (encap|insert)",
+  .long_help =
+    "Manipulation of SR policies.\n"
+    "A Segment Routing policy may contain several SID lists. Each SID list has\n"
+    "an associated weight (default 1), which will result in wECMP (uECMP).\n"
+    "Segment Routing policies might be of type encapsulation or srh insertion\n"
+    "Each SR policy will be associated with a unique BindingSID.\n"
+    "A BindingSID is a locally allocated SegmentID. For every packet that arrives\n"
+    "with IPv6_DA:BSID such traffic will be steered into the SR policy.\n"
+    "The add command will create a SR policy with its first segment list (sl)\n"
+    "The mod command allows you to add, remove, or modify the existing segment lists\n"
+    "within an SR policy.\n"
+    "The del command allows you to delete a SR policy along with all its associated\n"
+    "SID lists.\n",
+  .function = sr_policy_command_fn,
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief CLI to display onscreen all the SR policies
+ */
+static clib_error_t *
+show_sr_policies_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			     vlib_cli_command_t * cmd)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 *sl_index;
+  ip6_sr_sl_t *segment_list = 0;
+  ip6_sr_policy_t *sr_policy = 0;
+  ip6_sr_policy_t **vec_policies = 0;
+  ip6_address_t *addr;
+  u8 *s;
+  int i = 0;
+
+  vlib_cli_output (vm, "SR policies:");
+
+  /* *INDENT-OFF* */
+  pool_foreach  (sr_policy, sm->sr_policies,
+                {vec_add1 (vec_policies, sr_policy); } );
+  /* *INDENT-ON* */
+
+  vec_foreach_index (i, vec_policies)
+  {
+    sr_policy = vec_policies[i];
+    vlib_cli_output (vm, "[%u].-\tBSID: %U",
+		     (u32) (sr_policy - sm->sr_policies),
+		     format_ip6_address, &sr_policy->bsid);
+    vlib_cli_output (vm, "\tBehavior: %s",
+		     (sr_policy->is_encap ? "Encapsulation" :
+		      "SRH insertion"));
+    vlib_cli_output (vm, "\tType: %s",
+		     (sr_policy->type ==
+		      SR_POLICY_TYPE_DEFAULT ? "Default" : "Spray"));
+    vlib_cli_output (vm, "\tFIB table: %u",
+		     (sr_policy->fib_table !=
+		      (u32) ~ 0 ? sr_policy->fib_table : 0));
+    vlib_cli_output (vm, "\tSegment Lists:");
+    vec_foreach (sl_index, sr_policy->segments_lists)
+    {
+      s = NULL;
+      s = format (s, "\t[%u].- ", *sl_index);
+      segment_list = pool_elt_at_index (sm->sid_lists, *sl_index);
+      s = format (s, "< ");
+      vec_foreach (addr, segment_list->segments)
+      {
+	s = format (s, "%U, ", format_ip6_address, addr);
+      }
+      s = format (s, "\b\b > ");
+      s = format (s, "weight: %u", segment_list->weight);
+      vlib_cli_output (vm, "  %s", s);
+    }
+    vlib_cli_output (vm, "-----------");
+  }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_sr_policies_command, static) = {
+  .path = "show sr policies",
+  .short_help = "show sr policies",
+  .function = show_sr_policies_command_fn,
+};
+/* *INDENT-ON* */
+
+/*************************** SR rewrite graph node ****************************/
+/**
+ * @brief Trace for the SR Policy Rewrite graph node
+ */
+static u8 *
+format_sr_policy_rewrite_trace (u8 * s, va_list * args)
+{
+  //TODO
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  sr_policy_rewrite_trace_t *t = va_arg (*args, sr_policy_rewrite_trace_t *);
+
+  s = format
+    (s, "SR-policy-rewrite: src %U dst %U",
+     format_ip6_address, &t->src, format_ip6_address, &t->dst);
+
+  return s;
+}
+
+/**
+ * @brief IPv6 encapsulation processing as per RFC2473
+ */
+static_always_inline void
+encaps_processing_v6 (vlib_node_runtime_t * node,
+		      vlib_buffer_t * b0,
+		      ip6_header_t * ip0, ip6_header_t * ip0_encap)
+{
+  u32 new_l0;
+
+  ip0_encap->hop_limit -= 1;
+  new_l0 =
+    ip0->payload_length + sizeof (ip6_header_t) +
+    clib_net_to_host_u16 (ip0_encap->payload_length);
+  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+  ip0->ip_version_traffic_class_and_flow_label =
+    ip0_encap->ip_version_traffic_class_and_flow_label;
+}
+
+/**
+ * @brief Graph node for applying a SR policy into an IPv6 packet. Encapsulation
+ */
+static uword
+sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node,
+			  vlib_frame_t * from_frame)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 n_left_from, next_index, *from, *to_next;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  int encap_pkts = 0, bsid_pkts = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_header_t *ip0_encap, *ip1_encap, *ip2_encap, *ip3_encap;
+	  ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  sl1 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+	  sl2 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+	  sl3 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl1->rewrite) + b1->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl2->rewrite) + b2->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl3->rewrite) + b3->current_data));
+
+	  ip0_encap = vlib_buffer_get_current (b0);
+	  ip1_encap = vlib_buffer_get_current (b1);
+	  ip2_encap = vlib_buffer_get_current (b2);
+	  ip3_encap = vlib_buffer_get_current (b3);
+
+	  clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite),
+		       sl0->rewrite, vec_len (sl0->rewrite));
+	  clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite),
+		       sl1->rewrite, vec_len (sl1->rewrite));
+	  clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite),
+		       sl2->rewrite, vec_len (sl2->rewrite));
+	  clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite),
+		       sl3->rewrite, vec_len (sl3->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite));
+	  vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite));
+	  vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  encaps_processing_v6 (node, b0, ip0, ip0_encap);
+	  encaps_processing_v6 (node, b1, ip1, ip1_encap);
+	  encaps_processing_v6 (node, b2, ip2, ip2_encap);
+	  encaps_processing_v6 (node, b3, ip3, ip3_encap);
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b2, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b3, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+	    }
+
+	  encap_pkts += 4;
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0, *ip0_encap = 0;
+	  ip6_sr_sl_t *sl0;
+	  u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+
+	  ip0_encap = vlib_buffer_get_current (b0);
+
+	  clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite),
+		       sl0->rewrite, vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  encaps_processing_v6 (node, b0, ip0, ip0_encap);
+
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_policy_rewrite_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			   sizeof (tr->src.as_u8));
+	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			   sizeof (tr->dst.as_u8));
+	    }
+
+	  encap_pkts++;
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Update counters */
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL,
+			       encap_pkts);
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_BSID,
+			       bsid_pkts);
+
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_policy_rewrite_encaps_node) = {
+  .function = sr_policy_rewrite_encaps,
+  .name = "sr-pl-rewrite-encaps",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_policy_rewrite_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_POLICY_REWRITE_N_ERROR,
+  .error_strings = sr_policy_rewrite_error_strings,
+  .n_next_nodes = SR_POLICY_REWRITE_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n,
+    foreach_sr_policy_rewrite_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief IPv4 encapsulation processing as per RFC2473
+ */
+static_always_inline void
+encaps_processing_v4 (vlib_node_runtime_t * node,
+		      vlib_buffer_t * b0,
+		      ip6_header_t * ip0, ip4_header_t * ip0_encap)
+{
+  u32 new_l0;
+  ip6_sr_header_t *sr0;
+
+  u32 checksum0;
+
+  /* Inner IPv4: Decrement TTL & update checksum */
+  ip0_encap->ttl -= 1;
+  checksum0 = ip0_encap->checksum + clib_host_to_net_u16 (0x0100);
+  checksum0 += checksum0 >= 0xffff;
+  ip0_encap->checksum = checksum0;
+
+  /* Outer IPv6: Update length, FL, proto */
+  new_l0 = ip0->payload_length + clib_net_to_host_u16 (ip0_encap->length);
+  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+  ip0->ip_version_traffic_class_and_flow_label =
+    clib_host_to_net_u32 (0 | ((6 & 0xF) << 28) |
+			  ((ip0_encap->tos & 0xFF) << 20));
+  sr0 = (void *) (ip0 + 1);
+  sr0->protocol = IP_PROTOCOL_IP_IN_IP;
+}
+
+/**
+ * @brief Graph node for applying a SR policy into an IPv4 packet. Encapsulation
+ */
+static uword
+sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node,
+			     vlib_frame_t * from_frame)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 n_left_from, next_index, *from, *to_next;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  int encap_pkts = 0, bsid_pkts = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip4_header_t *ip0_encap, *ip1_encap, *ip2_encap, *ip3_encap;
+	  ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  sl1 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+	  sl2 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+	  sl3 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl1->rewrite) + b1->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl2->rewrite) + b2->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl3->rewrite) + b3->current_data));
+
+	  ip0_encap = vlib_buffer_get_current (b0);
+	  ip1_encap = vlib_buffer_get_current (b1);
+	  ip2_encap = vlib_buffer_get_current (b2);
+	  ip3_encap = vlib_buffer_get_current (b3);
+
+	  clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite),
+		       sl0->rewrite, vec_len (sl0->rewrite));
+	  clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite),
+		       sl1->rewrite, vec_len (sl1->rewrite));
+	  clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite),
+		       sl2->rewrite, vec_len (sl2->rewrite));
+	  clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite),
+		       sl3->rewrite, vec_len (sl3->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite));
+	  vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite));
+	  vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  encaps_processing_v4 (node, b0, ip0, ip0_encap);
+	  encaps_processing_v4 (node, b1, ip1, ip1_encap);
+	  encaps_processing_v4 (node, b2, ip2, ip2_encap);
+	  encaps_processing_v4 (node, b3, ip3, ip3_encap);
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b2, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b3, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+	    }
+
+	  encap_pkts += 4;
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0;
+	  ip4_header_t *ip0_encap = 0;
+	  ip6_sr_sl_t *sl0;
+	  u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+
+	  ip0_encap = vlib_buffer_get_current (b0);
+
+	  clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite),
+		       sl0->rewrite, vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  encaps_processing_v4 (node, b0, ip0, ip0_encap);
+
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_policy_rewrite_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			   sizeof (tr->src.as_u8));
+	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			   sizeof (tr->dst.as_u8));
+	    }
+
+	  encap_pkts++;
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Update counters */
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL,
+			       encap_pkts);
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_BSID,
+			       bsid_pkts);
+
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_policy_rewrite_encaps_v4_node) = {
+  .function = sr_policy_rewrite_encaps_v4,
+  .name = "sr-pl-rewrite-encaps-v4",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_policy_rewrite_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_POLICY_REWRITE_N_ERROR,
+  .error_strings = sr_policy_rewrite_error_strings,
+  .n_next_nodes = SR_POLICY_REWRITE_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n,
+    foreach_sr_policy_rewrite_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+always_inline u32
+ip_flow_hash (void *data)
+{
+  ip4_header_t *iph = (ip4_header_t *) data;
+
+  if ((iph->ip_version_and_header_length & 0xF0) == 0x40)
+    return ip4_compute_flow_hash (iph, IP_FLOW_HASH_DEFAULT);
+  else
+    return ip6_compute_flow_hash ((ip6_header_t *) iph, IP_FLOW_HASH_DEFAULT);
+}
+
+always_inline u64
+mac_to_u64 (u8 * m)
+{
+  return (*((u64 *) m) & 0xffffffffffff);
+}
+
+always_inline u32
+l2_flow_hash (vlib_buffer_t * b0)
+{
+  ethernet_header_t *eh;
+  u64 a, b, c;
+  uword is_ip, eh_size;
+  u16 eh_type;
+
+  eh = vlib_buffer_get_current (b0);
+  eh_type = clib_net_to_host_u16 (eh->type);
+  eh_size = ethernet_buffer_header_size (b0);
+
+  is_ip = (eh_type == ETHERNET_TYPE_IP4 || eh_type == ETHERNET_TYPE_IP6);
+
+  /* since we have 2 cache lines, use them */
+  if (is_ip)
+    a = ip_flow_hash ((u8 *) vlib_buffer_get_current (b0) + eh_size);
+  else
+    a = eh->type;
+
+  b = mac_to_u64 ((u8 *) eh->dst_address);
+  c = mac_to_u64 ((u8 *) eh->src_address);
+  hash_mix64 (a, b, c);
+
+  return (u32) c;
+}
+
+/**
+ * @brief Graph node for applying a SR policy into a L2 frame
+ */
+static uword
+sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node,
+			     vlib_frame_t * from_frame)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 n_left_from, next_index, *from, *to_next;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  int encap_pkts = 0, bsid_pkts = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  ethernet_header_t *en0, *en1, *en2, *en3;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_sr_header_t *sr0, *sr1, *sr2, *sr3;
+	  ip6_sr_policy_t *sp0, *sp1, *sp2, *sp3;
+	  ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  sp0 = pool_elt_at_index (sm->sr_policies,
+				   sm->sw_iface_sr_policies[vnet_buffer
+							    (b0)->sw_if_index
+							    [VLIB_RX]]);
+
+	  sp1 = pool_elt_at_index (sm->sr_policies,
+				   sm->sw_iface_sr_policies[vnet_buffer
+							    (b1)->sw_if_index
+							    [VLIB_RX]]);
+
+	  sp2 = pool_elt_at_index (sm->sr_policies,
+				   sm->sw_iface_sr_policies[vnet_buffer
+							    (b2)->sw_if_index
+							    [VLIB_RX]]);
+
+	  sp3 = pool_elt_at_index (sm->sr_policies,
+				   sm->sw_iface_sr_policies[vnet_buffer
+							    (b3)->sw_if_index
+							    [VLIB_RX]]);
+
+	  if (vec_len (sp0->segments_lists) == 1)
+	    vnet_buffer (b0)->ip.adj_index[VLIB_TX] = sp0->segments_lists[0];
+	  else
+	    {
+	      vnet_buffer (b0)->ip.flow_hash = l2_flow_hash (b0);
+	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
+		sp0->segments_lists[(vnet_buffer (b0)->ip.flow_hash &
+				     (vec_len (sp0->segments_lists) - 1))];
+	    }
+
+	  if (vec_len (sp1->segments_lists) == 1)
+	    vnet_buffer (b1)->ip.adj_index[VLIB_TX] = sp1->segments_lists[1];
+	  else
+	    {
+	      vnet_buffer (b1)->ip.flow_hash = l2_flow_hash (b1);
+	      vnet_buffer (b1)->ip.adj_index[VLIB_TX] =
+		sp1->segments_lists[(vnet_buffer (b1)->ip.flow_hash &
+				     (vec_len (sp1->segments_lists) - 1))];
+	    }
+
+	  if (vec_len (sp2->segments_lists) == 1)
+	    vnet_buffer (b2)->ip.adj_index[VLIB_TX] = sp2->segments_lists[2];
+	  else
+	    {
+	      vnet_buffer (b2)->ip.flow_hash = l2_flow_hash (b2);
+	      vnet_buffer (b2)->ip.adj_index[VLIB_TX] =
+		sp2->segments_lists[(vnet_buffer (b2)->ip.flow_hash &
+				     (vec_len (sp2->segments_lists) - 1))];
+	    }
+
+	  if (vec_len (sp3->segments_lists) == 1)
+	    vnet_buffer (b3)->ip.adj_index[VLIB_TX] = sp3->segments_lists[3];
+	  else
+	    {
+	      vnet_buffer (b3)->ip.flow_hash = l2_flow_hash (b3);
+	      vnet_buffer (b3)->ip.adj_index[VLIB_TX] =
+		sp3->segments_lists[(vnet_buffer (b3)->ip.flow_hash &
+				     (vec_len (sp3->segments_lists) - 1))];
+	    }
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  sl1 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+	  sl2 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+	  sl3 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl1->rewrite) + b1->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl2->rewrite) + b2->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl3->rewrite) + b3->current_data));
+
+	  en0 = vlib_buffer_get_current (b0);
+	  en1 = vlib_buffer_get_current (b1);
+	  en2 = vlib_buffer_get_current (b2);
+	  en3 = vlib_buffer_get_current (b3);
+
+	  clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite,
+		       vec_len (sl0->rewrite));
+	  clib_memcpy (((u8 *) en1) - vec_len (sl1->rewrite), sl1->rewrite,
+		       vec_len (sl1->rewrite));
+	  clib_memcpy (((u8 *) en2) - vec_len (sl2->rewrite), sl2->rewrite,
+		       vec_len (sl2->rewrite));
+	  clib_memcpy (((u8 *) en3) - vec_len (sl3->rewrite), sl3->rewrite,
+		       vec_len (sl3->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite));
+	  vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite));
+	  vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  ip0->payload_length =
+	    clib_host_to_net_u16 (b0->current_length - sizeof (ip6_header_t));
+	  ip1->payload_length =
+	    clib_host_to_net_u16 (b1->current_length - sizeof (ip6_header_t));
+	  ip2->payload_length =
+	    clib_host_to_net_u16 (b2->current_length - sizeof (ip6_header_t));
+	  ip3->payload_length =
+	    clib_host_to_net_u16 (b3->current_length - sizeof (ip6_header_t));
+
+	  sr0 = (void *) (ip0 + 1);
+	  sr1 = (void *) (ip1 + 1);
+	  sr2 = (void *) (ip2 + 1);
+	  sr3 = (void *) (ip3 + 1);
+
+	  sr0->protocol = sr1->protocol = sr2->protocol = sr3->protocol =
+	    IP_PROTOCOL_IP6_NONXT;
+
+	  /* Which Traffic class and flow label do I set ? */
+	  //ip0->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32(0|((6&0xF)<<28)|((ip0_encap->tos&0xFF)<<20));
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b2, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b3, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+	    }
+
+	  encap_pkts += 4;
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0;
+	  ip6_sr_header_t *sr0;
+	  ethernet_header_t *en0;
+	  ip6_sr_policy_t *sp0;
+	  ip6_sr_sl_t *sl0;
+	  u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  /* Find the SR policy */
+	  sp0 = pool_elt_at_index (sm->sr_policies,
+				   sm->sw_iface_sr_policies[vnet_buffer
+							    (b0)->sw_if_index
+							    [VLIB_RX]]);
+
+	  /* In case there is more than one SL, LB among them */
+	  if (vec_len (sp0->segments_lists) == 1)
+	    vnet_buffer (b0)->ip.adj_index[VLIB_TX] = sp0->segments_lists[0];
+	  else
+	    {
+	      vnet_buffer (b0)->ip.flow_hash = l2_flow_hash (b0);
+	      vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
+		sp0->segments_lists[(vnet_buffer (b0)->ip.flow_hash &
+				     (vec_len (sp0->segments_lists) - 1))];
+	    }
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+
+	  en0 = vlib_buffer_get_current (b0);
+
+	  clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite,
+		       vec_len (sl0->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  ip0->payload_length =
+	    clib_host_to_net_u16 (b0->current_length - sizeof (ip6_header_t));
+
+	  sr0 = (void *) (ip0 + 1);
+	  sr0->protocol = IP_PROTOCOL_IP6_NONXT;
+
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_policy_rewrite_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			   sizeof (tr->src.as_u8));
+	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			   sizeof (tr->dst.as_u8));
+	    }
+
+	  encap_pkts++;
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Update counters */
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL,
+			       encap_pkts);
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_BSID,
+			       bsid_pkts);
+
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_policy_rewrite_encaps_l2_node) = {
+  .function = sr_policy_rewrite_encaps_l2,
+  .name = "sr-pl-rewrite-encaps-l2",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_policy_rewrite_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_POLICY_REWRITE_N_ERROR,
+  .error_strings = sr_policy_rewrite_error_strings,
+  .n_next_nodes = SR_POLICY_REWRITE_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n,
+    foreach_sr_policy_rewrite_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Graph node for applying a SR policy into a packet. SRH insertion.
+ */
+static uword
+sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node,
+			  vlib_frame_t * from_frame)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 n_left_from, next_index, *from, *to_next;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  int insert_pkts = 0, bsid_pkts = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_sr_header_t *sr0, *sr1, *sr2, *sr3;
+	  ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3;
+	  u16 new_l0, new_l1, new_l2, new_l3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  sl1 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+	  sl2 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+	  sl3 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl1->rewrite) + b1->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl2->rewrite) + b2->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl3->rewrite) + b3->current_data));
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr0 =
+	      (ip6_sr_header_t *) (((void *) (ip0 + 1)) +
+				   ip6_ext_header_len (ip0 + 1));
+	  else
+	    sr0 = (ip6_sr_header_t *) (ip0 + 1);
+
+	  if (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr1 =
+	      (ip6_sr_header_t *) (((void *) (ip1 + 1)) +
+				   ip6_ext_header_len (ip1 + 1));
+	  else
+	    sr1 = (ip6_sr_header_t *) (ip1 + 1);
+
+	  if (ip2->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr2 =
+	      (ip6_sr_header_t *) (((void *) (ip2 + 1)) +
+				   ip6_ext_header_len (ip2 + 1));
+	  else
+	    sr2 = (ip6_sr_header_t *) (ip2 + 1);
+
+	  if (ip3->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr3 =
+	      (ip6_sr_header_t *) (((void *) (ip3 + 1)) +
+				   ip6_ext_header_len (ip3 + 1));
+	  else
+	    sr3 = (ip6_sr_header_t *) (ip3 + 1);
+
+	  clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0,
+		       (void *) sr0 - (void *) ip0);
+	  clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite), (u8 *) ip1,
+		       (void *) sr1 - (void *) ip1);
+	  clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite), (u8 *) ip2,
+		       (void *) sr2 - (void *) ip2);
+	  clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite), (u8 *) ip3,
+		       (void *) sr3 - (void *) ip3);
+
+	  clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite,
+		       vec_len (sl0->rewrite));
+	  clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite)), sl1->rewrite,
+		       vec_len (sl1->rewrite));
+	  clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite)), sl2->rewrite,
+		       vec_len (sl2->rewrite));
+	  clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite)), sl3->rewrite,
+		       vec_len (sl3->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite));
+	  vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite));
+	  vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite));
+
+	  ip0 = ((void *) ip0) - vec_len (sl0->rewrite);
+	  ip1 = ((void *) ip1) - vec_len (sl1->rewrite);
+	  ip2 = ((void *) ip2) - vec_len (sl2->rewrite);
+	  ip3 = ((void *) ip3) - vec_len (sl3->rewrite);
+
+	  ip0->hop_limit -= 1;
+	  ip1->hop_limit -= 1;
+	  ip2->hop_limit -= 1;
+	  ip3->hop_limit -= 1;
+
+	  new_l0 =
+	    clib_net_to_host_u16 (ip0->payload_length) +
+	    vec_len (sl0->rewrite);
+	  new_l1 =
+	    clib_net_to_host_u16 (ip1->payload_length) +
+	    vec_len (sl1->rewrite);
+	  new_l2 =
+	    clib_net_to_host_u16 (ip2->payload_length) +
+	    vec_len (sl2->rewrite);
+	  new_l3 =
+	    clib_net_to_host_u16 (ip3->payload_length) +
+	    vec_len (sl3->rewrite);
+
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+	  ip1->payload_length = clib_host_to_net_u16 (new_l1);
+	  ip2->payload_length = clib_host_to_net_u16 (new_l2);
+	  ip3->payload_length = clib_host_to_net_u16 (new_l3);
+
+	  sr0 = ((void *) sr0) - vec_len (sl0->rewrite);
+	  sr1 = ((void *) sr1) - vec_len (sl1->rewrite);
+	  sr2 = ((void *) sr2) - vec_len (sl2->rewrite);
+	  sr3 = ((void *) sr3) - vec_len (sl3->rewrite);
+
+	  sr0->segments->as_u64[0] = ip0->dst_address.as_u64[0];
+	  sr0->segments->as_u64[1] = ip0->dst_address.as_u64[1];
+	  sr1->segments->as_u64[0] = ip1->dst_address.as_u64[0];
+	  sr1->segments->as_u64[1] = ip1->dst_address.as_u64[1];
+	  sr2->segments->as_u64[0] = ip2->dst_address.as_u64[0];
+	  sr2->segments->as_u64[1] = ip2->dst_address.as_u64[1];
+	  sr3->segments->as_u64[0] = ip3->dst_address.as_u64[0];
+	  sr3->segments->as_u64[1] = ip3->dst_address.as_u64[1];
+
+	  ip0->dst_address.as_u64[0] =
+	    (sr0->segments + sr0->segments_left)->as_u64[0];
+	  ip0->dst_address.as_u64[1] =
+	    (sr0->segments + sr0->segments_left)->as_u64[1];
+	  ip1->dst_address.as_u64[0] =
+	    (sr1->segments + sr1->segments_left)->as_u64[0];
+	  ip1->dst_address.as_u64[1] =
+	    (sr1->segments + sr1->segments_left)->as_u64[1];
+	  ip2->dst_address.as_u64[0] =
+	    (sr2->segments + sr2->segments_left)->as_u64[0];
+	  ip2->dst_address.as_u64[1] =
+	    (sr2->segments + sr2->segments_left)->as_u64[1];
+	  ip3->dst_address.as_u64[0] =
+	    (sr3->segments + sr3->segments_left)->as_u64[0];
+	  ip3->dst_address.as_u64[1] =
+	    (sr3->segments + sr3->segments_left)->as_u64[1];
+
+	  ip6_ext_header_t *ip_ext;
+	  if (ip0 + 1 == (void *) sr0)
+	    {
+	      sr0->protocol = ip0->protocol;
+	      ip0->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip0 + 1);
+	      sr0->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (ip1 + 1 == (void *) sr1)
+	    {
+	      sr1->protocol = ip1->protocol;
+	      ip1->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip2 + 1);
+	      sr2->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (ip2 + 1 == (void *) sr2)
+	    {
+	      sr2->protocol = ip2->protocol;
+	      ip2->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip2 + 1);
+	      sr2->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (ip3 + 1 == (void *) sr3)
+	    {
+	      sr3->protocol = ip3->protocol;
+	      ip3->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip3 + 1);
+	      sr3->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  insert_pkts += 4;
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b2, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b3, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+	    }
+
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0;
+	  ip6_sr_header_t *sr0 = 0;
+	  ip6_sr_sl_t *sl0;
+	  u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  u16 new_l0 = 0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr0 =
+	      (ip6_sr_header_t *) (((void *) (ip0 + 1)) +
+				   ip6_ext_header_len (ip0 + 1));
+	  else
+	    sr0 = (ip6_sr_header_t *) (ip0 + 1);
+
+	  clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0,
+		       (void *) sr0 - (void *) ip0);
+	  clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite,
+		       vec_len (sl0->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+
+	  ip0 = ((void *) ip0) - vec_len (sl0->rewrite);
+	  ip0->hop_limit -= 1;
+	  new_l0 =
+	    clib_net_to_host_u16 (ip0->payload_length) +
+	    vec_len (sl0->rewrite);
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+
+	  sr0 = ((void *) sr0) - vec_len (sl0->rewrite);
+	  sr0->segments->as_u64[0] = ip0->dst_address.as_u64[0];
+	  sr0->segments->as_u64[1] = ip0->dst_address.as_u64[1];
+
+	  ip0->dst_address.as_u64[0] =
+	    (sr0->segments + sr0->segments_left)->as_u64[0];
+	  ip0->dst_address.as_u64[1] =
+	    (sr0->segments + sr0->segments_left)->as_u64[1];
+
+	  if (ip0 + 1 == (void *) sr0)
+	    {
+	      sr0->protocol = ip0->protocol;
+	      ip0->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip6_ext_header_t *ip_ext = (void *) (ip0 + 1);
+	      sr0->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_policy_rewrite_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			   sizeof (tr->src.as_u8));
+	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			   sizeof (tr->dst.as_u8));
+	    }
+
+	  insert_pkts++;
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Update counters */
+  vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL,
+			       insert_pkts);
+  vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_BSID,
+			       bsid_pkts);
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_policy_rewrite_insert_node) = {
+  .function = sr_policy_rewrite_insert,
+  .name = "sr-pl-rewrite-insert",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_policy_rewrite_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_POLICY_REWRITE_N_ERROR,
+  .error_strings = sr_policy_rewrite_error_strings,
+  .n_next_nodes = SR_POLICY_REWRITE_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n,
+    foreach_sr_policy_rewrite_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Graph node for applying a SR policy into a packet. BSID - SRH insertion.
+ */
+static uword
+sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node,
+			    vlib_frame_t * from_frame)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 n_left_from, next_index, *from, *to_next;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  int insert_pkts = 0, bsid_pkts = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_sr_header_t *sr0, *sr1, *sr2, *sr3;
+	  ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3;
+	  u16 new_l0, new_l1, new_l2, new_l3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  sl1 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+	  sl2 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+	  sl3 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite_bsid) + b0->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl1->rewrite_bsid) + b1->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl2->rewrite_bsid) + b2->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl3->rewrite_bsid) + b3->current_data));
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr0 =
+	      (ip6_sr_header_t *) (((void *) (ip0 + 1)) +
+				   ip6_ext_header_len (ip0 + 1));
+	  else
+	    sr0 = (ip6_sr_header_t *) (ip0 + 1);
+
+	  if (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr1 =
+	      (ip6_sr_header_t *) (((void *) (ip1 + 1)) +
+				   ip6_ext_header_len (ip1 + 1));
+	  else
+	    sr1 = (ip6_sr_header_t *) (ip1 + 1);
+
+	  if (ip2->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr2 =
+	      (ip6_sr_header_t *) (((void *) (ip2 + 1)) +
+				   ip6_ext_header_len (ip2 + 1));
+	  else
+	    sr2 = (ip6_sr_header_t *) (ip2 + 1);
+
+	  if (ip3->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr3 =
+	      (ip6_sr_header_t *) (((void *) (ip3 + 1)) +
+				   ip6_ext_header_len (ip3 + 1));
+	  else
+	    sr3 = (ip6_sr_header_t *) (ip3 + 1);
+
+	  clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0,
+		       (void *) sr0 - (void *) ip0);
+	  clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite_bsid), (u8 *) ip1,
+		       (void *) sr1 - (void *) ip1);
+	  clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite_bsid), (u8 *) ip2,
+		       (void *) sr2 - (void *) ip2);
+	  clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite_bsid), (u8 *) ip3,
+		       (void *) sr3 - (void *) ip3);
+
+	  clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)),
+		       sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid));
+	  clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite_bsid)),
+		       sl1->rewrite_bsid, vec_len (sl1->rewrite_bsid));
+	  clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite_bsid)),
+		       sl2->rewrite_bsid, vec_len (sl2->rewrite_bsid));
+	  clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite_bsid)),
+		       sl3->rewrite_bsid, vec_len (sl3->rewrite_bsid));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid));
+	  vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite_bsid));
+	  vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite_bsid));
+	  vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite_bsid));
+
+	  ip0 = ((void *) ip0) - vec_len (sl0->rewrite_bsid);
+	  ip1 = ((void *) ip1) - vec_len (sl1->rewrite_bsid);
+	  ip2 = ((void *) ip2) - vec_len (sl2->rewrite_bsid);
+	  ip3 = ((void *) ip3) - vec_len (sl3->rewrite_bsid);
+
+	  ip0->hop_limit -= 1;
+	  ip1->hop_limit -= 1;
+	  ip2->hop_limit -= 1;
+	  ip3->hop_limit -= 1;
+
+	  new_l0 =
+	    clib_net_to_host_u16 (ip0->payload_length) +
+	    vec_len (sl0->rewrite_bsid);
+	  new_l1 =
+	    clib_net_to_host_u16 (ip1->payload_length) +
+	    vec_len (sl1->rewrite_bsid);
+	  new_l2 =
+	    clib_net_to_host_u16 (ip2->payload_length) +
+	    vec_len (sl2->rewrite_bsid);
+	  new_l3 =
+	    clib_net_to_host_u16 (ip3->payload_length) +
+	    vec_len (sl3->rewrite_bsid);
+
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+	  ip1->payload_length = clib_host_to_net_u16 (new_l1);
+	  ip2->payload_length = clib_host_to_net_u16 (new_l2);
+	  ip3->payload_length = clib_host_to_net_u16 (new_l3);
+
+	  sr0 = ((void *) sr0) - vec_len (sl0->rewrite_bsid);
+	  sr1 = ((void *) sr1) - vec_len (sl1->rewrite_bsid);
+	  sr2 = ((void *) sr2) - vec_len (sl2->rewrite_bsid);
+	  sr3 = ((void *) sr3) - vec_len (sl3->rewrite_bsid);
+
+	  ip0->dst_address.as_u64[0] =
+	    (sr0->segments + sr0->segments_left)->as_u64[0];
+	  ip0->dst_address.as_u64[1] =
+	    (sr0->segments + sr0->segments_left)->as_u64[1];
+	  ip1->dst_address.as_u64[0] =
+	    (sr1->segments + sr1->segments_left)->as_u64[0];
+	  ip1->dst_address.as_u64[1] =
+	    (sr1->segments + sr1->segments_left)->as_u64[1];
+	  ip2->dst_address.as_u64[0] =
+	    (sr2->segments + sr2->segments_left)->as_u64[0];
+	  ip2->dst_address.as_u64[1] =
+	    (sr2->segments + sr2->segments_left)->as_u64[1];
+	  ip3->dst_address.as_u64[0] =
+	    (sr3->segments + sr3->segments_left)->as_u64[0];
+	  ip3->dst_address.as_u64[1] =
+	    (sr3->segments + sr3->segments_left)->as_u64[1];
+
+	  ip6_ext_header_t *ip_ext;
+	  if (ip0 + 1 == (void *) sr0)
+	    {
+	      sr0->protocol = ip0->protocol;
+	      ip0->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip0 + 1);
+	      sr0->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (ip1 + 1 == (void *) sr1)
+	    {
+	      sr1->protocol = ip1->protocol;
+	      ip1->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip2 + 1);
+	      sr2->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (ip2 + 1 == (void *) sr2)
+	    {
+	      sr2->protocol = ip2->protocol;
+	      ip2->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip2 + 1);
+	      sr2->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (ip3 + 1 == (void *) sr3)
+	    {
+	      sr3->protocol = ip3->protocol;
+	      ip3->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip_ext = (void *) (ip3 + 1);
+	      sr3->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  insert_pkts += 4;
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b2, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b3, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+	    }
+
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0;
+	  ip6_sr_header_t *sr0 = 0;
+	  ip6_sr_sl_t *sl0;
+	  u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  u16 new_l0 = 0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite_bsid) + b0->current_data));
+
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  if (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)
+	    sr0 =
+	      (ip6_sr_header_t *) (((void *) (ip0 + 1)) +
+				   ip6_ext_header_len (ip0 + 1));
+	  else
+	    sr0 = (ip6_sr_header_t *) (ip0 + 1);
+
+	  clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0,
+		       (void *) sr0 - (void *) ip0);
+	  clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)),
+		       sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid));
+
+	  ip0 = ((void *) ip0) - vec_len (sl0->rewrite_bsid);
+	  ip0->hop_limit -= 1;
+	  new_l0 =
+	    clib_net_to_host_u16 (ip0->payload_length) +
+	    vec_len (sl0->rewrite_bsid);
+	  ip0->payload_length = clib_host_to_net_u16 (new_l0);
+
+	  sr0 = ((void *) sr0) - vec_len (sl0->rewrite_bsid);
+
+	  ip0->dst_address.as_u64[0] =
+	    (sr0->segments + sr0->segments_left)->as_u64[0];
+	  ip0->dst_address.as_u64[1] =
+	    (sr0->segments + sr0->segments_left)->as_u64[1];
+
+	  if (ip0 + 1 == (void *) sr0)
+	    {
+	      sr0->protocol = ip0->protocol;
+	      ip0->protocol = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+	  else
+	    {
+	      ip6_ext_header_t *ip_ext = (void *) (ip0 + 1);
+	      sr0->protocol = ip_ext->next_hdr;
+	      ip_ext->next_hdr = IP_PROTOCOL_IPV6_ROUTE;
+	    }
+
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_policy_rewrite_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			   sizeof (tr->src.as_u8));
+	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			   sizeof (tr->dst.as_u8));
+	    }
+
+	  insert_pkts++;
+
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Update counters */
+  vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL,
+			       insert_pkts);
+  vlib_node_increment_counter (vm, sr_policy_rewrite_insert_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_BSID,
+			       bsid_pkts);
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_policy_rewrite_b_insert_node) = {
+  .function = sr_policy_rewrite_b_insert,
+  .name = "sr-pl-rewrite-b-insert",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_policy_rewrite_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_POLICY_REWRITE_N_ERROR,
+  .error_strings = sr_policy_rewrite_error_strings,
+  .n_next_nodes = SR_POLICY_REWRITE_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n,
+    foreach_sr_policy_rewrite_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Function BSID encapsulation
+ */
+static_always_inline void
+end_bsid_encaps_srh_processing (vlib_node_runtime_t * node,
+				vlib_buffer_t * b0,
+				ip6_header_t * ip0,
+				ip6_sr_header_t * sr0, u32 * next0)
+{
+  ip6_address_t *new_dst0;
+
+  if (PREDICT_FALSE (!sr0))
+    goto error_bsid_encaps;
+
+  if (PREDICT_TRUE (sr0->type == ROUTING_HEADER_TYPE_SR))
+    {
+      if (PREDICT_TRUE (sr0->segments_left != 0))
+	{
+	  sr0->segments_left -= 1;
+	  new_dst0 = (ip6_address_t *) (sr0->segments);
+	  new_dst0 += sr0->segments_left;
+	  ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
+	  ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
+	  return;
+	}
+    }
+
+error_bsid_encaps:
+  *next0 = SR_POLICY_REWRITE_NEXT_ERROR;
+  b0->error = node->errors[SR_POLICY_REWRITE_ERROR_BSID_ZERO];
+}
+
+/**
+ * @brief Graph node for applying a SR policy BSID - Encapsulation
+ */
+static uword
+sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node,
+			    vlib_frame_t * from_frame)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  u32 n_left_from, next_index, *from, *to_next;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+
+  int encap_pkts = 0, bsid_pkts = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      /* Quad - Loop */
+      while (n_left_from >= 8 && n_left_to_next >= 4)
+	{
+	  u32 bi0, bi1, bi2, bi3;
+	  vlib_buffer_t *b0, *b1, *b2, *b3;
+	  u32 next0, next1, next2, next3;
+	  next0 = next1 = next2 = next3 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+	  ip6_header_t *ip0, *ip1, *ip2, *ip3;
+	  ip6_header_t *ip0_encap, *ip1_encap, *ip2_encap, *ip3_encap;
+	  ip6_sr_header_t *sr0, *sr1, *sr2, *sr3;
+	  ip6_ext_header_t *prev0, *prev1, *prev2, *prev3;
+	  ip6_sr_sl_t *sl0, *sl1, *sl2, *sl3;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p4, *p5, *p6, *p7;
+
+	    p4 = vlib_get_buffer (vm, from[4]);
+	    p5 = vlib_get_buffer (vm, from[5]);
+	    p6 = vlib_get_buffer (vm, from[6]);
+	    p7 = vlib_get_buffer (vm, from[7]);
+
+	    /* Prefetch the buffer header and packet for the N+2 loop iteration */
+	    vlib_prefetch_buffer_header (p4, LOAD);
+	    vlib_prefetch_buffer_header (p5, LOAD);
+	    vlib_prefetch_buffer_header (p6, LOAD);
+	    vlib_prefetch_buffer_header (p7, LOAD);
+
+	    CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  to_next[2] = bi2 = from[2];
+	  to_next[3] = bi3 = from[3];
+	  from += 4;
+	  to_next += 4;
+	  n_left_from -= 4;
+	  n_left_to_next -= 4;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  b2 = vlib_get_buffer (vm, bi2);
+	  b3 = vlib_get_buffer (vm, bi3);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+	  sl1 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b1)->ip.adj_index[VLIB_TX]);
+	  sl2 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b2)->ip.adj_index[VLIB_TX]);
+	  sl3 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b3)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl1->rewrite) + b1->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl2->rewrite) + b2->current_data));
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl3->rewrite) + b3->current_data));
+
+	  ip0_encap = vlib_buffer_get_current (b0);
+	  ip1_encap = vlib_buffer_get_current (b1);
+	  ip2_encap = vlib_buffer_get_current (b2);
+	  ip3_encap = vlib_buffer_get_current (b3);
+
+	  ip6_ext_header_find_t (ip0_encap, prev0, sr0,
+				 IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip1_encap, prev1, sr1,
+				 IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip2_encap, prev2, sr2,
+				 IP_PROTOCOL_IPV6_ROUTE);
+	  ip6_ext_header_find_t (ip3_encap, prev3, sr3,
+				 IP_PROTOCOL_IPV6_ROUTE);
+
+	  end_bsid_encaps_srh_processing (node, b0, ip0_encap, sr0, &next0);
+	  end_bsid_encaps_srh_processing (node, b1, ip1_encap, sr1, &next1);
+	  end_bsid_encaps_srh_processing (node, b2, ip2_encap, sr2, &next2);
+	  end_bsid_encaps_srh_processing (node, b3, ip3_encap, sr3, &next3);
+
+	  clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite),
+		       sl0->rewrite, vec_len (sl0->rewrite));
+	  clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite),
+		       sl1->rewrite, vec_len (sl1->rewrite));
+	  clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite),
+		       sl2->rewrite, vec_len (sl2->rewrite));
+	  clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite),
+		       sl3->rewrite, vec_len (sl3->rewrite));
+
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite));
+	  vlib_buffer_advance (b2, -(word) vec_len (sl2->rewrite));
+	  vlib_buffer_advance (b3, -(word) vec_len (sl3->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+	  ip1 = vlib_buffer_get_current (b1);
+	  ip2 = vlib_buffer_get_current (b2);
+	  ip3 = vlib_buffer_get_current (b3);
+
+	  encaps_processing_v6 (node, b0, ip0, ip0_encap);
+	  encaps_processing_v6 (node, b1, ip1, ip1_encap);
+	  encaps_processing_v6 (node, b2, ip2, ip2_encap);
+	  encaps_processing_v6 (node, b3, ip3, ip3_encap);
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b2, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+
+	      if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED))
+		{
+		  sr_policy_rewrite_trace_t *tr =
+		    vlib_add_trace (vm, node, b3, sizeof (*tr));
+		  clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8,
+			       sizeof (tr->src.as_u8));
+		  clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8,
+			       sizeof (tr->dst.as_u8));
+		}
+	    }
+
+	  encap_pkts += 4;
+	  vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, bi1, bi2, bi3,
+					   next0, next1, next2, next3);
+	}
+
+      /* Single loop for potentially the last three packets */
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  ip6_header_t *ip0 = 0, *ip0_encap = 0;
+	  ip6_ext_header_t *prev0;
+	  ip6_sr_header_t *sr0;
+	  ip6_sr_sl_t *sl0;
+	  u32 next0 = SR_POLICY_REWRITE_NEXT_IP6_LOOKUP;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  sl0 =
+	    pool_elt_at_index (sm->sid_lists,
+			       vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
+
+	  ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >=
+		  (vec_len (sl0->rewrite) + b0->current_data));
+
+	  ip0_encap = vlib_buffer_get_current (b0);
+	  ip6_ext_header_find_t (ip0_encap, prev0, sr0,
+				 IP_PROTOCOL_IPV6_ROUTE);
+	  end_bsid_encaps_srh_processing (node, b0, ip0_encap, sr0, &next0);
+
+	  clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite),
+		       sl0->rewrite, vec_len (sl0->rewrite));
+	  vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite));
+
+	  ip0 = vlib_buffer_get_current (b0);
+
+	  encaps_processing_v6 (node, b0, ip0, ip0_encap);
+
+	  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
+	      PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      sr_policy_rewrite_trace_t *tr =
+		vlib_add_trace (vm, node, b0, sizeof (*tr));
+	      clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8,
+			   sizeof (tr->src.as_u8));
+	      clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8,
+			   sizeof (tr->dst.as_u8));
+	    }
+
+	  encap_pkts++;
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Update counters */
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_TOTAL,
+			       encap_pkts);
+  vlib_node_increment_counter (vm, sr_policy_rewrite_encaps_node.index,
+			       SR_POLICY_REWRITE_ERROR_COUNTER_BSID,
+			       bsid_pkts);
+
+  return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (sr_policy_rewrite_b_encaps_node) = {
+  .function = sr_policy_rewrite_b_encaps,
+  .name = "sr-pl-rewrite-b-encaps",
+  .vector_size = sizeof (u32),
+  .format_trace = format_sr_policy_rewrite_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SR_POLICY_REWRITE_N_ERROR,
+  .error_strings = sr_policy_rewrite_error_strings,
+  .n_next_nodes = SR_POLICY_REWRITE_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [SR_POLICY_REWRITE_NEXT_##s] = n,
+    foreach_sr_policy_rewrite_next
+#undef _
+  },
+};
+/* *INDENT-ON* */
+
+/*************************** SR Segment Lists DPOs ****************************/
+static u8 *
+format_sr_segment_list_dpo (u8 * s, va_list * args)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_address_t *addr;
+  ip6_sr_sl_t *sl;
+
+  index_t index = va_arg (*args, index_t);
+  CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
+  s = format (s, "SR: Segment List index:[%d]", index);
+  s = format (s, "\n\tSegments:");
+
+  sl = pool_elt_at_index (sm->sid_lists, index);
+
+  s = format (s, "< ");
+  vec_foreach (addr, sl->segments)
+  {
+    s = format (s, "%U, ", format_ip6_address, addr);
+  }
+  s = format (s, "\b\b > - ");
+  s = format (s, "Weight: %u", sl->weight);
+
+  return s;
+}
+
+const static dpo_vft_t sr_policy_rewrite_vft = {
+  .dv_lock = sr_dpo_lock,
+  .dv_unlock = sr_dpo_unlock,
+  .dv_format = format_sr_segment_list_dpo,
+};
+
+const static char *const sr_pr_encaps_ip6_nodes[] = {
+  "sr-pl-rewrite-encaps",
+  NULL,
+};
+
+const static char *const sr_pr_encaps_ip4_nodes[] = {
+  "sr-pl-rewrite-encaps-v4",
+  NULL,
+};
+
+const static char *const *const sr_pr_encaps_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = sr_pr_encaps_ip6_nodes,
+  [DPO_PROTO_IP4] = sr_pr_encaps_ip4_nodes,
+};
+
+const static char *const sr_pr_insert_ip6_nodes[] = {
+  "sr-pl-rewrite-insert",
+  NULL,
+};
+
+const static char *const *const sr_pr_insert_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = sr_pr_insert_ip6_nodes,
+};
+
+const static char *const sr_pr_bsid_insert_ip6_nodes[] = {
+  "sr-pl-rewrite-b-insert",
+  NULL,
+};
+
+const static char *const *const sr_pr_bsid_insert_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = sr_pr_bsid_insert_ip6_nodes,
+};
+
+const static char *const sr_pr_bsid_encaps_ip6_nodes[] = {
+  "sr-pl-rewrite-b-encaps",
+  NULL,
+};
+
+const static char *const *const sr_pr_bsid_encaps_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = sr_pr_bsid_encaps_ip6_nodes,
+};
+
+/********************* SR Policy Rewrite initialization ***********************/
+/**
+ * @brief SR Policy Rewrite initialization
+ */
+clib_error_t *
+sr_policy_rewrite_init (vlib_main_t * vm)
+{
+  ip6_sr_main_t *sm = &sr_main;
+
+  /* Init memory for sr policy keys (bsid <-> ip6_address_t) */
+  sm->sr_policy_index_by_key = hash_create_mem (0, sizeof (ip6_address_t),
+						sizeof (uword));
+
+  /* Init SR VPO DPOs type */
+  sr_pr_encaps_dpo_type =
+    dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_encaps_nodes);
+
+  sr_pr_insert_dpo_type =
+    dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_insert_nodes);
+
+  sr_pr_bsid_encaps_dpo_type =
+    dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_bsid_encaps_nodes);
+
+  sr_pr_bsid_insert_dpo_type =
+    dpo_register_new_type (&sr_policy_rewrite_vft, sr_pr_bsid_insert_nodes);
+
+  /* Register the L2 encaps node used in HW redirect */
+  sm->l2_sr_policy_rewrite_index = sr_policy_rewrite_encaps_node.index;
+
+  sm->fib_table_ip6 = (u32) ~ 0;
+  sm->fib_table_ip4 = (u32) ~ 0;
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (sr_policy_rewrite_init);
+
+
+/*
+* fd.io coding-style-patch-verification: ON
+*
+* Local Variables:
+* eval: (c-set-style "gnu")
+* End:
+*/
diff --git a/src/vnet/sr/sr_steering.c b/src/vnet/sr/sr_steering.c
new file mode 100755
index 00000000..86d6f27c
--- /dev/null
+++ b/src/vnet/sr/sr_steering.c
@@ -0,0 +1,568 @@
+/*
+ * sr_steering.c: ipv6 segment routing steering into SR policy
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ * @brief Packet steering into SR Policies
+ *
+ * This file is in charge of handling the FIB appropiatly to steer packets
+ * through SR Policies as defined in 'sr_policy_rewrite.c'. Notice that here
+ * we are only doing steering. SR policy application is done in
+ * sr_policy_rewrite.c
+ *
+ * Supports:
+ *  - Steering of IPv6 traffic Destination Address based
+ *  - Steering of IPv4 traffic Destination Address based
+ *  - Steering of L2 frames, interface based (sw interface)
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+/**
+ * @brief Steer traffic L2 and L3 traffic through a given SR policy
+ *
+ * @param is_del
+ * @param bsid is the bindingSID of the SR Policy (alt to sr_policy_index)
+ * @param sr_policy is the index of the SR Policy (alt to bsid)
+ * @param table_id is the VRF where to install the FIB entry for the BSID
+ * @param prefix is the IPv4/v6 address for L3 traffic type
+ * @param mask_width is the mask for L3 traffic type
+ * @param sw_if_index is the incoming interface for L2 traffic
+ * @param traffic_type describes the type of traffic
+ *
+ * @return 0 if correct, else error
+ */
+int
+sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
+		    u32 table_id, ip46_address_t * prefix, u32 mask_width,
+		    u32 sw_if_index, u8 traffic_type)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  sr_steering_key_t key, *key_copy;
+  ip6_sr_steering_policy_t *steer_pl;
+  fib_prefix_t pfx = { 0 };
+
+  ip6_sr_policy_t *sr_policy = 0;
+  uword *p = 0;
+
+  hash_pair_t *hp;
+
+  /* Compute the steer policy key */
+  if (prefix)
+    {
+      key.l3.prefix.as_u64[0] = prefix->as_u64[0];
+      key.l3.prefix.as_u64[1] = prefix->as_u64[1];
+      key.l3.mask_width = mask_width;
+      key.l3.fib_table = (table_id != (u32) ~ 0 ? table_id : 0);
+    }
+  else
+    {
+      key.l2.sw_if_index = sw_if_index;
+
+      /* Sanitise the SW_IF_INDEX */
+      if (pool_is_free_index (sm->vnet_main->interface_main.sw_interfaces,
+			      sw_if_index))
+	return -3;
+
+      vnet_sw_interface_t *sw =
+	vnet_get_sw_interface (sm->vnet_main, sw_if_index);
+      if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
+	return -3;
+    }
+
+  key.traffic_type = traffic_type;
+
+  /* Search for the item */
+  p = hash_get_mem (sm->steer_policies_index_by_key, &key);
+
+  if (p)
+    {
+      /* Retrieve Steer Policy function */
+      steer_pl = pool_elt_at_index (sm->steer_policies, p[0]);
+
+      if (is_del)
+	{
+	  if (steer_pl->classify.traffic_type == SR_STEER_IPV6)
+	    {
+	      /* Remove FIB entry */
+	      pfx.fp_proto = FIB_PROTOCOL_IP6;
+	      pfx.fp_len = steer_pl->classify.l3.mask_width;
+	      pfx.fp_addr.ip6 = steer_pl->classify.l3.prefix.ip6;
+
+	      fib_table_entry_delete (fib_table_id_find_fib_index
+				      (FIB_PROTOCOL_IP6,
+				       steer_pl->classify.l3.fib_table), &pfx,
+				      FIB_SOURCE_SR);
+	    }
+	  else if (steer_pl->classify.traffic_type == SR_STEER_IPV4)
+	    {
+	      /* Remove FIB entry */
+	      pfx.fp_proto = FIB_PROTOCOL_IP4;
+	      pfx.fp_len = steer_pl->classify.l3.mask_width;
+	      pfx.fp_addr.ip4 = steer_pl->classify.l3.prefix.ip4;
+
+	      fib_table_entry_delete (fib_table_id_find_fib_index
+				      (FIB_PROTOCOL_IP4,
+				       steer_pl->classify.l3.fib_table), &pfx,
+				      FIB_SOURCE_SR);
+	    }
+	  else if (steer_pl->classify.traffic_type == SR_STEER_L2)
+	    {
+	      /* Remove HW redirection */
+	      vnet_feature_enable_disable ("device-input",
+					   "sr-policy-rewrite-encaps-l2",
+					   sw_if_index, 0, 0, 0);
+	      sm->sw_iface_sr_policies[sw_if_index] = ~(u32) 0;
+
+	      /* Remove promiscous mode from interface */
+	      vnet_main_t *vnm = vnet_get_main ();
+	      ethernet_main_t *em = ðernet_main;
+	      ethernet_interface_t *eif =
+		ethernet_get_interface (em, sw_if_index);
+
+	      if (!eif)
+		goto cleanup_error_redirection;
+
+	      ethernet_set_flags (vnm, sw_if_index, 0);
+	    }
+
+	  /* Delete SR steering policy entry */
+	  pool_put (sm->steer_policies, steer_pl);
+	  hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
+	  key_copy = (void *) (hp->key);
+	  hash_unset_mem (sm->steer_policies_index_by_key, &key);
+	  vec_free (key_copy);
+	  return 1;
+	}
+      else			/* It means user requested to update an existing SR steering policy */
+	{
+	  /* Retrieve SR steering policy */
+	  if (bsid)
+	    {
+	      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+	      if (p)
+		sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
+	      else
+		return -2;
+	    }
+	  else
+	    sr_policy = pool_elt_at_index (sm->sr_policies, sr_policy_index);
+
+	  if (!sr_policy)
+	    return -2;
+
+	  steer_pl->sr_policy = sr_policy - sm->sr_policies;
+
+	  /* Remove old FIB/hw redirection and create a new one */
+	  if (steer_pl->classify.traffic_type == SR_STEER_IPV6)
+	    {
+	      /* Remove FIB entry */
+	      pfx.fp_proto = FIB_PROTOCOL_IP6;
+	      pfx.fp_len = steer_pl->classify.l3.mask_width;
+	      pfx.fp_addr.ip6 = steer_pl->classify.l3.prefix.ip6;
+
+	      fib_table_entry_delete (fib_table_id_find_fib_index
+				      (FIB_PROTOCOL_IP6,
+				       steer_pl->classify.l3.fib_table), &pfx,
+				      FIB_SOURCE_SR);
+
+	      /* Create a new one */
+	      goto update_fib;
+
+	    }
+	  else if (steer_pl->classify.traffic_type == SR_STEER_IPV4)
+	    {
+	      /* Remove FIB entry */
+	      pfx.fp_proto = FIB_PROTOCOL_IP4;
+	      pfx.fp_len = steer_pl->classify.l3.mask_width;
+	      pfx.fp_addr.ip4 = steer_pl->classify.l3.prefix.ip4;
+
+	      fib_table_entry_delete (fib_table_id_find_fib_index
+				      (FIB_PROTOCOL_IP4,
+				       steer_pl->classify.l3.fib_table), &pfx,
+				      FIB_SOURCE_SR);
+
+	      /* Create a new one */
+	      goto update_fib;
+	    }
+	  else if (steer_pl->classify.traffic_type == SR_STEER_L2)
+	    {
+	      /* Update L2-HW redirection */
+	      goto update_fib;
+	    }
+	}
+    }
+  else
+    /* delete; steering policy does not exist; complain */
+  if (is_del)
+    return -4;
+
+  /* Retrieve SR policy */
+  if (bsid)
+    {
+      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+      if (p)
+	sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
+      else
+	return -2;
+    }
+  else
+    sr_policy = pool_elt_at_index (sm->sr_policies, sr_policy_index);
+
+  /* Create a new steering policy */
+  pool_get (sm->steer_policies, steer_pl);
+  memset (steer_pl, 0, sizeof (*steer_pl));
+
+  if (traffic_type == SR_STEER_IPV4 || traffic_type == SR_STEER_IPV6)
+    {
+      clib_memcpy (&steer_pl->classify.l3.prefix, prefix,
+		   sizeof (ip46_address_t));
+      steer_pl->classify.l3.mask_width = mask_width;
+      steer_pl->classify.l3.fib_table =
+	(table_id != (u32) ~ 0 ? table_id : 0);
+      steer_pl->classify.traffic_type = traffic_type;
+    }
+  else if (traffic_type == SR_STEER_L2)
+    {
+      steer_pl->classify.l2.sw_if_index = sw_if_index;
+      steer_pl->classify.traffic_type = traffic_type;
+    }
+  else
+    {
+      /* Incorrect API usage. Should never get here */
+      pool_put (sm->steer_policies, steer_pl);
+      hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
+      key_copy = (void *) (hp->key);
+      hash_unset_mem (sm->steer_policies_index_by_key, &key);
+      vec_free (key_copy);
+      return -1;
+    }
+  steer_pl->sr_policy = sr_policy - sm->sr_policies;
+
+  /* Create and store key */
+  key_copy = vec_new (sr_steering_key_t, 1);
+  clib_memcpy (key_copy, &key, sizeof (sr_steering_key_t));
+  hash_set_mem (sm->steer_policies_index_by_key,
+		key_copy, steer_pl - sm->steer_policies);
+
+  if (traffic_type == SR_STEER_L2)
+    {
+      if (!sr_policy->is_encap)
+	goto cleanup_error_encap;
+
+      if (vnet_feature_enable_disable
+	  ("device-input", "sr-policy-rewrite-encaps-l2", sw_if_index, 1, 0,
+	   0))
+	goto cleanup_error_redirection;
+
+      /* Set promiscous mode on interface */
+      vnet_main_t *vnm = vnet_get_main ();
+      ethernet_main_t *em = ðernet_main;
+      ethernet_interface_t *eif = ethernet_get_interface (em, sw_if_index);
+
+      if (!eif)
+	goto cleanup_error_redirection;
+
+      ethernet_set_flags (vnm, sw_if_index,
+			  ETHERNET_INTERFACE_FLAG_ACCEPT_ALL);
+    }
+  else if (traffic_type == SR_STEER_IPV4)
+    if (!sr_policy->is_encap)
+      goto cleanup_error_encap;
+
+update_fib:
+  /* FIB API calls - Recursive route through the BindingSID */
+  if (traffic_type == SR_STEER_IPV6)
+    {
+      pfx.fp_proto = FIB_PROTOCOL_IP6;
+      pfx.fp_len = steer_pl->classify.l3.mask_width;
+      pfx.fp_addr.ip6 = steer_pl->classify.l3.prefix.ip6;
+
+      fib_table_entry_path_add (fib_table_id_find_fib_index (FIB_PROTOCOL_IP6,
+							     (table_id !=
+							      (u32) ~ 0 ?
+							      table_id : 0)),
+				&pfx, FIB_SOURCE_CLI, FIB_ENTRY_FLAG_NONE,
+				FIB_PROTOCOL_IP6,
+				(ip46_address_t *) & sr_policy->bsid, ~0,
+				sm->fib_table_ip6, 1, NULL,
+				FIB_ROUTE_PATH_FLAG_NONE);
+    }
+  else if (traffic_type == SR_STEER_IPV4)
+    {
+      pfx.fp_proto = FIB_PROTOCOL_IP4;
+      pfx.fp_len = steer_pl->classify.l3.mask_width;
+      pfx.fp_addr.ip4 = steer_pl->classify.l3.prefix.ip4;
+
+      fib_table_entry_path_add (fib_table_id_find_fib_index (FIB_PROTOCOL_IP4,
+							     (table_id !=
+							      (u32) ~ 0 ?
+							      table_id : 0)),
+				&pfx, FIB_SOURCE_CLI, FIB_ENTRY_FLAG_NONE,
+				FIB_PROTOCOL_IP6,
+				(ip46_address_t *) & sr_policy->bsid, ~0,
+				sm->fib_table_ip4, 1, NULL,
+				FIB_ROUTE_PATH_FLAG_NONE);
+    }
+  else if (traffic_type == SR_STEER_L2)
+    {
+      if (sw_if_index < vec_len (sm->sw_iface_sr_policies))
+	sm->sw_iface_sr_policies[sw_if_index] = steer_pl->sr_policy;
+      else
+	{
+	  vec_resize (sm->sw_iface_sr_policies,
+		      (pool_len (sm->vnet_main->interface_main.sw_interfaces)
+		       - vec_len (sm->sw_iface_sr_policies)));
+	  sm->sw_iface_sr_policies[sw_if_index] = steer_pl->sr_policy;
+	}
+    }
+
+  return 0;
+
+cleanup_error_encap:
+  pool_put (sm->steer_policies, steer_pl);
+  hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
+  key_copy = (void *) (hp->key);
+  hash_unset_mem (sm->steer_policies_index_by_key, &key);
+  vec_free (key_copy);
+  return -5;
+
+cleanup_error_redirection:
+  pool_put (sm->steer_policies, steer_pl);
+  hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
+  key_copy = (void *) (hp->key);
+  hash_unset_mem (sm->steer_policies_index_by_key, &key);
+  vec_free (key_copy);
+  return -3;
+}
+
+static clib_error_t *
+sr_steer_policy_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+
+  int is_del = 0;
+
+  ip46_address_t prefix;
+  u32 dst_mask_width = 0;
+  u32 sw_if_index = (u32) ~ 0;
+  u8 traffic_type = 0;
+  u32 fib_table = (u32) ~ 0;
+
+  ip6_address_t bsid;
+  u32 sr_policy_index = (u32) ~ 0;
+
+  u8 sr_policy_set = 0;
+
+  int rv;
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "del"))
+	is_del = 1;
+      else if (!traffic_type
+	       && unformat (input, "l3 %U/%d", unformat_ip6_address,
+			    &prefix.ip6, &dst_mask_width))
+	traffic_type = SR_STEER_IPV6;
+      else if (!traffic_type
+	       && unformat (input, "l3 %U/%d", unformat_ip4_address,
+			    &prefix.ip4, &dst_mask_width))
+	traffic_type = SR_STEER_IPV4;
+      else if (!traffic_type
+	       && unformat (input, "l2 %U", unformat_vnet_sw_interface, vnm,
+			    &sw_if_index))
+	traffic_type = SR_STEER_L2;
+      else if (!sr_policy_set
+	       && unformat (input, "via sr policy index %d",
+			    &sr_policy_index))
+	sr_policy_set = 1;
+      else if (!sr_policy_set
+	       && unformat (input, "via sr policy bsid %U",
+			    unformat_ip6_address, &bsid))
+	sr_policy_set = 1;
+      else if (fib_table == (u32) ~ 0
+	       && unformat (input, "fib-table %d", &fib_table));
+      else
+	break;
+    }
+
+  if (!traffic_type)
+    return clib_error_return (0, "No L2/L3 traffic specified");
+  if (!sr_policy_set)
+    return clib_error_return (0, "No SR policy specified");
+
+  /* Make sure that the prefixes are clean */
+  if (traffic_type == SR_STEER_IPV4)
+    {
+      u32 mask =
+	(dst_mask_width ? (0xFFFFFFFFu >> (32 - dst_mask_width)) : 0);
+      prefix.ip4.as_u32 &= mask;
+    }
+  else if (traffic_type == SR_STEER_IPV6)
+    {
+      ip6_address_t mask;
+      ip6_address_mask_from_width (&mask, dst_mask_width);
+      ip6_address_mask (&prefix.ip6, &mask);
+    }
+
+  rv =
+    sr_steering_policy (is_del, (sr_policy_index == ~(u32) 0 ? &bsid : NULL),
+			sr_policy_index, fib_table, &prefix, dst_mask_width,
+			sw_if_index, traffic_type);
+
+  switch (rv)
+    {
+    case 0:
+      break;
+    case 1:
+      return 0;
+    case -1:
+      return clib_error_return (0, "Incorrect API usage.");
+    case -2:
+      return clib_error_return (0,
+				"The requested SR policy could not be located. Review the BSID/index.");
+    case -3:
+      return clib_error_return (0,
+				"Unable to do SW redirect. Incorrect interface.");
+    case -4:
+      return clib_error_return (0,
+				"The requested SR policy could not be deleted. Review the BSID/index.");
+    case -5:
+      return clib_error_return (0,
+				"The SR policy is not an encapsulation one.");
+    default:
+      return clib_error_return (0, "BUG: sr steer policy returns %d", rv);
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (sr_steer_policy_command, static) = {
+  .path = "sr steer",
+  .short_help = "sr steer (del) [l3 |l2 ]"
+    "via sr policy [index |bsid ]"
+    "(fib-table )",
+  .long_help =
+    "\tSteer a L2 or L3 traffic through an existing SR policy.\n"
+    "\tExamples:\n"
+    "\t\tsr steer l3 2001::/64 via sr_policy index 5\n"
+    "\t\tsr steer l3 2001::/64 via sr_policy bsid 2010::9999:1\n"
+    "\t\tsr steer l2 GigabitEthernet0/5/0 via sr_policy index 5\n"
+    "\t\tsr steer del l3 2001::/64 via sr_policy index 5\n",
+  .function = sr_steer_policy_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_sr_steering_policies_command_fn (vlib_main_t * vm,
+				      unformat_input_t * input,
+				      vlib_cli_command_t * cmd)
+{
+  ip6_sr_main_t *sm = &sr_main;
+  ip6_sr_steering_policy_t **steer_policies = 0;
+  ip6_sr_steering_policy_t *steer_pl;
+
+  vnet_main_t *vnm = vnet_get_main ();
+
+  ip6_sr_policy_t *pl = 0;
+  int i;
+
+  vlib_cli_output (vm, "SR steering policies:");
+  /* *INDENT-OFF* */
+  pool_foreach (steer_pl, sm->steer_policies, ({vec_add1(steer_policies, steer_pl);}));
+  /* *INDENT-ON* */
+  vlib_cli_output (vm, "Traffic\t\tSR policy BSID");
+  for (i = 0; i < vec_len (steer_policies); i++)
+    {
+      steer_pl = steer_policies[i];
+      pl = pool_elt_at_index (sm->sr_policies, steer_pl->sr_policy);
+      if (steer_pl->classify.traffic_type == SR_STEER_L2)
+	{
+	  vlib_cli_output (vm, "L2 %U\t%U",
+			   format_vnet_sw_if_index_name, vnm,
+			   steer_pl->classify.l2.sw_if_index,
+			   format_ip6_address, &pl->bsid);
+	}
+      else if (steer_pl->classify.traffic_type == SR_STEER_IPV4)
+	{
+	  vlib_cli_output (vm, "L3 %U/%d\t%U",
+			   format_ip4_address,
+			   &steer_pl->classify.l3.prefix.ip4,
+			   steer_pl->classify.l3.mask_width,
+			   format_ip6_address, &pl->bsid);
+	}
+      else if (steer_pl->classify.traffic_type == SR_STEER_IPV6)
+	{
+	  vlib_cli_output (vm, "L3 %U/%d\t%U",
+			   format_ip6_address,
+			   &steer_pl->classify.l3.prefix.ip6,
+			   steer_pl->classify.l3.mask_width,
+			   format_ip6_address, &pl->bsid);
+	}
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_sr_steering_policies_command, static) = {
+  .path = "show sr steering policies",
+  .short_help = "show sr steering policies",
+  .function = show_sr_steering_policies_command_fn,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+sr_steering_init (vlib_main_t * vm)
+{
+  ip6_sr_main_t *sm = &sr_main;
+
+  /* Init memory for function keys */
+  sm->steer_policies_index_by_key =
+    hash_create_mem (0, sizeof (sr_steering_key_t), sizeof (uword));
+
+  sm->sw_iface_sr_policies = 0;
+
+  sm->vnet_main = vnet_get_main ();
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (sr_steering_init);
+
+VNET_FEATURE_INIT (sr_policy_rewrite_encaps_l2, static) =
+{
+.arc_name = "device-input",.node_name =
+    "sr-pl-rewrite-encaps-l2",.runs_before =
+    VNET_FEATURES ("ethernet-input"),};
+
+/*
+* fd.io coding-style-patch-verification: ON
+*
+* Local Variables:
+* eval: (c-set-style "gnu")
+* End:
+*/
diff --git a/src/vpp/api/custom_dump.c b/src/vpp/api/custom_dump.c
index ee0c4629..8250279e 100644
--- a/src/vpp/api/custom_dump.c
+++ b/src/vpp/api/custom_dump.c
@@ -24,7 +24,7 @@
 #include 
 #include 
 #include 
-#include 
+#include 
 #include 
 #include 
 #include 
@@ -960,143 +960,215 @@ static void *vl_api_l2_patch_add_del_t_print
   FINISH;
 }
 
-static void *vl_api_sr_tunnel_add_del_t_print
-  (vl_api_sr_tunnel_add_del_t * mp, void *handle)
+static void *vl_api_sr_localsid_add_del_t_print
+  (vl_api_sr_localsid_add_del_t * mp, void *handle)
 {
+  vnet_main_t *vnm = vnet_get_main ();
   u8 *s;
-  ip6_address_t *this_address;
-  int i;
-  u16 flags_host_byte_order;
-  u8 pl_flag;
-
-  s = format (0, "SCRIPT: sr_tunnel_add_del ");
-
-  if (mp->name[0])
-    s = format (s, "name %s ", mp->name);
 
-  s = format (s, "src %U dst %U/%d ", format_ip6_address,
-	      (ip6_address_t *) mp->src_address,
-	      format_ip6_address,
-	      (ip6_address_t *) mp->dst_address, mp->dst_mask_width);
+  s = format (0, "SCRIPT: sr_localsid_add_del ");
 
-  this_address = (ip6_address_t *) mp->segs_and_tags;
-  for (i = 0; i < mp->n_segments; i++)
+  switch (mp->behavior)
     {
-      s = format (s, "next %U ", format_ip6_address, this_address);
-      this_address++;
-    }
-  for (i = 0; i < mp->n_tags; i++)
-    {
-      s = format (s, "tag %U ", format_ip6_address, this_address);
-      this_address++;
+    case SR_BEHAVIOR_END:
+      s = format (s, "Address: %U\nBehavior: End",
+		  format_ip6_address, (ip6_address_t *) mp->localsid_addr);
+      s = format (s, (mp->end_psp ? "End.PSP: True" : "End.PSP: False"));
+      break;
+    case SR_BEHAVIOR_X:
+      s =
+	format (s,
+		"Address: %U\nBehavior: X (Endpoint with Layer-3 cross-connect)"
+		"\nIface: %U\nNext hop: %U", format_ip6_address,
+		(ip6_address_t *) mp->localsid_addr,
+		format_vnet_sw_if_index_name, vnm, ntohl (mp->sw_if_index),
+		format_ip6_address, (ip6_address_t *) mp->nh_addr);
+      s = format (s, (mp->end_psp ? "End.PSP: True" : "End.PSP: False"));
+      break;
+    case SR_BEHAVIOR_DX4:
+      s =
+	format (s,
+		"Address: %U\nBehavior: DX4 (Endpoint with decapsulation with IPv4 cross-connect)"
+		"\nIface: %U\nNext hop: %U", format_ip6_address,
+		(ip6_address_t *) mp->localsid_addr,
+		format_vnet_sw_if_index_name, vnm, ntohl (mp->sw_if_index),
+		format_ip4_address, (ip4_address_t *) mp->nh_addr);
+      break;
+    case SR_BEHAVIOR_DX6:
+      s =
+	format (s,
+		"Address: %U\nBehavior: DX6 (Endpoint with decapsulation with IPv6 cross-connect)"
+		"\nIface: %UNext hop: %U", format_ip6_address,
+		(ip6_address_t *) mp->localsid_addr,
+		format_vnet_sw_if_index_name, vnm, ntohl (mp->sw_if_index),
+		format_ip6_address, (ip6_address_t *) mp->nh_addr);
+      break;
+    case SR_BEHAVIOR_DX2:
+      s =
+	format (s,
+		"Address: %U\nBehavior: DX2 (Endpoint with decapulation and Layer-2 cross-connect)"
+		"\nIface: %U", format_ip6_address,
+		(ip6_address_t *) mp->localsid_addr,
+		format_vnet_sw_if_index_name, vnm, ntohl (mp->sw_if_index));
+      break;
+    case SR_BEHAVIOR_DT6:
+      s =
+	format (s,
+		"Address: %U\nBehavior: DT6 (Endpoint with decapsulation and specific IPv6 table lookup)"
+		"\nTable: %u", format_ip6_address,
+		(ip6_address_t *) mp->localsid_addr, ntohl (mp->fib_table));
+      break;
+    case SR_BEHAVIOR_DT4:
+      s =
+	format (s,
+		"Address: %U\nBehavior: DT4 (Endpoint with decapsulation and specific IPv4 table lookup)"
+		"\nTable: %u", format_ip6_address,
+		(ip6_address_t *) mp->localsid_addr, ntohl (mp->fib_table));
+      break;
+    default:
+      if (mp->behavior >= SR_BEHAVIOR_LAST)
+	{
+	  s = format (s, "Address: %U\n Behavior: %u",
+		      format_ip6_address, (ip6_address_t *) mp->localsid_addr,
+		      mp->behavior);
+	}
+      else
+	//Should never get here...
+	s = format (s, "Internal error");
+      break;
     }
+  FINISH;
+}
 
-  flags_host_byte_order = clib_net_to_host_u16 (mp->flags_net_byte_order);
+static void *vl_api_sr_steering_add_del_t_print
+  (vl_api_sr_steering_add_del_t * mp, void *handle)
+{
+  u8 *s;
 
-  if (flags_host_byte_order & IP6_SR_HEADER_FLAG_CLEANUP)
-    s = format (s, " clean ");
+  s = format (0, "SCRIPT: sr_steering_add_del ");
 
-  if (flags_host_byte_order & IP6_SR_HEADER_FLAG_PROTECTED)
-    s = format (s, "protected ");
+  s = format (s, (mp->is_del ? "Del: True" : "Del: False"));
 
-  for (i = 1; i <= 4; i++)
+  switch (mp->traffic_type)
     {
-      pl_flag = ip6_sr_policy_list_flags (flags_host_byte_order, i);
-
-      switch (pl_flag)
-	{
-	case IP6_SR_HEADER_FLAG_PL_ELT_NOT_PRESENT:
-	  continue;
-
-	case IP6_SR_HEADER_FLAG_PL_ELT_INGRESS_PE:
-	  s = format (s, "InPE %d ", i);
-	  break;
-
-	case IP6_SR_HEADER_FLAG_PL_ELT_EGRESS_PE:
-	  s = format (s, "EgPE %d ", i);
-	  break;
-
-	case IP6_SR_HEADER_FLAG_PL_ELT_ORIG_SRC_ADDR:
-	  s = format (s, "OrgSrc %d ", i);
-	  break;
-
-	default:
-	  clib_warning ("BUG: pl elt %d value %d", i, pl_flag);
-	  break;
-	}
+    case SR_STEER_L2:
+      s = format (s, "Traffic type: L2 iface: %u", ntohl (mp->sw_if_index));
+      break;
+    case SR_STEER_IPV4:
+      s = format (s, "Traffic type: IPv4 %U/%u", format_ip4_address,
+		  (ip4_address_t *) mp->prefix_addr, ntohl (mp->mask_width));
+      break;
+    case SR_STEER_IPV6:
+      s = format (s, "Traffic type: IPv6 %U/%u", format_ip6_address,
+		  (ip6_address_t *) mp->prefix_addr, ntohl (mp->mask_width));
+      break;
+    default:
+      s = format (s, "Traffic type: Unknown(%u)", mp->traffic_type);
+      break;
     }
+  s = format (s, "BindingSID: %U", format_ip6_address,
+	      (ip6_address_t *) mp->bsid_addr);
 
-  if (mp->policy_name[0])
-    s = format (s, "policy_name %s ", mp->policy_name);
+  s = format (s, "SR Policy Index: %u", ntohl (mp->sr_policy_index));
 
-  if (mp->is_add == 0)
-    s = format (s, "del ");
+  s = format (s, "FIB_table: %u", ntohl (mp->table_id));
 
   FINISH;
 }
 
-static void *vl_api_sr_policy_add_del_t_print
-  (vl_api_sr_policy_add_del_t * mp, void *handle)
+static void *vl_api_sr_policy_add_t_print
+  (vl_api_sr_policy_add_t * mp, void *handle)
 {
   u8 *s;
+
+  ip6_address_t *segments = 0, *seg;
+  ip6_address_t *this_address = (ip6_address_t *) mp->segments;
+
   int i;
+  for (i = 0; i < mp->n_segments; i++)
+    {
+      vec_add2 (segments, seg, 1);
+      clib_memcpy (seg->as_u8, this_address->as_u8, sizeof (*this_address));
+      this_address++;
+    }
 
-  s = format (0, "SCRIPT: sr_policy_add_del ");
+  s = format (0, "SCRIPT: sr_policy_add ");
 
-  if (mp->name[0])
-    s = format (s, "name %s ", mp->name);
+  s = format (s, "BSID: %U", format_ip6_address,
+	      (ip6_address_t *) mp->bsid_addr);
 
+  s =
+    format (s,
+	    (mp->is_encap ? "Behavior: Encapsulation" :
+	     "Behavior: SRH insertion"));
 
-  if (mp->tunnel_names[0])
-    {
-      // start deserializing tunnel_names
-      int num_tunnels = mp->tunnel_names[0];	//number of tunnels
-      u8 *deser_tun_names = mp->tunnel_names;
-      deser_tun_names += 1;	//moving along
+  s = format (s, "FIB_table: %u", ntohl (mp->fib_table));
 
-      u8 *tun_name = 0;
-      int tun_name_len = 0;
+  s = format (s, (mp->type ? "Type: Default" : "Type: Spray"));
 
-      for (i = 0; i < num_tunnels; i++)
-	{
-	  tun_name_len = *deser_tun_names;
-	  deser_tun_names += 1;
-	  vec_resize (tun_name, tun_name_len);
-	  memcpy (tun_name, deser_tun_names, tun_name_len);
-	  s = format (s, "tunnel %s ", tun_name);
-	  deser_tun_names += tun_name_len;
-	  tun_name = 0;
-	}
-    }
+  s = format (s, "SID list weight: %u", ntohl (mp->weight));
 
-  if (mp->is_add == 0)
-    s = format (s, "del ");
+  s = format (s, "{");
+  vec_foreach (seg, segments)
+  {
+    s = format (s, "%U, ", format_ip6_address, seg);
+  }
+  s = format (s, "\b\b } ");
 
   FINISH;
 }
 
-static void *vl_api_sr_multicast_map_add_del_t_print
-  (vl_api_sr_multicast_map_add_del_t * mp, void *handle)
+static void *vl_api_sr_policy_mod_t_print
+  (vl_api_sr_policy_mod_t * mp, void *handle)
 {
+  u8 *s;
+  u32 weight;
 
-  u8 *s = 0;
-  /* int i; */
+  ip6_address_t *segments = 0, *seg;
+  ip6_address_t *this_address = (ip6_address_t *) mp->segments;
 
-  s = format (0, "SCRIPT: sr_multicast_map_add_del ");
+  int i;
+  for (i = 0; i < mp->n_segments; i++)
+    {
+      vec_add2 (segments, seg, 1);
+      clib_memcpy (seg->as_u8, this_address->as_u8, sizeof (*this_address));
+      this_address++;
+    }
 
-  if (mp->multicast_address[0])
-    s = format (s, "address %U ", format_ip6_address, &mp->multicast_address);
+  s = format (0, "SCRIPT: sr_policy_mod ");
 
-  if (mp->policy_name[0])
-    s = format (s, "sr-policy %s ", &mp->policy_name);
+  s = format (s, "BSID: %U", format_ip6_address,
+	      (ip6_address_t *) mp->bsid_addr);
 
+  s = format (s, "SR Policy index: %u", ntohl (mp->sr_policy_index));
 
-  if (mp->is_add == 0)
-    s = format (s, "del ");
+  s = format (s, "Operation: %u", mp->operation);
+
+  s = format (s, "SID list index: %u", ntohl (mp->sl_index));
+
+  s = format (s, "SID list weight: %u", ntohl (mp->weight));
+
+  s = format (s, "{");
+  vec_foreach (seg, segments)
+  {
+    s = format (s, "%U, ", format_ip6_address, seg);
+  }
+  s = format (s, "\b\b } ");
 
   FINISH;
 }
 
+static void *vl_api_sr_policy_del_t_print
+  (vl_api_sr_policy_del_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: sr_policy_del ");
+  u8 bsid_addr[16];
+  u32 sr_policy_index;
+  s = format (s, "To be delivered. Good luck.");
+  FINISH;
+}
 
 static void *vl_api_classify_add_del_table_t_print
   (vl_api_classify_add_del_table_t * mp, void *handle)
@@ -2864,9 +2936,11 @@ _(SW_INTERFACE_IP6ND_RA_PREFIX, sw_interface_ip6nd_ra_prefix)           \
 _(SW_INTERFACE_IP6ND_RA_CONFIG, sw_interface_ip6nd_ra_config)           \
 _(SET_ARP_NEIGHBOR_LIMIT, set_arp_neighbor_limit)                       \
 _(L2_PATCH_ADD_DEL, l2_patch_add_del)                                   \
-_(SR_TUNNEL_ADD_DEL, sr_tunnel_add_del)					\
-_(SR_POLICY_ADD_DEL, sr_policy_add_del)					\
-_(SR_MULTICAST_MAP_ADD_DEL, sr_multicast_map_add_del)                   \
+_(SR_LOCALSID_ADD_DEL, sr_localsid_add_del)                             \
+_(SR_STEERING_ADD_DEL, sr_steering_add_del)                             \
+_(SR_POLICY_ADD, sr_policy_add)                                         \
+_(SR_POLICY_MOD, sr_policy_mod)                                         \
+_(SR_POLICY_DEL, sr_policy_del)                                         \
 _(SW_INTERFACE_SET_L2_XCONNECT, sw_interface_set_l2_xconnect)           \
 _(L2FIB_ADD_DEL, l2fib_add_del)                                         \
 _(L2_FLAGS, l2_flags)                                                   \
diff --git a/src/vpp/api/test_client.c b/src/vpp/api/test_client.c
index 8b61f4f3..551bdab9 100644
--- a/src/vpp/api/test_client.c
+++ b/src/vpp/api/test_client.c
@@ -607,7 +607,6 @@ _(SW_INTERFACE_IP6_SET_LINK_LOCAL_ADDRESS_REPLY, sw_interface_ip6_set_link_local
  _(CREATE_LOOPBACK_REPLY, create_loopback_reply)			\
  _(CREATE_LOOPBACK_INSTANCE_REPLY, create_loopback_instance_reply)	\
 _(L2_PATCH_ADD_DEL_REPLY, l2_patch_add_del_reply)			\
-_(SR_TUNNEL_ADD_DEL_REPLY,sr_tunnel_add_del_reply)          \
 _(SW_INTERFACE_SET_L2_XCONNECT_REPLY, sw_interface_set_l2_xconnect_reply) \
 _(SW_INTERFACE_SET_L2_BRIDGE_REPLY, sw_interface_set_l2_bridge_reply)
 
-- 
cgit 1.2.3-korg


From 374e2c5fc30a5bfabfd2eb6c2d3ca5797402af16 Mon Sep 17 00:00:00 2001
From: Damjan Marion 
Date: Thu, 9 Mar 2017 20:38:15 +0100
Subject: Retire vpp_lite

vpp_lite platform is not needed anymore as same efect can be
achieved with following startup.conf config:

plugins {
	plugin dpdk_plugin.so { disable }
}

Change-Id: I690ea8ceb1c6e1fe32e01e7da54e9958019a93bf
Signed-off-by: Damjan Marion 
---
 Makefile                                        |   16 +-
 build-data/platforms/vpp.mk                     |    5 -
 build-data/platforms/vpp_lite.mk                |   50 -
 src/configure.ac                                |    4 +-
 src/plugins/Makefile.am                         |    6 +-
 src/plugins/ixge.am                             |   20 +
 src/plugins/ixge/ixge.c                         | 2947 +++++++++++++++++++++++
 src/plugins/ixge/ixge.h                         | 1293 ++++++++++
 src/vlib/buffer.c                               |    6 +
 src/vnet.am                                     |   14 +-
 src/vnet/devices/nic/ixge.c                     | 2938 ----------------------
 src/vnet/devices/nic/ixge.h                     | 1293 ----------
 src/vnet/devices/nic/sfp.c                      |  117 -
 src/vnet/devices/nic/sfp.h                      |  117 -
 src/vnet/ethernet/sfp.c                         |  117 +
 src/vnet/ethernet/sfp.h                         |  117 +
 src/vpp-api/lua/bench.lua                       |    4 +-
 src/vpp-api/lua/examples/cli/lua-cli.lua        |    4 +-
 src/vpp-api/lua/examples/example-classifier.lua |    4 +-
 src/vpp-api/lua/examples/example-cli.lua        |    4 +-
 src/vpp/vnet/main.c                             |    3 -
 test/framework.py                               |    4 +-
 22 files changed, 4529 insertions(+), 4554 deletions(-)
 delete mode 100644 build-data/platforms/vpp_lite.mk
 create mode 100644 src/plugins/ixge.am
 create mode 100644 src/plugins/ixge/ixge.c
 create mode 100644 src/plugins/ixge/ixge.h
 delete mode 100644 src/vnet/devices/nic/ixge.c
 delete mode 100644 src/vnet/devices/nic/ixge.h
 delete mode 100644 src/vnet/devices/nic/sfp.c
 delete mode 100644 src/vnet/devices/nic/sfp.h
 create mode 100644 src/vnet/ethernet/sfp.c
 create mode 100644 src/vnet/ethernet/sfp.h

(limited to 'src/plugins/Makefile.am')

diff --git a/Makefile b/Makefile
index f0173cc1..1527d60a 100644
--- a/Makefile
+++ b/Makefile
@@ -230,18 +230,18 @@ define test
 endef
 
 test: bootstrap
-	$(call test,vpp_lite,vpp_lite,test)
+	$(call test,vpp,vpp,test)
 
 test-debug: bootstrap
-	$(call test,vpp_lite,vpp_lite_debug,test)
+	$(call test,vpp,vpp_debug,test)
 
 test-all: bootstrap
 	$(eval EXTENDED_TESTS=yes)
-	$(call test,vpp_lite,vpp_lite,test)
+	$(call test,vpp,vpp,test)
 
 test-all-debug: bootstrap
 	$(eval EXTENDED_TESTS=yes)
-	$(call test,vpp_lite,vpp_lite_debug,test)
+	$(call test,vpp,vpp_debug,test)
 
 test-help:
 	@make -C test help
@@ -262,7 +262,7 @@ test-wipe-doc:
 	@make -C test wipe-doc
 
 test-cov: bootstrap
-	$(call test,vpp_lite,vpp_lite_gcov,cov)
+	$(call test,vpp,vpp_gcov,cov)
 
 test-wipe-cov:
 	@make -C test wipe-cov
@@ -271,10 +271,10 @@ test-checkstyle:
 	@make -C test checkstyle
 
 retest:
-	$(call test,vpp_lite,vpp_lite,retest)
+	$(call test,vpp,vpp,retest)
 
 retest-debug:
-	$(call test,vpp_lite,vpp_lite_debug,retest)
+	$(call test,vpp,vpp_debug,retest)
 
 STARTUP_DIR ?= $(PWD)
 ifeq ("$(wildcard $(STARTUP_CONF))","")
@@ -376,8 +376,6 @@ endef
 verify: install-dep $(BR)/.bootstrap.ok dpdk-install-dev
 	$(call banner,"Building for PLATFORM=vpp using gcc")
 	@make -C build-root PLATFORM=vpp TAG=vpp wipe-all install-packages
-	$(call banner,"Building for PLATFORM=vpp_lite using gcc")
-	@make -C build-root PLATFORM=vpp_lite TAG=vpp_lite wipe-all install-packages
 ifeq ($(OS_ID)-$(OS_VERSION_ID),ubuntu-16.04)
 	$(call banner,"Installing dependencies")
 	@sudo -E apt-get update
diff --git a/build-data/platforms/vpp.mk b/build-data/platforms/vpp.mk
index 401a383a..c61375d8 100644
--- a/build-data/platforms/vpp.mk
+++ b/build-data/platforms/vpp.mk
@@ -35,11 +35,6 @@ vpp_uses_dpdk = yes
 
 vpp_root_packages = vpp gmod
 
-vpp_configure_args_vpp = --with-dpdk
-
-# Set these parameters carefully. The vlib_buffer_t is 128 bytes, i.e.
-vlib_configure_args_vpp = --with-pre-data=128
-
 # DPDK configuration parameters
 # vpp_uses_dpdk_cryptodev_sw = yes
 # vpp_uses_dpdk_mlx5_pmd = yes
diff --git a/build-data/platforms/vpp_lite.mk b/build-data/platforms/vpp_lite.mk
deleted file mode 100644
index a556b487..00000000
--- a/build-data/platforms/vpp_lite.mk
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# vector packet processor
-vpp_lite_arch = native
-ifeq ($(shell uname -m),x86_64)
-vpp_lite_march = corei7				# Nehalem Instruction set
-vpp_lite_mtune = corei7-avx			# Optimize for Sandy Bridge
-else
-vpp_lite_march = native
-vpp_lite_mtune = generic
-endif
-vpp_lite_native_tools = vppapigen
-
-vpp_lite_uses_dpdk = no
-
-# Uncoment to enable building unit tests
-#vpp_lite_enable_tests = yes
-
-vpp_lite_root_packages = vpp gmod
-
-vlib_configure_args_vpp_lite = --with-pre-data=128
-
-vnet_configure_args_vpp_lite =
-vpp_configure_args_vpp_lite =
-
-vpp_lite_debug_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \
-	-fstack-protector-all -fPIC -Werror
-vpp_lite_debug_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \
-	-fstack-protector-all -fPIC -Werror
-
-vpp_lite_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \
-	-fstack-protector -fPIC -Werror
-vpp_lite_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \
-	-fstack-protector -fPIC -Werror
-
-vpp_lite_gcov_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -march=$(MARCH) \
-	-fPIC -Werror -fprofile-arcs -ftest-coverage
-vpp_lite_gcov_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -march=$(MARCH) \
-	-fPIC -Werror -coverage
diff --git a/src/configure.ac b/src/configure.ac
index c22d152e..d90740d9 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -97,7 +97,6 @@ DISABLE_ARG(papi,	[Disable Python API bindings])
 DISABLE_ARG(japi,	[Disable Java API bindings])
 
 # --with-X
-WITH_ARG(dpdk,		[Use use DPDK])
 WITH_ARG(dpdk_crypto_sw,[Use DPDK cryptodev SW PMDs])
 WITH_ARG(dpdk_mlx5_pmd,	[Use DPDK with mlx5 PMD])
 
@@ -130,7 +129,6 @@ AC_ARG_WITH(pre-data,
 AC_SUBST(PRE_DATA_SIZE,		[$with_pre_data])
 AC_SUBST(APICLI,		[-DVPP_API_TEST_BUILTIN=${n_with_apicli}])
 
-AC_DEFINE_UNQUOTED(DPDK, 		[${n_with_dpdk}])
 AC_DEFINE_UNQUOTED(DPDK_SHARED_LIB,	[${n_enable_dpdk_shared}])
 AC_DEFINE_UNQUOTED(DPDK_CRYPTO_SW,	[${n_with_dpdk_crypto_sw}])
 AC_DEFINE_UNQUOTED(WITH_LIBSSL,		[${n_with_libssl}])
@@ -147,9 +145,11 @@ AC_SUBST(AR_FLAGS)
 
 # Please keep alphabetical order
 PLUGIN_ENABLED(acl)
+PLUGIN_ENABLED(dpdk)
 PLUGIN_ENABLED(flowperpkt)
 PLUGIN_ENABLED(ila)
 PLUGIN_ENABLED(ioam)
+PLUGIN_ENABLED(ixge)
 PLUGIN_ENABLED(lb)
 PLUGIN_ENABLED(sixrd)
 PLUGIN_ENABLED(snat)
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 7b36049e..255e644f 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -33,7 +33,7 @@ if ENABLE_ACL_PLUGIN
 include acl.am
 endif
 
-if WITH_DPDK
+if ENABLE_DPDK_PLUGIN
 include dpdk.am
 endif
 
@@ -49,6 +49,10 @@ if ENABLE_IOAM_PLUGIN
 include ioam.am
 endif
 
+if ENABLE_IXGE_PLUGIN
+include ixge.am
+endif
+
 if ENABLE_LB_PLUGIN
 include lb.am
 endif
diff --git a/src/plugins/ixge.am b/src/plugins/ixge.am
new file mode 100644
index 00000000..7e61344b
--- /dev/null
+++ b/src/plugins/ixge.am
@@ -0,0 +1,20 @@
+# Copyright (c) 2016 Cisco Systems, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppplugins_LTLIBRARIES += ixge_plugin.la
+
+ixge_plugin_la_SOURCES = ixge/ixge.c
+
+noinst_HEADERS += ixge/ixge.h
+
+# vi:syntax=automake
diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c
new file mode 100644
index 00000000..4eebc457
--- /dev/null
+++ b/src/plugins/ixge/ixge.c
@@ -0,0 +1,2947 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *   WARNING!
+ *   This driver is not intended for production use and it is unsupported.
+ *   It is provided for educational use only.
+ *   Please use supported DPDK driver instead.
+ */
+
+#if __x86_64__
+#include 
+
+#ifndef CLIB_HAVE_VEC128
+#warning HACK: ixge driver wont really work, missing u32x4
+typedef unsigned long long u32x4;
+#endif
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define IXGE_ALWAYS_POLL 0
+
+#define EVENT_SET_FLAGS 0
+#define IXGE_HWBP_RACE_ELOG 0
+
+#define PCI_VENDOR_ID_INTEL 0x8086
+
+/* 10 GIG E (XGE) PHY IEEE 802.3 clause 45 definitions. */
+#define XGE_PHY_DEV_TYPE_PMA_PMD 1
+#define XGE_PHY_DEV_TYPE_PHY_XS 4
+#define XGE_PHY_ID1 0x2
+#define XGE_PHY_ID2 0x3
+#define XGE_PHY_CONTROL 0x0
+#define XGE_PHY_CONTROL_RESET (1 << 15)
+
+ixge_main_t ixge_main;
+static vlib_node_registration_t ixge_input_node;
+static vlib_node_registration_t ixge_process_node;
+
+static void
+ixge_semaphore_get (ixge_device_t * xd)
+{
+  ixge_main_t *xm = &ixge_main;
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_regs_t *r = xd->regs;
+  u32 i;
+
+  i = 0;
+  while (!(r->software_semaphore & (1 << 0)))
+    {
+      if (i > 0)
+	vlib_process_suspend (vm, 100e-6);
+      i++;
+    }
+  do
+    {
+      r->software_semaphore |= 1 << 1;
+    }
+  while (!(r->software_semaphore & (1 << 1)));
+}
+
+static void
+ixge_semaphore_release (ixge_device_t * xd)
+{
+  ixge_regs_t *r = xd->regs;
+  r->software_semaphore &= ~3;
+}
+
+static void
+ixge_software_firmware_sync (ixge_device_t * xd, u32 sw_mask)
+{
+  ixge_main_t *xm = &ixge_main;
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_regs_t *r = xd->regs;
+  u32 fw_mask = sw_mask << 5;
+  u32 m, done = 0;
+
+  while (!done)
+    {
+      ixge_semaphore_get (xd);
+      m = r->software_firmware_sync;
+      done = (m & fw_mask) == 0;
+      if (done)
+	r->software_firmware_sync = m | sw_mask;
+      ixge_semaphore_release (xd);
+      if (!done)
+	vlib_process_suspend (vm, 10e-3);
+    }
+}
+
+static void
+ixge_software_firmware_sync_release (ixge_device_t * xd, u32 sw_mask)
+{
+  ixge_regs_t *r = xd->regs;
+  ixge_semaphore_get (xd);
+  r->software_firmware_sync &= ~sw_mask;
+  ixge_semaphore_release (xd);
+}
+
+u32
+ixge_read_write_phy_reg (ixge_device_t * xd, u32 dev_type, u32 reg_index,
+			 u32 v, u32 is_read)
+{
+  ixge_regs_t *r = xd->regs;
+  const u32 busy_bit = 1 << 30;
+  u32 x;
+
+  ASSERT (xd->phy_index < 2);
+  ixge_software_firmware_sync (xd, 1 << (1 + xd->phy_index));
+
+  ASSERT (reg_index < (1 << 16));
+  ASSERT (dev_type < (1 << 5));
+  if (!is_read)
+    r->xge_mac.phy_data = v;
+
+  /* Address cycle. */
+  x =
+    reg_index | (dev_type << 16) | (xd->
+				    phys[xd->phy_index].mdio_address << 21);
+  r->xge_mac.phy_command = x | busy_bit;
+  /* Busy wait timed to take 28e-6 secs.  No suspend. */
+  while (r->xge_mac.phy_command & busy_bit)
+    ;
+
+  r->xge_mac.phy_command = x | ((is_read ? 2 : 1) << 26) | busy_bit;
+  while (r->xge_mac.phy_command & busy_bit)
+    ;
+
+  if (is_read)
+    v = r->xge_mac.phy_data >> 16;
+
+  ixge_software_firmware_sync_release (xd, 1 << (1 + xd->phy_index));
+
+  return v;
+}
+
+static u32
+ixge_read_phy_reg (ixge_device_t * xd, u32 dev_type, u32 reg_index)
+{
+  return ixge_read_write_phy_reg (xd, dev_type, reg_index, 0,	/* is_read */
+				  1);
+}
+
+static void
+ixge_write_phy_reg (ixge_device_t * xd, u32 dev_type, u32 reg_index, u32 v)
+{
+  (void) ixge_read_write_phy_reg (xd, dev_type, reg_index, v,	/* is_read */
+				  0);
+}
+
+static void
+ixge_i2c_put_bits (i2c_bus_t * b, int scl, int sda)
+{
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, b->private_data);
+  u32 v;
+
+  v = 0;
+  v |= (sda != 0) << 3;
+  v |= (scl != 0) << 1;
+  xd->regs->i2c_control = v;
+}
+
+static void
+ixge_i2c_get_bits (i2c_bus_t * b, int *scl, int *sda)
+{
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, b->private_data);
+  u32 v;
+
+  v = xd->regs->i2c_control;
+  *sda = (v & (1 << 2)) != 0;
+  *scl = (v & (1 << 0)) != 0;
+}
+
+static u16
+ixge_read_eeprom (ixge_device_t * xd, u32 address)
+{
+  ixge_regs_t *r = xd->regs;
+  u32 v;
+  r->eeprom_read = (( /* start bit */ (1 << 0)) | (address << 2));
+  /* Wait for done bit. */
+  while (!((v = r->eeprom_read) & (1 << 1)))
+    ;
+  return v >> 16;
+}
+
+static void
+ixge_sfp_enable_disable_laser (ixge_device_t * xd, uword enable)
+{
+  u32 tx_disable_bit = 1 << 3;
+  if (enable)
+    xd->regs->sdp_control &= ~tx_disable_bit;
+  else
+    xd->regs->sdp_control |= tx_disable_bit;
+}
+
+static void
+ixge_sfp_enable_disable_10g (ixge_device_t * xd, uword enable)
+{
+  u32 is_10g_bit = 1 << 5;
+  if (enable)
+    xd->regs->sdp_control |= is_10g_bit;
+  else
+    xd->regs->sdp_control &= ~is_10g_bit;
+}
+
+static clib_error_t *
+ixge_sfp_phy_init_from_eeprom (ixge_device_t * xd, u16 sfp_type)
+{
+  u16 a, id, reg_values_addr = 0;
+
+  a = ixge_read_eeprom (xd, 0x2b);
+  if (a == 0 || a == 0xffff)
+    return clib_error_create ("no init sequence in eeprom");
+
+  while (1)
+    {
+      id = ixge_read_eeprom (xd, ++a);
+      if (id == 0xffff)
+	break;
+      reg_values_addr = ixge_read_eeprom (xd, ++a);
+      if (id == sfp_type)
+	break;
+    }
+  if (id != sfp_type)
+    return clib_error_create ("failed to find id 0x%x", sfp_type);
+
+  ixge_software_firmware_sync (xd, 1 << 3);
+  while (1)
+    {
+      u16 v = ixge_read_eeprom (xd, ++reg_values_addr);
+      if (v == 0xffff)
+	break;
+      xd->regs->core_analog_config = v;
+    }
+  ixge_software_firmware_sync_release (xd, 1 << 3);
+
+  /* Make sure laser is off.  We'll turn on the laser when
+     the interface is brought up. */
+  ixge_sfp_enable_disable_laser (xd, /* enable */ 0);
+  ixge_sfp_enable_disable_10g (xd, /* is_10g */ 1);
+
+  return 0;
+}
+
+static void
+ixge_sfp_device_up_down (ixge_device_t * xd, uword is_up)
+{
+  u32 v;
+
+  if (is_up)
+    {
+      /* pma/pmd 10g serial SFI. */
+      xd->regs->xge_mac.auto_negotiation_control2 &= ~(3 << 16);
+      xd->regs->xge_mac.auto_negotiation_control2 |= 2 << 16;
+
+      v = xd->regs->xge_mac.auto_negotiation_control;
+      v &= ~(7 << 13);
+      v |= (0 << 13);
+      /* Restart autoneg. */
+      v |= (1 << 12);
+      xd->regs->xge_mac.auto_negotiation_control = v;
+
+      while (!(xd->regs->xge_mac.link_partner_ability[0] & 0xf0000))
+	;
+
+      v = xd->regs->xge_mac.auto_negotiation_control;
+
+      /* link mode 10g sfi serdes */
+      v &= ~(7 << 13);
+      v |= (3 << 13);
+
+      /* Restart autoneg. */
+      v |= (1 << 12);
+      xd->regs->xge_mac.auto_negotiation_control = v;
+
+      xd->regs->xge_mac.link_status;
+    }
+
+  ixge_sfp_enable_disable_laser (xd, /* enable */ is_up);
+
+  /* Give time for link partner to notice that we're up. */
+  if (is_up && vlib_in_process_context (vlib_get_main ()))
+    {
+      vlib_process_suspend (vlib_get_main (), 300e-3);
+    }
+}
+
+always_inline ixge_dma_regs_t *
+get_dma_regs (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 qi)
+{
+  ixge_regs_t *r = xd->regs;
+  ASSERT (qi < 128);
+  if (rt == VLIB_RX)
+    return qi < 64 ? &r->rx_dma0[qi] : &r->rx_dma1[qi - 64];
+  else
+    return &r->tx_dma[qi];
+}
+
+static clib_error_t *
+ixge_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
+{
+  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
+  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, hif->dev_instance);
+  ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_RX, 0);
+
+  if (is_up)
+    {
+      xd->regs->rx_enable |= 1;
+      xd->regs->tx_dma_control |= 1;
+      dr->control |= 1 << 25;
+      while (!(dr->control & (1 << 25)))
+	;
+    }
+  else
+    {
+      xd->regs->rx_enable &= ~1;
+      xd->regs->tx_dma_control &= ~1;
+    }
+
+  ixge_sfp_device_up_down (xd, is_up);
+
+  return /* no error */ 0;
+}
+
+static void
+ixge_sfp_phy_init (ixge_device_t * xd)
+{
+  ixge_phy_t *phy = xd->phys + xd->phy_index;
+  i2c_bus_t *ib = &xd->i2c_bus;
+
+  ib->private_data = xd->device_index;
+  ib->put_bits = ixge_i2c_put_bits;
+  ib->get_bits = ixge_i2c_get_bits;
+  vlib_i2c_init (ib);
+
+  vlib_i2c_read_eeprom (ib, 0x50, 0, 128, (u8 *) & xd->sfp_eeprom);
+
+  if (vlib_i2c_bus_timed_out (ib) || !sfp_eeprom_is_valid (&xd->sfp_eeprom))
+    xd->sfp_eeprom.id = SFP_ID_unknown;
+  else
+    {
+      /* FIXME 5 => SR/LR eeprom ID. */
+      clib_error_t *e =
+	ixge_sfp_phy_init_from_eeprom (xd, 5 + xd->pci_function);
+      if (e)
+	clib_error_report (e);
+    }
+
+  phy->mdio_address = ~0;
+}
+
+static void
+ixge_phy_init (ixge_device_t * xd)
+{
+  ixge_main_t *xm = &ixge_main;
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_phy_t *phy = xd->phys + xd->phy_index;
+
+  switch (xd->device_id)
+    {
+    case IXGE_82599_sfp:
+    case IXGE_82599_sfp_em:
+    case IXGE_82599_sfp_fcoe:
+      /* others? */
+      return ixge_sfp_phy_init (xd);
+
+    default:
+      break;
+    }
+
+  /* Probe address of phy. */
+  {
+    u32 i, v;
+
+    phy->mdio_address = ~0;
+    for (i = 0; i < 32; i++)
+      {
+	phy->mdio_address = i;
+	v = ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PMA_PMD, XGE_PHY_ID1);
+	if (v != 0xffff && v != 0)
+	  break;
+      }
+
+    /* No PHY found? */
+    if (i >= 32)
+      return;
+  }
+
+  phy->id =
+    ((ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PMA_PMD, XGE_PHY_ID1) << 16) |
+     ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PMA_PMD, XGE_PHY_ID2));
+
+  {
+    ELOG_TYPE_DECLARE (e) =
+    {
+    .function = (char *) __FUNCTION__,.format =
+	"ixge %d, phy id 0x%d mdio address %d",.format_args = "i4i4i4",};
+    struct
+    {
+      u32 instance, id, address;
+    } *ed;
+    ed = ELOG_DATA (&vm->elog_main, e);
+    ed->instance = xd->device_index;
+    ed->id = phy->id;
+    ed->address = phy->mdio_address;
+  }
+
+  /* Reset phy. */
+  ixge_write_phy_reg (xd, XGE_PHY_DEV_TYPE_PHY_XS, XGE_PHY_CONTROL,
+		      XGE_PHY_CONTROL_RESET);
+
+  /* Wait for self-clearning reset bit to clear. */
+  do
+    {
+      vlib_process_suspend (vm, 1e-3);
+    }
+  while (ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PHY_XS, XGE_PHY_CONTROL) &
+	 XGE_PHY_CONTROL_RESET);
+}
+
+static u8 *
+format_ixge_rx_from_hw_descriptor (u8 * s, va_list * va)
+{
+  ixge_rx_from_hw_descriptor_t *d =
+    va_arg (*va, ixge_rx_from_hw_descriptor_t *);
+  u32 s0 = d->status[0], s2 = d->status[2];
+  u32 is_ip4, is_ip6, is_ip, is_tcp, is_udp;
+  uword indent = format_get_indent (s);
+
+  s = format (s, "%s-owned",
+	      (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE) ? "sw" :
+	      "hw");
+  s =
+    format (s, ", length this descriptor %d, l3 offset %d",
+	    d->n_packet_bytes_this_descriptor,
+	    IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s0));
+  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET)
+    s = format (s, ", end-of-packet");
+
+  s = format (s, "\n%U", format_white_space, indent);
+
+  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_ETHERNET_ERROR)
+    s = format (s, "layer2 error");
+
+  if (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_LAYER2)
+    {
+      s = format (s, "layer 2 type %d", (s0 & 0x1f));
+      return s;
+    }
+
+  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_VLAN)
+    s = format (s, "vlan header 0x%x\n%U", d->vlan_tag,
+		format_white_space, indent);
+
+  if ((is_ip4 = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4)))
+    {
+      s = format (s, "ip4%s",
+		  (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4_EXT) ? " options" :
+		  "");
+      if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED)
+	s = format (s, " checksum %s",
+		    (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR) ?
+		    "bad" : "ok");
+    }
+  if ((is_ip6 = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6)))
+    s = format (s, "ip6%s",
+		(s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6_EXT) ? " extended" :
+		"");
+  is_tcp = is_udp = 0;
+  if ((is_ip = (is_ip4 | is_ip6)))
+    {
+      is_tcp = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_TCP) != 0;
+      is_udp = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_UDP) != 0;
+      if (is_tcp)
+	s = format (s, ", tcp");
+      if (is_udp)
+	s = format (s, ", udp");
+    }
+
+  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED)
+    s = format (s, ", tcp checksum %s",
+		(s2 & IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR) ? "bad" :
+		"ok");
+  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED)
+    s = format (s, ", udp checksum %s",
+		(s2 & IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR) ? "bad" :
+		"ok");
+
+  return s;
+}
+
+static u8 *
+format_ixge_tx_descriptor (u8 * s, va_list * va)
+{
+  ixge_tx_descriptor_t *d = va_arg (*va, ixge_tx_descriptor_t *);
+  u32 s0 = d->status0, s1 = d->status1;
+  uword indent = format_get_indent (s);
+  u32 v;
+
+  s = format (s, "buffer 0x%Lx, %d packet bytes, %d bytes this buffer",
+	      d->buffer_address, s1 >> 14, d->n_bytes_this_buffer);
+
+  s = format (s, "\n%U", format_white_space, indent);
+
+  if ((v = (s0 >> 0) & 3))
+    s = format (s, "reserved 0x%x, ", v);
+
+  if ((v = (s0 >> 2) & 3))
+    s = format (s, "mac 0x%x, ", v);
+
+  if ((v = (s0 >> 4) & 0xf) != 3)
+    s = format (s, "type 0x%x, ", v);
+
+  s = format (s, "%s%s%s%s%s%s%s%s",
+	      (s0 & (1 << 8)) ? "eop, " : "",
+	      (s0 & (1 << 9)) ? "insert-fcs, " : "",
+	      (s0 & (1 << 10)) ? "reserved26, " : "",
+	      (s0 & (1 << 11)) ? "report-status, " : "",
+	      (s0 & (1 << 12)) ? "reserved28, " : "",
+	      (s0 & (1 << 13)) ? "is-advanced, " : "",
+	      (s0 & (1 << 14)) ? "vlan-enable, " : "",
+	      (s0 & (1 << 15)) ? "tx-segmentation, " : "");
+
+  if ((v = s1 & 0xf) != 0)
+    s = format (s, "status 0x%x, ", v);
+
+  if ((v = (s1 >> 4) & 0xf))
+    s = format (s, "context 0x%x, ", v);
+
+  if ((v = (s1 >> 8) & 0x3f))
+    s = format (s, "options 0x%x, ", v);
+
+  return s;
+}
+
+typedef struct
+{
+  ixge_descriptor_t before, after;
+
+  u32 buffer_index;
+
+  u16 device_index;
+
+  u8 queue_index;
+
+  u8 is_start_of_packet;
+
+  /* Copy of VLIB buffer; packet data stored in pre_data. */
+  vlib_buffer_t buffer;
+} ixge_rx_dma_trace_t;
+
+static u8 *
+format_ixge_rx_dma_trace (u8 * s, va_list * va)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
+  vlib_node_t *node = va_arg (*va, vlib_node_t *);
+  vnet_main_t *vnm = vnet_get_main ();
+  ixge_rx_dma_trace_t *t = va_arg (*va, ixge_rx_dma_trace_t *);
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, t->device_index);
+  format_function_t *f;
+  uword indent = format_get_indent (s);
+
+  {
+    vnet_sw_interface_t *sw =
+      vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
+    s =
+      format (s, "%U rx queue %d", format_vnet_sw_interface_name, vnm, sw,
+	      t->queue_index);
+  }
+
+  s = format (s, "\n%Ubefore: %U",
+	      format_white_space, indent,
+	      format_ixge_rx_from_hw_descriptor, &t->before);
+  s = format (s, "\n%Uafter : head/tail address 0x%Lx/0x%Lx",
+	      format_white_space, indent,
+	      t->after.rx_to_hw.head_address, t->after.rx_to_hw.tail_address);
+
+  s = format (s, "\n%Ubuffer 0x%x: %U",
+	      format_white_space, indent,
+	      t->buffer_index, format_vlib_buffer, &t->buffer);
+
+  s = format (s, "\n%U", format_white_space, indent);
+
+  f = node->format_buffer;
+  if (!f || !t->is_start_of_packet)
+    f = format_hex_bytes;
+  s = format (s, "%U", f, t->buffer.pre_data, sizeof (t->buffer.pre_data));
+
+  return s;
+}
+
+#define foreach_ixge_error					\
+  _ (none, "no error")						\
+  _ (tx_full_drops, "tx ring full drops")			\
+  _ (ip4_checksum_error, "ip4 checksum errors")			\
+  _ (rx_alloc_fail, "rx buf alloc from free list failed")	\
+  _ (rx_alloc_no_physmem, "rx buf alloc failed no physmem")
+
+typedef enum
+{
+#define _(f,s) IXGE_ERROR_##f,
+  foreach_ixge_error
+#undef _
+    IXGE_N_ERROR,
+} ixge_error_t;
+
+always_inline void
+ixge_rx_next_and_error_from_status_x1 (ixge_device_t * xd,
+				       u32 s00, u32 s02,
+				       u8 * next0, u8 * error0, u32 * flags0)
+{
+  u8 is0_ip4, is0_ip6, n0, e0;
+  u32 f0;
+
+  e0 = IXGE_ERROR_none;
+  n0 = IXGE_RX_NEXT_ETHERNET_INPUT;
+
+  is0_ip4 = s02 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED;
+  n0 = is0_ip4 ? IXGE_RX_NEXT_IP4_INPUT : n0;
+
+  e0 = (is0_ip4 && (s02 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR)
+	? IXGE_ERROR_ip4_checksum_error : e0);
+
+  is0_ip6 = s00 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6;
+  n0 = is0_ip6 ? IXGE_RX_NEXT_IP6_INPUT : n0;
+
+  n0 = (xd->per_interface_next_index != ~0) ?
+    xd->per_interface_next_index : n0;
+
+  /* Check for error. */
+  n0 = e0 != IXGE_ERROR_none ? IXGE_RX_NEXT_DROP : n0;
+
+  f0 = ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED
+		| IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED))
+	? IP_BUFFER_L4_CHECKSUM_COMPUTED : 0);
+
+  f0 |= ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR
+		 | IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR))
+	 ? 0 : IP_BUFFER_L4_CHECKSUM_CORRECT);
+
+  *error0 = e0;
+  *next0 = n0;
+  *flags0 = f0;
+}
+
+always_inline void
+ixge_rx_next_and_error_from_status_x2 (ixge_device_t * xd,
+				       u32 s00, u32 s02,
+				       u32 s10, u32 s12,
+				       u8 * next0, u8 * error0, u32 * flags0,
+				       u8 * next1, u8 * error1, u32 * flags1)
+{
+  u8 is0_ip4, is0_ip6, n0, e0;
+  u8 is1_ip4, is1_ip6, n1, e1;
+  u32 f0, f1;
+
+  e0 = e1 = IXGE_ERROR_none;
+  n0 = n1 = IXGE_RX_NEXT_IP4_INPUT;
+
+  is0_ip4 = s02 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED;
+  is1_ip4 = s12 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED;
+
+  n0 = is0_ip4 ? IXGE_RX_NEXT_IP4_INPUT : n0;
+  n1 = is1_ip4 ? IXGE_RX_NEXT_IP4_INPUT : n1;
+
+  e0 = (is0_ip4 && (s02 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR)
+	? IXGE_ERROR_ip4_checksum_error : e0);
+  e1 = (is1_ip4 && (s12 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR)
+	? IXGE_ERROR_ip4_checksum_error : e1);
+
+  is0_ip6 = s00 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6;
+  is1_ip6 = s10 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6;
+
+  n0 = is0_ip6 ? IXGE_RX_NEXT_IP6_INPUT : n0;
+  n1 = is1_ip6 ? IXGE_RX_NEXT_IP6_INPUT : n1;
+
+  n0 = (xd->per_interface_next_index != ~0) ?
+    xd->per_interface_next_index : n0;
+  n1 = (xd->per_interface_next_index != ~0) ?
+    xd->per_interface_next_index : n1;
+
+  /* Check for error. */
+  n0 = e0 != IXGE_ERROR_none ? IXGE_RX_NEXT_DROP : n0;
+  n1 = e1 != IXGE_ERROR_none ? IXGE_RX_NEXT_DROP : n1;
+
+  *error0 = e0;
+  *error1 = e1;
+
+  *next0 = n0;
+  *next1 = n1;
+
+  f0 = ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED
+		| IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED))
+	? IP_BUFFER_L4_CHECKSUM_COMPUTED : 0);
+  f1 = ((s12 & (IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED
+		| IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED))
+	? IP_BUFFER_L4_CHECKSUM_COMPUTED : 0);
+
+  f0 |= ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR
+		 | IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR))
+	 ? 0 : IP_BUFFER_L4_CHECKSUM_CORRECT);
+  f1 |= ((s12 & (IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR
+		 | IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR))
+	 ? 0 : IP_BUFFER_L4_CHECKSUM_CORRECT);
+
+  *flags0 = f0;
+  *flags1 = f1;
+}
+
+static void
+ixge_rx_trace (ixge_main_t * xm,
+	       ixge_device_t * xd,
+	       ixge_dma_queue_t * dq,
+	       ixge_descriptor_t * before_descriptors,
+	       u32 * before_buffers,
+	       ixge_descriptor_t * after_descriptors, uword n_descriptors)
+{
+  vlib_main_t *vm = xm->vlib_main;
+  vlib_node_runtime_t *node = dq->rx.node;
+  ixge_rx_from_hw_descriptor_t *bd;
+  ixge_rx_to_hw_descriptor_t *ad;
+  u32 *b, n_left, is_sop, next_index_sop;
+
+  n_left = n_descriptors;
+  b = before_buffers;
+  bd = &before_descriptors->rx_from_hw;
+  ad = &after_descriptors->rx_to_hw;
+  is_sop = dq->rx.is_start_of_packet;
+  next_index_sop = dq->rx.saved_start_of_packet_next_index;
+
+  while (n_left >= 2)
+    {
+      u32 bi0, bi1, flags0, flags1;
+      vlib_buffer_t *b0, *b1;
+      ixge_rx_dma_trace_t *t0, *t1;
+      u8 next0, error0, next1, error1;
+
+      bi0 = b[0];
+      bi1 = b[1];
+      n_left -= 2;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      b1 = vlib_get_buffer (vm, bi1);
+
+      ixge_rx_next_and_error_from_status_x2 (xd,
+					     bd[0].status[0], bd[0].status[2],
+					     bd[1].status[0], bd[1].status[2],
+					     &next0, &error0, &flags0,
+					     &next1, &error1, &flags1);
+
+      next_index_sop = is_sop ? next0 : next_index_sop;
+      vlib_trace_buffer (vm, node, next_index_sop, b0, /* follow_chain */ 0);
+      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+      t0->is_start_of_packet = is_sop;
+      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      next_index_sop = is_sop ? next1 : next_index_sop;
+      vlib_trace_buffer (vm, node, next_index_sop, b1, /* follow_chain */ 0);
+      t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
+      t1->is_start_of_packet = is_sop;
+      is_sop = (b1->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      t0->queue_index = dq->queue_index;
+      t1->queue_index = dq->queue_index;
+      t0->device_index = xd->device_index;
+      t1->device_index = xd->device_index;
+      t0->before.rx_from_hw = bd[0];
+      t1->before.rx_from_hw = bd[1];
+      t0->after.rx_to_hw = ad[0];
+      t1->after.rx_to_hw = ad[1];
+      t0->buffer_index = bi0;
+      t1->buffer_index = bi1;
+      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
+      memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b0->pre_data));
+      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
+	      sizeof (t0->buffer.pre_data));
+      memcpy (t1->buffer.pre_data, b1->data + b1->current_data,
+	      sizeof (t1->buffer.pre_data));
+
+      b += 2;
+      bd += 2;
+      ad += 2;
+    }
+
+  while (n_left >= 1)
+    {
+      u32 bi0, flags0;
+      vlib_buffer_t *b0;
+      ixge_rx_dma_trace_t *t0;
+      u8 next0, error0;
+
+      bi0 = b[0];
+      n_left -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      ixge_rx_next_and_error_from_status_x1 (xd,
+					     bd[0].status[0], bd[0].status[2],
+					     &next0, &error0, &flags0);
+
+      next_index_sop = is_sop ? next0 : next_index_sop;
+      vlib_trace_buffer (vm, node, next_index_sop, b0, /* follow_chain */ 0);
+      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+      t0->is_start_of_packet = is_sop;
+      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      t0->queue_index = dq->queue_index;
+      t0->device_index = xd->device_index;
+      t0->before.rx_from_hw = bd[0];
+      t0->after.rx_to_hw = ad[0];
+      t0->buffer_index = bi0;
+      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
+      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
+	      sizeof (t0->buffer.pre_data));
+
+      b += 1;
+      bd += 1;
+      ad += 1;
+    }
+}
+
+typedef struct
+{
+  ixge_tx_descriptor_t descriptor;
+
+  u32 buffer_index;
+
+  u16 device_index;
+
+  u8 queue_index;
+
+  u8 is_start_of_packet;
+
+  /* Copy of VLIB buffer; packet data stored in pre_data. */
+  vlib_buffer_t buffer;
+} ixge_tx_dma_trace_t;
+
+static u8 *
+format_ixge_tx_dma_trace (u8 * s, va_list * va)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
+  ixge_tx_dma_trace_t *t = va_arg (*va, ixge_tx_dma_trace_t *);
+  vnet_main_t *vnm = vnet_get_main ();
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, t->device_index);
+  format_function_t *f;
+  uword indent = format_get_indent (s);
+
+  {
+    vnet_sw_interface_t *sw =
+      vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
+    s =
+      format (s, "%U tx queue %d", format_vnet_sw_interface_name, vnm, sw,
+	      t->queue_index);
+  }
+
+  s = format (s, "\n%Udescriptor: %U",
+	      format_white_space, indent,
+	      format_ixge_tx_descriptor, &t->descriptor);
+
+  s = format (s, "\n%Ubuffer 0x%x: %U",
+	      format_white_space, indent,
+	      t->buffer_index, format_vlib_buffer, &t->buffer);
+
+  s = format (s, "\n%U", format_white_space, indent);
+
+  f = format_ethernet_header_with_length;
+  if (!f || !t->is_start_of_packet)
+    f = format_hex_bytes;
+  s = format (s, "%U", f, t->buffer.pre_data, sizeof (t->buffer.pre_data));
+
+  return s;
+}
+
+typedef struct
+{
+  vlib_node_runtime_t *node;
+
+  u32 is_start_of_packet;
+
+  u32 n_bytes_in_packet;
+
+  ixge_tx_descriptor_t *start_of_packet_descriptor;
+} ixge_tx_state_t;
+
+static void
+ixge_tx_trace (ixge_main_t * xm,
+	       ixge_device_t * xd,
+	       ixge_dma_queue_t * dq,
+	       ixge_tx_state_t * tx_state,
+	       ixge_tx_descriptor_t * descriptors,
+	       u32 * buffers, uword n_descriptors)
+{
+  vlib_main_t *vm = xm->vlib_main;
+  vlib_node_runtime_t *node = tx_state->node;
+  ixge_tx_descriptor_t *d;
+  u32 *b, n_left, is_sop;
+
+  n_left = n_descriptors;
+  b = buffers;
+  d = descriptors;
+  is_sop = tx_state->is_start_of_packet;
+
+  while (n_left >= 2)
+    {
+      u32 bi0, bi1;
+      vlib_buffer_t *b0, *b1;
+      ixge_tx_dma_trace_t *t0, *t1;
+
+      bi0 = b[0];
+      bi1 = b[1];
+      n_left -= 2;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      b1 = vlib_get_buffer (vm, bi1);
+
+      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+      t0->is_start_of_packet = is_sop;
+      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
+      t1->is_start_of_packet = is_sop;
+      is_sop = (b1->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      t0->queue_index = dq->queue_index;
+      t1->queue_index = dq->queue_index;
+      t0->device_index = xd->device_index;
+      t1->device_index = xd->device_index;
+      t0->descriptor = d[0];
+      t1->descriptor = d[1];
+      t0->buffer_index = bi0;
+      t1->buffer_index = bi1;
+      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
+      memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b0->pre_data));
+      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
+	      sizeof (t0->buffer.pre_data));
+      memcpy (t1->buffer.pre_data, b1->data + b1->current_data,
+	      sizeof (t1->buffer.pre_data));
+
+      b += 2;
+      d += 2;
+    }
+
+  while (n_left >= 1)
+    {
+      u32 bi0;
+      vlib_buffer_t *b0;
+      ixge_tx_dma_trace_t *t0;
+
+      bi0 = b[0];
+      n_left -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+      t0->is_start_of_packet = is_sop;
+      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      t0->queue_index = dq->queue_index;
+      t0->device_index = xd->device_index;
+      t0->descriptor = d[0];
+      t0->buffer_index = bi0;
+      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
+      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
+	      sizeof (t0->buffer.pre_data));
+
+      b += 1;
+      d += 1;
+    }
+}
+
+always_inline uword
+ixge_ring_sub (ixge_dma_queue_t * q, u32 i0, u32 i1)
+{
+  i32 d = i1 - i0;
+  ASSERT (i0 < q->n_descriptors);
+  ASSERT (i1 < q->n_descriptors);
+  return d < 0 ? q->n_descriptors + d : d;
+}
+
+always_inline uword
+ixge_ring_add (ixge_dma_queue_t * q, u32 i0, u32 i1)
+{
+  u32 d = i0 + i1;
+  ASSERT (i0 < q->n_descriptors);
+  ASSERT (i1 < q->n_descriptors);
+  d -= d >= q->n_descriptors ? q->n_descriptors : 0;
+  return d;
+}
+
+always_inline uword
+ixge_tx_descriptor_matches_template (ixge_main_t * xm,
+				     ixge_tx_descriptor_t * d)
+{
+  u32 cmp;
+
+  cmp = ((d->status0 & xm->tx_descriptor_template_mask.status0)
+	 ^ xm->tx_descriptor_template.status0);
+  if (cmp)
+    return 0;
+  cmp = ((d->status1 & xm->tx_descriptor_template_mask.status1)
+	 ^ xm->tx_descriptor_template.status1);
+  if (cmp)
+    return 0;
+
+  return 1;
+}
+
+static uword
+ixge_tx_no_wrap (ixge_main_t * xm,
+		 ixge_device_t * xd,
+		 ixge_dma_queue_t * dq,
+		 u32 * buffers,
+		 u32 start_descriptor_index,
+		 u32 n_descriptors, ixge_tx_state_t * tx_state)
+{
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_tx_descriptor_t *d, *d_sop;
+  u32 n_left = n_descriptors;
+  u32 *to_free = vec_end (xm->tx_buffers_pending_free);
+  u32 *to_tx =
+    vec_elt_at_index (dq->descriptor_buffer_indices, start_descriptor_index);
+  u32 is_sop = tx_state->is_start_of_packet;
+  u32 len_sop = tx_state->n_bytes_in_packet;
+  u16 template_status = xm->tx_descriptor_template.status0;
+  u32 descriptor_prefetch_rotor = 0;
+
+  ASSERT (start_descriptor_index + n_descriptors <= dq->n_descriptors);
+  d = &dq->descriptors[start_descriptor_index].tx;
+  d_sop = is_sop ? d : tx_state->start_of_packet_descriptor;
+
+  while (n_left >= 4)
+    {
+      vlib_buffer_t *b0, *b1;
+      u32 bi0, fi0, len0;
+      u32 bi1, fi1, len1;
+      u8 is_eop0, is_eop1;
+
+      /* Prefetch next iteration. */
+      vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
+
+      if ((descriptor_prefetch_rotor & 0x3) == 0)
+	CLIB_PREFETCH (d + 4, CLIB_CACHE_LINE_BYTES, STORE);
+
+      descriptor_prefetch_rotor += 2;
+
+      bi0 = buffers[0];
+      bi1 = buffers[1];
+
+      to_free[0] = fi0 = to_tx[0];
+      to_tx[0] = bi0;
+      to_free += fi0 != 0;
+
+      to_free[0] = fi1 = to_tx[1];
+      to_tx[1] = bi1;
+      to_free += fi1 != 0;
+
+      buffers += 2;
+      n_left -= 2;
+      to_tx += 2;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      b1 = vlib_get_buffer (vm, bi1);
+
+      is_eop0 = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+      is_eop1 = (b1->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      len0 = b0->current_length;
+      len1 = b1->current_length;
+
+      ASSERT (ixge_tx_descriptor_matches_template (xm, d + 0));
+      ASSERT (ixge_tx_descriptor_matches_template (xm, d + 1));
+
+      d[0].buffer_address =
+	vlib_get_buffer_data_physical_address (vm, bi0) + b0->current_data;
+      d[1].buffer_address =
+	vlib_get_buffer_data_physical_address (vm, bi1) + b1->current_data;
+
+      d[0].n_bytes_this_buffer = len0;
+      d[1].n_bytes_this_buffer = len1;
+
+      d[0].status0 =
+	template_status | (is_eop0 <<
+			   IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET);
+      d[1].status0 =
+	template_status | (is_eop1 <<
+			   IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET);
+
+      len_sop = (is_sop ? 0 : len_sop) + len0;
+      d_sop[0].status1 =
+	IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET (len_sop);
+      d += 1;
+      d_sop = is_eop0 ? d : d_sop;
+
+      is_sop = is_eop0;
+
+      len_sop = (is_sop ? 0 : len_sop) + len1;
+      d_sop[0].status1 =
+	IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET (len_sop);
+      d += 1;
+      d_sop = is_eop1 ? d : d_sop;
+
+      is_sop = is_eop1;
+    }
+
+  while (n_left > 0)
+    {
+      vlib_buffer_t *b0;
+      u32 bi0, fi0, len0;
+      u8 is_eop0;
+
+      bi0 = buffers[0];
+
+      to_free[0] = fi0 = to_tx[0];
+      to_tx[0] = bi0;
+      to_free += fi0 != 0;
+
+      buffers += 1;
+      n_left -= 1;
+      to_tx += 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      is_eop0 = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
+
+      len0 = b0->current_length;
+
+      ASSERT (ixge_tx_descriptor_matches_template (xm, d + 0));
+
+      d[0].buffer_address =
+	vlib_get_buffer_data_physical_address (vm, bi0) + b0->current_data;
+
+      d[0].n_bytes_this_buffer = len0;
+
+      d[0].status0 =
+	template_status | (is_eop0 <<
+			   IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET);
+
+      len_sop = (is_sop ? 0 : len_sop) + len0;
+      d_sop[0].status1 =
+	IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET (len_sop);
+      d += 1;
+      d_sop = is_eop0 ? d : d_sop;
+
+      is_sop = is_eop0;
+    }
+
+  if (tx_state->node->flags & VLIB_NODE_FLAG_TRACE)
+    {
+      to_tx =
+	vec_elt_at_index (dq->descriptor_buffer_indices,
+			  start_descriptor_index);
+      ixge_tx_trace (xm, xd, dq, tx_state,
+		     &dq->descriptors[start_descriptor_index].tx, to_tx,
+		     n_descriptors);
+    }
+
+  _vec_len (xm->tx_buffers_pending_free) =
+    to_free - xm->tx_buffers_pending_free;
+
+  /* When we are done d_sop can point to end of ring.  Wrap it if so. */
+  {
+    ixge_tx_descriptor_t *d_start = &dq->descriptors[0].tx;
+
+    ASSERT (d_sop - d_start <= dq->n_descriptors);
+    d_sop = d_sop - d_start == dq->n_descriptors ? d_start : d_sop;
+  }
+
+  tx_state->is_start_of_packet = is_sop;
+  tx_state->start_of_packet_descriptor = d_sop;
+  tx_state->n_bytes_in_packet = len_sop;
+
+  return n_descriptors;
+}
+
+static uword
+ixge_interface_tx (vlib_main_t * vm,
+		   vlib_node_runtime_t * node, vlib_frame_t * f)
+{
+  ixge_main_t *xm = &ixge_main;
+  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, rd->dev_instance);
+  ixge_dma_queue_t *dq;
+  u32 *from, n_left_tx, n_descriptors_to_tx, n_tail_drop;
+  u32 queue_index = 0;		/* fixme parameter */
+  ixge_tx_state_t tx_state;
+
+  tx_state.node = node;
+  tx_state.is_start_of_packet = 1;
+  tx_state.start_of_packet_descriptor = 0;
+  tx_state.n_bytes_in_packet = 0;
+
+  from = vlib_frame_vector_args (f);
+
+  dq = vec_elt_at_index (xd->dma_queues[VLIB_TX], queue_index);
+
+  dq->head_index = dq->tx.head_index_write_back[0];
+
+  /* Since head == tail means ring is empty we can send up to dq->n_descriptors - 1. */
+  n_left_tx = dq->n_descriptors - 1;
+  n_left_tx -= ixge_ring_sub (dq, dq->head_index, dq->tail_index);
+
+  _vec_len (xm->tx_buffers_pending_free) = 0;
+
+  n_descriptors_to_tx = f->n_vectors;
+  n_tail_drop = 0;
+  if (PREDICT_FALSE (n_descriptors_to_tx > n_left_tx))
+    {
+      i32 i, n_ok, i_eop, i_sop;
+
+      i_sop = i_eop = ~0;
+      for (i = n_left_tx - 1; i >= 0; i--)
+	{
+	  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
+	  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
+	    {
+	      if (i_sop != ~0 && i_eop != ~0)
+		break;
+	      i_eop = i;
+	      i_sop = i + 1;
+	    }
+	}
+      if (i == 0)
+	n_ok = 0;
+      else
+	n_ok = i_eop + 1;
+
+      {
+	ELOG_TYPE_DECLARE (e) =
+	{
+	.function = (char *) __FUNCTION__,.format =
+	    "ixge %d, ring full to tx %d head %d tail %d",.format_args =
+	    "i2i2i2i2",};
+	struct
+	{
+	  u16 instance, to_tx, head, tail;
+	} *ed;
+	ed = ELOG_DATA (&vm->elog_main, e);
+	ed->instance = xd->device_index;
+	ed->to_tx = n_descriptors_to_tx;
+	ed->head = dq->head_index;
+	ed->tail = dq->tail_index;
+      }
+
+      if (n_ok < n_descriptors_to_tx)
+	{
+	  n_tail_drop = n_descriptors_to_tx - n_ok;
+	  vec_add (xm->tx_buffers_pending_free, from + n_ok, n_tail_drop);
+	  vlib_error_count (vm, ixge_input_node.index,
+			    IXGE_ERROR_tx_full_drops, n_tail_drop);
+	}
+
+      n_descriptors_to_tx = n_ok;
+    }
+
+  dq->tx.n_buffers_on_ring += n_descriptors_to_tx;
+
+  /* Process from tail to end of descriptor ring. */
+  if (n_descriptors_to_tx > 0 && dq->tail_index < dq->n_descriptors)
+    {
+      u32 n =
+	clib_min (dq->n_descriptors - dq->tail_index, n_descriptors_to_tx);
+      n = ixge_tx_no_wrap (xm, xd, dq, from, dq->tail_index, n, &tx_state);
+      from += n;
+      n_descriptors_to_tx -= n;
+      dq->tail_index += n;
+      ASSERT (dq->tail_index <= dq->n_descriptors);
+      if (dq->tail_index == dq->n_descriptors)
+	dq->tail_index = 0;
+    }
+
+  if (n_descriptors_to_tx > 0)
+    {
+      u32 n =
+	ixge_tx_no_wrap (xm, xd, dq, from, 0, n_descriptors_to_tx, &tx_state);
+      from += n;
+      ASSERT (n == n_descriptors_to_tx);
+      dq->tail_index += n;
+      ASSERT (dq->tail_index <= dq->n_descriptors);
+      if (dq->tail_index == dq->n_descriptors)
+	dq->tail_index = 0;
+    }
+
+  /* We should only get full packets. */
+  ASSERT (tx_state.is_start_of_packet);
+
+  /* Report status when last descriptor is done. */
+  {
+    u32 i = dq->tail_index == 0 ? dq->n_descriptors - 1 : dq->tail_index - 1;
+    ixge_tx_descriptor_t *d = &dq->descriptors[i].tx;
+    d->status0 |= IXGE_TX_DESCRIPTOR_STATUS0_REPORT_STATUS;
+  }
+
+  /* Give new descriptors to hardware. */
+  {
+    ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_TX, queue_index);
+
+    CLIB_MEMORY_BARRIER ();
+
+    dr->tail_index = dq->tail_index;
+  }
+
+  /* Free any buffers that are done. */
+  {
+    u32 n = _vec_len (xm->tx_buffers_pending_free);
+    if (n > 0)
+      {
+	vlib_buffer_free_no_next (vm, xm->tx_buffers_pending_free, n);
+	_vec_len (xm->tx_buffers_pending_free) = 0;
+	ASSERT (dq->tx.n_buffers_on_ring >= n);
+	dq->tx.n_buffers_on_ring -= (n - n_tail_drop);
+      }
+  }
+
+  return f->n_vectors;
+}
+
+static uword
+ixge_rx_queue_no_wrap (ixge_main_t * xm,
+		       ixge_device_t * xd,
+		       ixge_dma_queue_t * dq,
+		       u32 start_descriptor_index, u32 n_descriptors)
+{
+  vlib_main_t *vm = xm->vlib_main;
+  vlib_node_runtime_t *node = dq->rx.node;
+  ixge_descriptor_t *d;
+  static ixge_descriptor_t *d_trace_save;
+  static u32 *d_trace_buffers;
+  u32 n_descriptors_left = n_descriptors;
+  u32 *to_rx =
+    vec_elt_at_index (dq->descriptor_buffer_indices, start_descriptor_index);
+  u32 *to_add;
+  u32 bi_sop = dq->rx.saved_start_of_packet_buffer_index;
+  u32 bi_last = dq->rx.saved_last_buffer_index;
+  u32 next_index_sop = dq->rx.saved_start_of_packet_next_index;
+  u32 is_sop = dq->rx.is_start_of_packet;
+  u32 next_index, n_left_to_next, *to_next;
+  u32 n_packets = 0;
+  u32 n_bytes = 0;
+  u32 n_trace = vlib_get_trace_count (vm, node);
+  vlib_buffer_t *b_last, b_dummy;
+
+  ASSERT (start_descriptor_index + n_descriptors <= dq->n_descriptors);
+  d = &dq->descriptors[start_descriptor_index];
+
+  b_last = bi_last != ~0 ? vlib_get_buffer (vm, bi_last) : &b_dummy;
+  next_index = dq->rx.next_index;
+
+  if (n_trace > 0)
+    {
+      u32 n = clib_min (n_trace, n_descriptors);
+      if (d_trace_save)
+	{
+	  _vec_len (d_trace_save) = 0;
+	  _vec_len (d_trace_buffers) = 0;
+	}
+      vec_add (d_trace_save, (ixge_descriptor_t *) d, n);
+      vec_add (d_trace_buffers, to_rx, n);
+    }
+
+  {
+    uword l = vec_len (xm->rx_buffers_to_add);
+
+    if (l < n_descriptors_left)
+      {
+	u32 n_to_alloc = 2 * dq->n_descriptors - l;
+	u32 n_allocated;
+
+	vec_resize (xm->rx_buffers_to_add, n_to_alloc);
+
+	_vec_len (xm->rx_buffers_to_add) = l;
+	n_allocated = vlib_buffer_alloc_from_free_list
+	  (vm, xm->rx_buffers_to_add + l, n_to_alloc,
+	   xm->vlib_buffer_free_list_index);
+	_vec_len (xm->rx_buffers_to_add) += n_allocated;
+
+	/* Handle transient allocation failure */
+	if (PREDICT_FALSE (l + n_allocated <= n_descriptors_left))
+	  {
+	    if (n_allocated == 0)
+	      vlib_error_count (vm, ixge_input_node.index,
+				IXGE_ERROR_rx_alloc_no_physmem, 1);
+	    else
+	      vlib_error_count (vm, ixge_input_node.index,
+				IXGE_ERROR_rx_alloc_fail, 1);
+
+	    n_descriptors_left = l + n_allocated;
+	  }
+	n_descriptors = n_descriptors_left;
+      }
+
+    /* Add buffers from end of vector going backwards. */
+    to_add = vec_end (xm->rx_buffers_to_add) - 1;
+  }
+
+  while (n_descriptors_left > 0)
+    {
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_descriptors_left >= 4 && n_left_to_next >= 2)
+	{
+	  vlib_buffer_t *b0, *b1;
+	  u32 bi0, fi0, len0, l3_offset0, s20, s00, flags0;
+	  u32 bi1, fi1, len1, l3_offset1, s21, s01, flags1;
+	  u8 is_eop0, error0, next0;
+	  u8 is_eop1, error1, next1;
+	  ixge_descriptor_t d0, d1;
+
+	  vlib_prefetch_buffer_with_index (vm, to_rx[2], STORE);
+	  vlib_prefetch_buffer_with_index (vm, to_rx[3], STORE);
+
+	  CLIB_PREFETCH (d + 2, 32, STORE);
+
+	  d0.as_u32x4 = d[0].as_u32x4;
+	  d1.as_u32x4 = d[1].as_u32x4;
+
+	  s20 = d0.rx_from_hw.status[2];
+	  s21 = d1.rx_from_hw.status[2];
+
+	  s00 = d0.rx_from_hw.status[0];
+	  s01 = d1.rx_from_hw.status[0];
+
+	  if (!
+	      ((s20 & s21) & IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE))
+	    goto found_hw_owned_descriptor_x2;
+
+	  bi0 = to_rx[0];
+	  bi1 = to_rx[1];
+
+	  ASSERT (to_add - 1 >= xm->rx_buffers_to_add);
+	  fi0 = to_add[0];
+	  fi1 = to_add[-1];
+
+	  to_rx[0] = fi0;
+	  to_rx[1] = fi1;
+	  to_rx += 2;
+	  to_add -= 2;
+
+	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
+		  vlib_buffer_is_known (vm, bi0));
+	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
+		  vlib_buffer_is_known (vm, bi1));
+	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
+		  vlib_buffer_is_known (vm, fi0));
+	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
+		  vlib_buffer_is_known (vm, fi1));
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+	  /*
+	   * Turn this on if you run into
+	   * "bad monkey" contexts, and you want to know exactly
+	   * which nodes they've visited... See main.c...
+	   */
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
+
+	  CLIB_PREFETCH (b0->data, CLIB_CACHE_LINE_BYTES, LOAD);
+	  CLIB_PREFETCH (b1->data, CLIB_CACHE_LINE_BYTES, LOAD);
+
+	  is_eop0 = (s20 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET) != 0;
+	  is_eop1 = (s21 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET) != 0;
+
+	  ixge_rx_next_and_error_from_status_x2 (xd, s00, s20, s01, s21,
+						 &next0, &error0, &flags0,
+						 &next1, &error1, &flags1);
+
+	  next0 = is_sop ? next0 : next_index_sop;
+	  next1 = is_eop0 ? next1 : next0;
+	  next_index_sop = next1;
+
+	  b0->flags |= flags0 | (!is_eop0 << VLIB_BUFFER_LOG2_NEXT_PRESENT);
+	  b1->flags |= flags1 | (!is_eop1 << VLIB_BUFFER_LOG2_NEXT_PRESENT);
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b1)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+	  b0->error = node->errors[error0];
+	  b1->error = node->errors[error1];
+
+	  len0 = d0.rx_from_hw.n_packet_bytes_this_descriptor;
+	  len1 = d1.rx_from_hw.n_packet_bytes_this_descriptor;
+	  n_bytes += len0 + len1;
+	  n_packets += is_eop0 + is_eop1;
+
+	  /* Give new buffers to hardware. */
+	  d0.rx_to_hw.tail_address =
+	    vlib_get_buffer_data_physical_address (vm, fi0);
+	  d1.rx_to_hw.tail_address =
+	    vlib_get_buffer_data_physical_address (vm, fi1);
+	  d0.rx_to_hw.head_address = d[0].rx_to_hw.tail_address;
+	  d1.rx_to_hw.head_address = d[1].rx_to_hw.tail_address;
+	  d[0].as_u32x4 = d0.as_u32x4;
+	  d[1].as_u32x4 = d1.as_u32x4;
+
+	  d += 2;
+	  n_descriptors_left -= 2;
+
+	  /* Point to either l2 or l3 header depending on next. */
+	  l3_offset0 = (is_sop && (next0 != IXGE_RX_NEXT_ETHERNET_INPUT))
+	    ? IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s00) : 0;
+	  l3_offset1 = (is_eop0 && (next1 != IXGE_RX_NEXT_ETHERNET_INPUT))
+	    ? IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s01) : 0;
+
+	  b0->current_length = len0 - l3_offset0;
+	  b1->current_length = len1 - l3_offset1;
+	  b0->current_data = l3_offset0;
+	  b1->current_data = l3_offset1;
+
+	  b_last->next_buffer = is_sop ? ~0 : bi0;
+	  b0->next_buffer = is_eop0 ? ~0 : bi1;
+	  bi_last = bi1;
+	  b_last = b1;
+
+	  if (CLIB_DEBUG > 0)
+	    {
+	      u32 bi_sop0 = is_sop ? bi0 : bi_sop;
+	      u32 bi_sop1 = is_eop0 ? bi1 : bi_sop0;
+
+	      if (is_eop0)
+		{
+		  u8 *msg = vlib_validate_buffer (vm, bi_sop0,
+						  /* follow_buffer_next */ 1);
+		  ASSERT (!msg);
+		}
+	      if (is_eop1)
+		{
+		  u8 *msg = vlib_validate_buffer (vm, bi_sop1,
+						  /* follow_buffer_next */ 1);
+		  ASSERT (!msg);
+		}
+	    }
+	  if (0)		/* "Dave" version */
+	    {
+	      u32 bi_sop0 = is_sop ? bi0 : bi_sop;
+	      u32 bi_sop1 = is_eop0 ? bi1 : bi_sop0;
+
+	      if (is_eop0)
+		{
+		  to_next[0] = bi_sop0;
+		  to_next++;
+		  n_left_to_next--;
+
+		  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+						   to_next, n_left_to_next,
+						   bi_sop0, next0);
+		}
+	      if (is_eop1)
+		{
+		  to_next[0] = bi_sop1;
+		  to_next++;
+		  n_left_to_next--;
+
+		  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+						   to_next, n_left_to_next,
+						   bi_sop1, next1);
+		}
+	      is_sop = is_eop1;
+	      bi_sop = bi_sop1;
+	    }
+	  if (1)		/* "Eliot" version */
+	    {
+	      /* Speculatively enqueue to cached next. */
+	      u8 saved_is_sop = is_sop;
+	      u32 bi_sop_save = bi_sop;
+
+	      bi_sop = saved_is_sop ? bi0 : bi_sop;
+	      to_next[0] = bi_sop;
+	      to_next += is_eop0;
+	      n_left_to_next -= is_eop0;
+
+	      bi_sop = is_eop0 ? bi1 : bi_sop;
+	      to_next[0] = bi_sop;
+	      to_next += is_eop1;
+	      n_left_to_next -= is_eop1;
+
+	      is_sop = is_eop1;
+
+	      if (PREDICT_FALSE
+		  (!(next0 == next_index && next1 == next_index)))
+		{
+		  /* Undo speculation. */
+		  to_next -= is_eop0 + is_eop1;
+		  n_left_to_next += is_eop0 + is_eop1;
+
+		  /* Re-do both descriptors being careful about where we enqueue. */
+		  bi_sop = saved_is_sop ? bi0 : bi_sop_save;
+		  if (is_eop0)
+		    {
+		      if (next0 != next_index)
+			vlib_set_next_frame_buffer (vm, node, next0, bi_sop);
+		      else
+			{
+			  to_next[0] = bi_sop;
+			  to_next += 1;
+			  n_left_to_next -= 1;
+			}
+		    }
+
+		  bi_sop = is_eop0 ? bi1 : bi_sop;
+		  if (is_eop1)
+		    {
+		      if (next1 != next_index)
+			vlib_set_next_frame_buffer (vm, node, next1, bi_sop);
+		      else
+			{
+			  to_next[0] = bi_sop;
+			  to_next += 1;
+			  n_left_to_next -= 1;
+			}
+		    }
+
+		  /* Switch cached next index when next for both packets is the same. */
+		  if (is_eop0 && is_eop1 && next0 == next1)
+		    {
+		      vlib_put_next_frame (vm, node, next_index,
+					   n_left_to_next);
+		      next_index = next0;
+		      vlib_get_next_frame (vm, node, next_index,
+					   to_next, n_left_to_next);
+		    }
+		}
+	    }
+	}
+
+      /* Bail out of dual loop and proceed with single loop. */
+    found_hw_owned_descriptor_x2:
+
+      while (n_descriptors_left > 0 && n_left_to_next > 0)
+	{
+	  vlib_buffer_t *b0;
+	  u32 bi0, fi0, len0, l3_offset0, s20, s00, flags0;
+	  u8 is_eop0, error0, next0;
+	  ixge_descriptor_t d0;
+
+	  d0.as_u32x4 = d[0].as_u32x4;
+
+	  s20 = d0.rx_from_hw.status[2];
+	  s00 = d0.rx_from_hw.status[0];
+
+	  if (!(s20 & IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE))
+	    goto found_hw_owned_descriptor_x1;
+
+	  bi0 = to_rx[0];
+	  ASSERT (to_add >= xm->rx_buffers_to_add);
+	  fi0 = to_add[0];
+
+	  to_rx[0] = fi0;
+	  to_rx += 1;
+	  to_add -= 1;
+
+	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
+		  vlib_buffer_is_known (vm, bi0));
+	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
+		  vlib_buffer_is_known (vm, fi0));
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  /*
+	   * Turn this on if you run into
+	   * "bad monkey" contexts, and you want to know exactly
+	   * which nodes they've visited...
+	   */
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+
+	  is_eop0 = (s20 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET) != 0;
+	  ixge_rx_next_and_error_from_status_x1
+	    (xd, s00, s20, &next0, &error0, &flags0);
+
+	  next0 = is_sop ? next0 : next_index_sop;
+	  next_index_sop = next0;
+
+	  b0->flags |= flags0 | (!is_eop0 << VLIB_BUFFER_LOG2_NEXT_PRESENT);
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+	  b0->error = node->errors[error0];
+
+	  len0 = d0.rx_from_hw.n_packet_bytes_this_descriptor;
+	  n_bytes += len0;
+	  n_packets += is_eop0;
+
+	  /* Give new buffer to hardware. */
+	  d0.rx_to_hw.tail_address =
+	    vlib_get_buffer_data_physical_address (vm, fi0);
+	  d0.rx_to_hw.head_address = d0.rx_to_hw.tail_address;
+	  d[0].as_u32x4 = d0.as_u32x4;
+
+	  d += 1;
+	  n_descriptors_left -= 1;
+
+	  /* Point to either l2 or l3 header depending on next. */
+	  l3_offset0 = (is_sop && (next0 != IXGE_RX_NEXT_ETHERNET_INPUT))
+	    ? IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s00) : 0;
+	  b0->current_length = len0 - l3_offset0;
+	  b0->current_data = l3_offset0;
+
+	  b_last->next_buffer = is_sop ? ~0 : bi0;
+	  bi_last = bi0;
+	  b_last = b0;
+
+	  bi_sop = is_sop ? bi0 : bi_sop;
+
+	  if (CLIB_DEBUG > 0 && is_eop0)
+	    {
+	      u8 *msg =
+		vlib_validate_buffer (vm, bi_sop, /* follow_buffer_next */ 1);
+	      ASSERT (!msg);
+	    }
+
+	  if (0)		/* "Dave" version */
+	    {
+	      if (is_eop0)
+		{
+		  to_next[0] = bi_sop;
+		  to_next++;
+		  n_left_to_next--;
+
+		  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+						   to_next, n_left_to_next,
+						   bi_sop, next0);
+		}
+	    }
+	  if (1)		/* "Eliot" version */
+	    {
+	      if (PREDICT_TRUE (next0 == next_index))
+		{
+		  to_next[0] = bi_sop;
+		  to_next += is_eop0;
+		  n_left_to_next -= is_eop0;
+		}
+	      else
+		{
+		  if (next0 != next_index && is_eop0)
+		    vlib_set_next_frame_buffer (vm, node, next0, bi_sop);
+
+		  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+		  next_index = next0;
+		  vlib_get_next_frame (vm, node, next_index,
+				       to_next, n_left_to_next);
+		}
+	    }
+	  is_sop = is_eop0;
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+found_hw_owned_descriptor_x1:
+  if (n_descriptors_left > 0)
+    vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+
+  _vec_len (xm->rx_buffers_to_add) = (to_add + 1) - xm->rx_buffers_to_add;
+
+  {
+    u32 n_done = n_descriptors - n_descriptors_left;
+
+    if (n_trace > 0 && n_done > 0)
+      {
+	u32 n = clib_min (n_trace, n_done);
+	ixge_rx_trace (xm, xd, dq,
+		       d_trace_save,
+		       d_trace_buffers,
+		       &dq->descriptors[start_descriptor_index], n);
+	vlib_set_trace_count (vm, node, n_trace - n);
+      }
+    if (d_trace_save)
+      {
+	_vec_len (d_trace_save) = 0;
+	_vec_len (d_trace_buffers) = 0;
+      }
+
+    /* Don't keep a reference to b_last if we don't have to.
+       Otherwise we can over-write a next_buffer pointer after already haven
+       enqueued a packet. */
+    if (is_sop)
+      {
+	b_last->next_buffer = ~0;
+	bi_last = ~0;
+      }
+
+    dq->rx.n_descriptors_done_this_call = n_done;
+    dq->rx.n_descriptors_done_total += n_done;
+    dq->rx.is_start_of_packet = is_sop;
+    dq->rx.saved_start_of_packet_buffer_index = bi_sop;
+    dq->rx.saved_last_buffer_index = bi_last;
+    dq->rx.saved_start_of_packet_next_index = next_index_sop;
+    dq->rx.next_index = next_index;
+    dq->rx.n_bytes += n_bytes;
+
+    return n_packets;
+  }
+}
+
+static uword
+ixge_rx_queue (ixge_main_t * xm,
+	       ixge_device_t * xd,
+	       vlib_node_runtime_t * node, u32 queue_index)
+{
+  ixge_dma_queue_t *dq =
+    vec_elt_at_index (xd->dma_queues[VLIB_RX], queue_index);
+  ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_RX, dq->queue_index);
+  uword n_packets = 0;
+  u32 hw_head_index, sw_head_index;
+
+  /* One time initialization. */
+  if (!dq->rx.node)
+    {
+      dq->rx.node = node;
+      dq->rx.is_start_of_packet = 1;
+      dq->rx.saved_start_of_packet_buffer_index = ~0;
+      dq->rx.saved_last_buffer_index = ~0;
+    }
+
+  dq->rx.next_index = node->cached_next_index;
+
+  dq->rx.n_descriptors_done_total = 0;
+  dq->rx.n_descriptors_done_this_call = 0;
+  dq->rx.n_bytes = 0;
+
+  /* Fetch head from hardware and compare to where we think we are. */
+  hw_head_index = dr->head_index;
+  sw_head_index = dq->head_index;
+
+  if (hw_head_index == sw_head_index)
+    goto done;
+
+  if (hw_head_index < sw_head_index)
+    {
+      u32 n_tried = dq->n_descriptors - sw_head_index;
+      n_packets += ixge_rx_queue_no_wrap (xm, xd, dq, sw_head_index, n_tried);
+      sw_head_index =
+	ixge_ring_add (dq, sw_head_index,
+		       dq->rx.n_descriptors_done_this_call);
+
+      if (dq->rx.n_descriptors_done_this_call != n_tried)
+	goto done;
+    }
+  if (hw_head_index >= sw_head_index)
+    {
+      u32 n_tried = hw_head_index - sw_head_index;
+      n_packets += ixge_rx_queue_no_wrap (xm, xd, dq, sw_head_index, n_tried);
+      sw_head_index =
+	ixge_ring_add (dq, sw_head_index,
+		       dq->rx.n_descriptors_done_this_call);
+    }
+
+done:
+  dq->head_index = sw_head_index;
+  dq->tail_index =
+    ixge_ring_add (dq, dq->tail_index, dq->rx.n_descriptors_done_total);
+
+  /* Give tail back to hardware. */
+  CLIB_MEMORY_BARRIER ();
+
+  dr->tail_index = dq->tail_index;
+
+  vlib_increment_combined_counter (vnet_main.
+				   interface_main.combined_sw_if_counters +
+				   VNET_INTERFACE_COUNTER_RX,
+				   0 /* cpu_index */ ,
+				   xd->vlib_sw_if_index, n_packets,
+				   dq->rx.n_bytes);
+
+  return n_packets;
+}
+
+static void
+ixge_interrupt (ixge_main_t * xm, ixge_device_t * xd, u32 i)
+{
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_regs_t *r = xd->regs;
+
+  if (i != 20)
+    {
+      ELOG_TYPE_DECLARE (e) =
+      {
+	.function = (char *) __FUNCTION__,.format =
+	  "ixge %d, %s",.format_args = "i1t1",.n_enum_strings =
+	  16,.enum_strings =
+	{
+      "flow director",
+	    "rx miss",
+	    "pci exception",
+	    "mailbox",
+	    "link status change",
+	    "linksec key exchange",
+	    "manageability event",
+	    "reserved23",
+	    "sdp0",
+	    "sdp1",
+	    "sdp2",
+	    "sdp3",
+	    "ecc", "descriptor handler error", "tcp timer", "other",},};
+      struct
+      {
+	u8 instance;
+	u8 index;
+      } *ed;
+      ed = ELOG_DATA (&vm->elog_main, e);
+      ed->instance = xd->device_index;
+      ed->index = i - 16;
+    }
+  else
+    {
+      u32 v = r->xge_mac.link_status;
+      uword is_up = (v & (1 << 30)) != 0;
+
+      ELOG_TYPE_DECLARE (e) =
+      {
+      .function = (char *) __FUNCTION__,.format =
+	  "ixge %d, link status change 0x%x",.format_args = "i4i4",};
+      struct
+      {
+	u32 instance, link_status;
+      } *ed;
+      ed = ELOG_DATA (&vm->elog_main, e);
+      ed->instance = xd->device_index;
+      ed->link_status = v;
+      xd->link_status_at_last_link_change = v;
+
+      vlib_process_signal_event (vm, ixge_process_node.index,
+				 EVENT_SET_FLAGS,
+				 ((is_up << 31) | xd->vlib_hw_if_index));
+    }
+}
+
+always_inline u32
+clean_block (u32 * b, u32 * t, u32 n_left)
+{
+  u32 *t0 = t;
+
+  while (n_left >= 4)
+    {
+      u32 bi0, bi1, bi2, bi3;
+
+      t[0] = bi0 = b[0];
+      b[0] = 0;
+      t += bi0 != 0;
+
+      t[0] = bi1 = b[1];
+      b[1] = 0;
+      t += bi1 != 0;
+
+      t[0] = bi2 = b[2];
+      b[2] = 0;
+      t += bi2 != 0;
+
+      t[0] = bi3 = b[3];
+      b[3] = 0;
+      t += bi3 != 0;
+
+      b += 4;
+      n_left -= 4;
+    }
+
+  while (n_left > 0)
+    {
+      u32 bi0;
+
+      t[0] = bi0 = b[0];
+      b[0] = 0;
+      t += bi0 != 0;
+      b += 1;
+      n_left -= 1;
+    }
+
+  return t - t0;
+}
+
+static void
+ixge_tx_queue (ixge_main_t * xm, ixge_device_t * xd, u32 queue_index)
+{
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_dma_queue_t *dq =
+    vec_elt_at_index (xd->dma_queues[VLIB_TX], queue_index);
+  u32 n_clean, *b, *t, *t0;
+  i32 n_hw_owned_descriptors;
+  i32 first_to_clean, last_to_clean;
+  u64 hwbp_race = 0;
+
+  /* Handle case where head write back pointer update
+   * arrives after the interrupt during high PCI bus loads.
+   */
+  while ((dq->head_index == dq->tx.head_index_write_back[0]) &&
+	 dq->tx.n_buffers_on_ring && (dq->head_index != dq->tail_index))
+    {
+      hwbp_race++;
+      if (IXGE_HWBP_RACE_ELOG && (hwbp_race == 1))
+	{
+	  ELOG_TYPE_DECLARE (e) =
+	  {
+	  .function = (char *) __FUNCTION__,.format =
+	      "ixge %d tx head index race: head %4d, tail %4d, buffs %4d",.format_args
+	      = "i4i4i4i4",};
+	  struct
+	  {
+	    u32 instance, head_index, tail_index, n_buffers_on_ring;
+	  } *ed;
+	  ed = ELOG_DATA (&vm->elog_main, e);
+	  ed->instance = xd->device_index;
+	  ed->head_index = dq->head_index;
+	  ed->tail_index = dq->tail_index;
+	  ed->n_buffers_on_ring = dq->tx.n_buffers_on_ring;
+	}
+    }
+
+  dq->head_index = dq->tx.head_index_write_back[0];
+  n_hw_owned_descriptors = ixge_ring_sub (dq, dq->head_index, dq->tail_index);
+  ASSERT (dq->tx.n_buffers_on_ring >= n_hw_owned_descriptors);
+  n_clean = dq->tx.n_buffers_on_ring - n_hw_owned_descriptors;
+
+  if (IXGE_HWBP_RACE_ELOG && hwbp_race)
+    {
+      ELOG_TYPE_DECLARE (e) =
+      {
+      .function = (char *) __FUNCTION__,.format =
+	  "ixge %d tx head index race: head %4d, hw_owned %4d, n_clean %4d, retries %d",.format_args
+	  = "i4i4i4i4i4",};
+      struct
+      {
+	u32 instance, head_index, n_hw_owned_descriptors, n_clean, retries;
+      } *ed;
+      ed = ELOG_DATA (&vm->elog_main, e);
+      ed->instance = xd->device_index;
+      ed->head_index = dq->head_index;
+      ed->n_hw_owned_descriptors = n_hw_owned_descriptors;
+      ed->n_clean = n_clean;
+      ed->retries = hwbp_race;
+    }
+
+  /*
+   * This function used to wait until hardware owned zero descriptors.
+   * At high PPS rates, that doesn't happen until the TX ring is
+   * completely full of descriptors which need to be cleaned up.
+   * That, in turn, causes TX ring-full drops and/or long RX service
+   * interruptions.
+   */
+  if (n_clean == 0)
+    return;
+
+  /* Clean the n_clean descriptors prior to the reported hardware head */
+  last_to_clean = dq->head_index - 1;
+  last_to_clean = (last_to_clean < 0) ? last_to_clean + dq->n_descriptors :
+    last_to_clean;
+
+  first_to_clean = (last_to_clean) - (n_clean - 1);
+  first_to_clean = (first_to_clean < 0) ? first_to_clean + dq->n_descriptors :
+    first_to_clean;
+
+  vec_resize (xm->tx_buffers_pending_free, dq->n_descriptors - 1);
+  t0 = t = xm->tx_buffers_pending_free;
+  b = dq->descriptor_buffer_indices + first_to_clean;
+
+  /* Wrap case: clean from first to end, then start to last */
+  if (first_to_clean > last_to_clean)
+    {
+      t += clean_block (b, t, (dq->n_descriptors - 1) - first_to_clean);
+      first_to_clean = 0;
+      b = dq->descriptor_buffer_indices;
+    }
+
+  /* Typical case: clean from first to last */
+  if (first_to_clean <= last_to_clean)
+    t += clean_block (b, t, (last_to_clean - first_to_clean) + 1);
+
+  if (t > t0)
+    {
+      u32 n = t - t0;
+      vlib_buffer_free_no_next (vm, t0, n);
+      ASSERT (dq->tx.n_buffers_on_ring >= n);
+      dq->tx.n_buffers_on_ring -= n;
+      _vec_len (xm->tx_buffers_pending_free) = 0;
+    }
+}
+
+/* RX queue interrupts 0 thru 7; TX 8 thru 15. */
+always_inline uword
+ixge_interrupt_is_rx_queue (uword i)
+{
+  return i < 8;
+}
+
+always_inline uword
+ixge_interrupt_is_tx_queue (uword i)
+{
+  return i >= 8 && i < 16;
+}
+
+always_inline uword
+ixge_tx_queue_to_interrupt (uword i)
+{
+  return 8 + i;
+}
+
+always_inline uword
+ixge_rx_queue_to_interrupt (uword i)
+{
+  return 0 + i;
+}
+
+always_inline uword
+ixge_interrupt_rx_queue (uword i)
+{
+  ASSERT (ixge_interrupt_is_rx_queue (i));
+  return i - 0;
+}
+
+always_inline uword
+ixge_interrupt_tx_queue (uword i)
+{
+  ASSERT (ixge_interrupt_is_tx_queue (i));
+  return i - 8;
+}
+
+static uword
+ixge_device_input (ixge_main_t * xm,
+		   ixge_device_t * xd, vlib_node_runtime_t * node)
+{
+  ixge_regs_t *r = xd->regs;
+  u32 i, s;
+  uword n_rx_packets = 0;
+
+  s = r->interrupt.status_write_1_to_set;
+  if (s)
+    r->interrupt.status_write_1_to_clear = s;
+
+  /* *INDENT-OFF* */
+  foreach_set_bit (i, s, ({
+    if (ixge_interrupt_is_rx_queue (i))
+      n_rx_packets += ixge_rx_queue (xm, xd, node, ixge_interrupt_rx_queue (i));
+
+    else if (ixge_interrupt_is_tx_queue (i))
+      ixge_tx_queue (xm, xd, ixge_interrupt_tx_queue (i));
+
+    else
+      ixge_interrupt (xm, xd, i);
+  }));
+  /* *INDENT-ON* */
+
+  return n_rx_packets;
+}
+
+static uword
+ixge_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f)
+{
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd;
+  uword n_rx_packets = 0;
+
+  if (node->state == VLIB_NODE_STATE_INTERRUPT)
+    {
+      uword i;
+
+      /* Loop over devices with interrupts. */
+      /* *INDENT-OFF* */
+      foreach_set_bit (i, node->runtime_data[0], ({
+	xd = vec_elt_at_index (xm->devices, i);
+	n_rx_packets += ixge_device_input (xm, xd, node);
+
+	/* Re-enable interrupts since we're going to stay in interrupt mode. */
+	if (! (node->flags & VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
+	  xd->regs->interrupt.enable_write_1_to_set = ~0;
+      }));
+      /* *INDENT-ON* */
+
+      /* Clear mask of devices with pending interrupts. */
+      node->runtime_data[0] = 0;
+    }
+  else
+    {
+      /* Poll all devices for input/interrupts. */
+      vec_foreach (xd, xm->devices)
+      {
+	n_rx_packets += ixge_device_input (xm, xd, node);
+
+	/* Re-enable interrupts when switching out of polling mode. */
+	if (node->flags &
+	    VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)
+	  xd->regs->interrupt.enable_write_1_to_set = ~0;
+      }
+    }
+
+  return n_rx_packets;
+}
+
+static char *ixge_error_strings[] = {
+#define _(n,s) s,
+  foreach_ixge_error
+#undef _
+};
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (ixge_input_node, static) = {
+  .function = ixge_input,
+  .type = VLIB_NODE_TYPE_INPUT,
+  .name = "ixge-input",
+
+  /* Will be enabled if/when hardware is detected. */
+  .state = VLIB_NODE_STATE_DISABLED,
+
+  .format_buffer = format_ethernet_header_with_length,
+  .format_trace = format_ixge_rx_dma_trace,
+
+  .n_errors = IXGE_N_ERROR,
+  .error_strings = ixge_error_strings,
+
+  .n_next_nodes = IXGE_RX_N_NEXT,
+  .next_nodes = {
+    [IXGE_RX_NEXT_DROP] = "error-drop",
+    [IXGE_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
+    [IXGE_RX_NEXT_IP4_INPUT] = "ip4-input",
+    [IXGE_RX_NEXT_IP6_INPUT] = "ip6-input",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH_CLONE (ixge_input)
+CLIB_MULTIARCH_SELECT_FN (ixge_input)
+/* *INDENT-ON* */
+
+static u8 *
+format_ixge_device_name (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, i);
+  return format (s, "TenGigabitEthernet%U",
+		 format_vlib_pci_handle, &xd->pci_device.bus_address);
+}
+
+#define IXGE_COUNTER_IS_64_BIT (1 << 0)
+#define IXGE_COUNTER_NOT_CLEAR_ON_READ (1 << 1)
+
+static u8 ixge_counter_flags[] = {
+#define _(a,f) 0,
+#define _64(a,f) IXGE_COUNTER_IS_64_BIT,
+  foreach_ixge_counter
+#undef _
+#undef _64
+};
+
+static void
+ixge_update_counters (ixge_device_t * xd)
+{
+  /* Byte offset for counter registers. */
+  static u32 reg_offsets[] = {
+#define _(a,f) (a) / sizeof (u32),
+#define _64(a,f) _(a,f)
+    foreach_ixge_counter
+#undef _
+#undef _64
+  };
+  volatile u32 *r = (volatile u32 *) xd->regs;
+  int i;
+
+  for (i = 0; i < ARRAY_LEN (xd->counters); i++)
+    {
+      u32 o = reg_offsets[i];
+      xd->counters[i] += r[o];
+      if (ixge_counter_flags[i] & IXGE_COUNTER_NOT_CLEAR_ON_READ)
+	r[o] = 0;
+      if (ixge_counter_flags[i] & IXGE_COUNTER_IS_64_BIT)
+	xd->counters[i] += (u64) r[o + 1] << (u64) 32;
+    }
+}
+
+static u8 *
+format_ixge_device_id (u8 * s, va_list * args)
+{
+  u32 device_id = va_arg (*args, u32);
+  char *t = 0;
+  switch (device_id)
+    {
+#define _(f,n) case n: t = #f; break;
+      foreach_ixge_pci_device_id;
+#undef _
+    default:
+      t = 0;
+      break;
+    }
+  if (t == 0)
+    s = format (s, "unknown 0x%x", device_id);
+  else
+    s = format (s, "%s", t);
+  return s;
+}
+
+static u8 *
+format_ixge_link_status (u8 * s, va_list * args)
+{
+  ixge_device_t *xd = va_arg (*args, ixge_device_t *);
+  u32 v = xd->link_status_at_last_link_change;
+
+  s = format (s, "%s", (v & (1 << 30)) ? "up" : "down");
+
+  {
+    char *modes[] = {
+      "1g", "10g parallel", "10g serial", "autoneg",
+    };
+    char *speeds[] = {
+      "unknown", "100m", "1g", "10g",
+    };
+    s = format (s, ", mode %s, speed %s",
+		modes[(v >> 26) & 3], speeds[(v >> 28) & 3]);
+  }
+
+  return s;
+}
+
+static u8 *
+format_ixge_device (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  CLIB_UNUSED (int verbose) = va_arg (*args, int);
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, dev_instance);
+  ixge_phy_t *phy = xd->phys + xd->phy_index;
+  uword indent = format_get_indent (s);
+
+  ixge_update_counters (xd);
+  xd->link_status_at_last_link_change = xd->regs->xge_mac.link_status;
+
+  s = format (s, "Intel 8259X: id %U\n%Ulink %U",
+	      format_ixge_device_id, xd->device_id,
+	      format_white_space, indent + 2, format_ixge_link_status, xd);
+
+  {
+
+    s = format (s, "\n%UPCIe %U", format_white_space, indent + 2,
+		format_vlib_pci_link_speed, &xd->pci_device);
+  }
+
+  s = format (s, "\n%U", format_white_space, indent + 2);
+  if (phy->mdio_address != ~0)
+    s = format (s, "PHY address %d, id 0x%x", phy->mdio_address, phy->id);
+  else if (xd->sfp_eeprom.id == SFP_ID_sfp)
+    s = format (s, "SFP %U", format_sfp_eeprom, &xd->sfp_eeprom);
+  else
+    s = format (s, "PHY not found");
+
+  /* FIXME */
+  {
+    ixge_dma_queue_t *dq = vec_elt_at_index (xd->dma_queues[VLIB_RX], 0);
+    ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_RX, 0);
+    u32 hw_head_index = dr->head_index;
+    u32 sw_head_index = dq->head_index;
+    u32 nitems;
+
+    nitems = ixge_ring_sub (dq, hw_head_index, sw_head_index);
+    s = format (s, "\n%U%d unprocessed, %d total buffers on rx queue 0 ring",
+		format_white_space, indent + 2, nitems, dq->n_descriptors);
+
+    s = format (s, "\n%U%d buffers in driver rx cache",
+		format_white_space, indent + 2,
+		vec_len (xm->rx_buffers_to_add));
+
+    s = format (s, "\n%U%d buffers on tx queue 0 ring",
+		format_white_space, indent + 2,
+		xd->dma_queues[VLIB_TX][0].tx.n_buffers_on_ring);
+  }
+  {
+    u32 i;
+    u64 v;
+    static char *names[] = {
+#define _(a,f) #f,
+#define _64(a,f) _(a,f)
+      foreach_ixge_counter
+#undef _
+#undef _64
+    };
+
+    for (i = 0; i < ARRAY_LEN (names); i++)
+      {
+	v = xd->counters[i] - xd->counters_last_clear[i];
+	if (v != 0)
+	  s = format (s, "\n%U%-40U%16Ld",
+		      format_white_space, indent + 2,
+		      format_c_identifier, names[i], v);
+      }
+  }
+
+  return s;
+}
+
+static void
+ixge_clear_hw_interface_counters (u32 instance)
+{
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, instance);
+  ixge_update_counters (xd);
+  memcpy (xd->counters_last_clear, xd->counters, sizeof (xd->counters));
+}
+
+/*
+ * Dynamically redirect all pkts from a specific interface
+ * to the specified node
+ */
+static void
+ixge_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
+			      u32 node_index)
+{
+  ixge_main_t *xm = &ixge_main;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  ixge_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
+
+  /* Shut off redirection */
+  if (node_index == ~0)
+    {
+      xd->per_interface_next_index = node_index;
+      return;
+    }
+
+  xd->per_interface_next_index =
+    vlib_node_add_next (xm->vlib_main, ixge_input_node.index, node_index);
+}
+
+
+/* *INDENT-OFF* */
+VNET_DEVICE_CLASS (ixge_device_class) = {
+  .name = "ixge",
+  .tx_function = ixge_interface_tx,
+  .format_device_name = format_ixge_device_name,
+  .format_device = format_ixge_device,
+  .format_tx_trace = format_ixge_tx_dma_trace,
+  .clear_counters = ixge_clear_hw_interface_counters,
+  .admin_up_down_function = ixge_interface_admin_up_down,
+  .rx_redirect_to_node = ixge_set_interface_next_node,
+  .flatten_output_chains = 1,
+};
+/* *INDENT-ON* */
+
+#define IXGE_N_BYTES_IN_RX_BUFFER  (2048)	// DAW-HACK: Set Rx buffer size so all packets < ETH_MTU_SIZE fit in the buffer (i.e. sop & eop for all descriptors).
+
+static clib_error_t *
+ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index)
+{
+  ixge_main_t *xm = &ixge_main;
+  vlib_main_t *vm = xm->vlib_main;
+  ixge_dma_queue_t *dq;
+  clib_error_t *error = 0;
+
+  vec_validate (xd->dma_queues[rt], queue_index);
+  dq = vec_elt_at_index (xd->dma_queues[rt], queue_index);
+
+  if (!xm->n_descriptors_per_cache_line)
+    xm->n_descriptors_per_cache_line =
+      CLIB_CACHE_LINE_BYTES / sizeof (dq->descriptors[0]);
+
+  if (!xm->n_bytes_in_rx_buffer)
+    xm->n_bytes_in_rx_buffer = IXGE_N_BYTES_IN_RX_BUFFER;
+  xm->n_bytes_in_rx_buffer = round_pow2 (xm->n_bytes_in_rx_buffer, 1024);
+  if (!xm->vlib_buffer_free_list_index)
+    {
+      xm->vlib_buffer_free_list_index =
+	vlib_buffer_get_or_create_free_list (vm, xm->n_bytes_in_rx_buffer,
+					     "ixge rx");
+      ASSERT (xm->vlib_buffer_free_list_index != 0);
+    }
+
+  if (!xm->n_descriptors[rt])
+    xm->n_descriptors[rt] = 4 * VLIB_FRAME_SIZE;
+
+  dq->queue_index = queue_index;
+  dq->n_descriptors =
+    round_pow2 (xm->n_descriptors[rt], xm->n_descriptors_per_cache_line);
+  dq->head_index = dq->tail_index = 0;
+
+  dq->descriptors = vlib_physmem_alloc_aligned (vm, &error,
+						dq->n_descriptors *
+						sizeof (dq->descriptors[0]),
+						128 /* per chip spec */ );
+  if (error)
+    return error;
+
+  memset (dq->descriptors, 0,
+	  dq->n_descriptors * sizeof (dq->descriptors[0]));
+  vec_resize (dq->descriptor_buffer_indices, dq->n_descriptors);
+
+  if (rt == VLIB_RX)
+    {
+      u32 n_alloc, i;
+
+      n_alloc = vlib_buffer_alloc_from_free_list
+	(vm, dq->descriptor_buffer_indices,
+	 vec_len (dq->descriptor_buffer_indices),
+	 xm->vlib_buffer_free_list_index);
+      ASSERT (n_alloc == vec_len (dq->descriptor_buffer_indices));
+      for (i = 0; i < n_alloc; i++)
+	{
+	  vlib_buffer_t *b =
+	    vlib_get_buffer (vm, dq->descriptor_buffer_indices[i]);
+	  dq->descriptors[i].rx_to_hw.tail_address =
+	    vlib_physmem_virtual_to_physical (vm, b->data);
+	}
+    }
+  else
+    {
+      u32 i;
+
+      dq->tx.head_index_write_back =
+	vlib_physmem_alloc (vm, &error, CLIB_CACHE_LINE_BYTES);
+
+      for (i = 0; i < dq->n_descriptors; i++)
+	dq->descriptors[i].tx = xm->tx_descriptor_template;
+
+      vec_validate (xm->tx_buffers_pending_free, dq->n_descriptors - 1);
+    }
+
+  {
+    ixge_dma_regs_t *dr = get_dma_regs (xd, rt, queue_index);
+    u64 a;
+
+    a = vlib_physmem_virtual_to_physical (vm, dq->descriptors);
+    dr->descriptor_address[0] = a & 0xFFFFFFFF;
+    dr->descriptor_address[1] = a >> (u64) 32;
+    dr->n_descriptor_bytes = dq->n_descriptors * sizeof (dq->descriptors[0]);
+    dq->head_index = dq->tail_index = 0;
+
+    if (rt == VLIB_RX)
+      {
+	ASSERT ((xm->n_bytes_in_rx_buffer / 1024) < 32);
+	dr->rx_split_control =
+	  ( /* buffer size */ ((xm->n_bytes_in_rx_buffer / 1024) << 0)
+	   | (			/* lo free descriptor threshold (units of 64 descriptors) */
+	       (1 << 22)) | (	/* descriptor type: advanced one buffer */
+			      (1 << 25)) | (	/* drop if no descriptors available */
+					     (1 << 28)));
+
+	/* Give hardware all but last 16 cache lines' worth of descriptors. */
+	dq->tail_index = dq->n_descriptors -
+	  16 * xm->n_descriptors_per_cache_line;
+      }
+    else
+      {
+	/* Make sure its initialized before hardware can get to it. */
+	dq->tx.head_index_write_back[0] = dq->head_index;
+
+	a =
+	  vlib_physmem_virtual_to_physical (vm, dq->tx.head_index_write_back);
+	dr->tx.head_index_write_back_address[0] = /* enable bit */ 1 | a;
+	dr->tx.head_index_write_back_address[1] = (u64) a >> (u64) 32;
+      }
+
+    /* DMA on 82599 does not work with [13] rx data write relaxed ordering
+       and [12] undocumented set. */
+    if (rt == VLIB_RX)
+      dr->dca_control &= ~((1 << 13) | (1 << 12));
+
+    CLIB_MEMORY_BARRIER ();
+
+    if (rt == VLIB_TX)
+      {
+	xd->regs->tx_dma_control |= (1 << 0);
+	dr->control |= ((32 << 0)	/* prefetch threshold */
+			| (64 << 8)	/* host threshold */
+			| (0 << 16) /* writeback threshold */ );
+      }
+
+    /* Enable this queue and wait for hardware to initialize
+       before adding to tail. */
+    if (rt == VLIB_TX)
+      {
+	dr->control |= 1 << 25;
+	while (!(dr->control & (1 << 25)))
+	  ;
+      }
+
+    /* Set head/tail indices and enable DMA. */
+    dr->head_index = dq->head_index;
+    dr->tail_index = dq->tail_index;
+  }
+
+  return error;
+}
+
+static u32
+ixge_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
+{
+  ixge_device_t *xd;
+  ixge_regs_t *r;
+  u32 old;
+  ixge_main_t *xm = &ixge_main;
+
+  xd = vec_elt_at_index (xm->devices, hw->dev_instance);
+  r = xd->regs;
+
+  old = r->filter_control;
+
+  if (flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL)
+    r->filter_control = old | (1 << 9) /* unicast promiscuous */ ;
+  else
+    r->filter_control = old & ~(1 << 9);
+
+  return old;
+}
+
+static void
+ixge_device_init (ixge_main_t * xm)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ixge_device_t *xd;
+
+  /* Reset chip(s). */
+  vec_foreach (xd, xm->devices)
+  {
+    ixge_regs_t *r = xd->regs;
+    const u32 reset_bit = (1 << 26) | (1 << 3);
+
+    r->control |= reset_bit;
+
+    /* No need to suspend.  Timed to take ~1e-6 secs */
+    while (r->control & reset_bit)
+      ;
+
+    /* Software loaded. */
+    r->extended_control |= (1 << 28);
+
+    ixge_phy_init (xd);
+
+    /* Register ethernet interface. */
+    {
+      u8 addr8[6];
+      u32 i, addr32[2];
+      clib_error_t *error;
+
+      addr32[0] = r->rx_ethernet_address0[0][0];
+      addr32[1] = r->rx_ethernet_address0[0][1];
+      for (i = 0; i < 6; i++)
+	addr8[i] = addr32[i / 4] >> ((i % 4) * 8);
+
+      error = ethernet_register_interface
+	(vnm, ixge_device_class.index, xd->device_index,
+	 /* ethernet address */ addr8,
+	 &xd->vlib_hw_if_index, ixge_flag_change);
+      if (error)
+	clib_error_report (error);
+    }
+
+    {
+      vnet_sw_interface_t *sw =
+	vnet_get_hw_sw_interface (vnm, xd->vlib_hw_if_index);
+      xd->vlib_sw_if_index = sw->sw_if_index;
+    }
+
+    ixge_dma_init (xd, VLIB_RX, /* queue_index */ 0);
+
+    xm->n_descriptors[VLIB_TX] = 20 * VLIB_FRAME_SIZE;
+
+    ixge_dma_init (xd, VLIB_TX, /* queue_index */ 0);
+
+    /* RX/TX queue 0 gets mapped to interrupt bits 0 & 8. */
+    r->interrupt.queue_mapping[0] = (( /* valid bit */ (1 << 7) |
+				      ixge_rx_queue_to_interrupt (0)) << 0);
+
+    r->interrupt.queue_mapping[0] |= (( /* valid bit */ (1 << 7) |
+				       ixge_tx_queue_to_interrupt (0)) << 8);
+
+    /* No use in getting too many interrupts.
+       Limit them to one every 3/4 ring size at line rate
+       min sized packets.
+       No need for this since kernel/vlib main loop provides adequate interrupt
+       limiting scheme. */
+    if (0)
+      {
+	f64 line_rate_max_pps =
+	  10e9 / (8 * (64 + /* interframe padding */ 20));
+	ixge_throttle_queue_interrupt (r, 0,
+				       .75 * xm->n_descriptors[VLIB_RX] /
+				       line_rate_max_pps);
+      }
+
+    /* Accept all multicast and broadcast packets. Should really add them
+       to the dst_ethernet_address register array. */
+    r->filter_control |= (1 << 10) | (1 << 8);
+
+    /* Enable frames up to size in mac frame size register. */
+    r->xge_mac.control |= 1 << 2;
+    r->xge_mac.rx_max_frame_size = (9216 + 14) << 16;
+
+    /* Enable all interrupts. */
+    if (!IXGE_ALWAYS_POLL)
+      r->interrupt.enable_write_1_to_set = ~0;
+  }
+}
+
+static uword
+ixge_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ixge_main_t *xm = &ixge_main;
+  ixge_device_t *xd;
+  uword event_type, *event_data = 0;
+  f64 timeout, link_debounce_deadline;
+
+  ixge_device_init (xm);
+
+  /* Clear all counters. */
+  vec_foreach (xd, xm->devices)
+  {
+    ixge_update_counters (xd);
+    memset (xd->counters, 0, sizeof (xd->counters));
+  }
+
+  timeout = 30.0;
+  link_debounce_deadline = 1e70;
+
+  while (1)
+    {
+      /* 36 bit stat counters could overflow in ~50 secs.
+         We poll every 30 secs to be conservative. */
+      vlib_process_wait_for_event_or_clock (vm, timeout);
+
+      event_type = vlib_process_get_events (vm, &event_data);
+
+      switch (event_type)
+	{
+	case EVENT_SET_FLAGS:
+	  /* 1 ms */
+	  link_debounce_deadline = vlib_time_now (vm) + 1e-3;
+	  timeout = 1e-3;
+	  break;
+
+	case ~0:
+	  /* No events found: timer expired. */
+	  if (vlib_time_now (vm) > link_debounce_deadline)
+	    {
+	      vec_foreach (xd, xm->devices)
+	      {
+		ixge_regs_t *r = xd->regs;
+		u32 v = r->xge_mac.link_status;
+		uword is_up = (v & (1 << 30)) != 0;
+
+		vnet_hw_interface_set_flags
+		  (vnm, xd->vlib_hw_if_index,
+		   is_up ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
+	      }
+	      link_debounce_deadline = 1e70;
+	      timeout = 30.0;
+	    }
+	  break;
+
+	default:
+	  ASSERT (0);
+	}
+
+      if (event_data)
+	_vec_len (event_data) = 0;
+
+      /* Query stats every 30 secs. */
+      {
+	f64 now = vlib_time_now (vm);
+	if (now - xm->time_last_stats_update > 30)
+	  {
+	    xm->time_last_stats_update = now;
+	    vec_foreach (xd, xm->devices) ixge_update_counters (xd);
+	  }
+      }
+    }
+
+  return 0;
+}
+
+static vlib_node_registration_t ixge_process_node = {
+  .function = ixge_process,
+  .type = VLIB_NODE_TYPE_PROCESS,
+  .name = "ixge-process",
+};
+
+clib_error_t *
+ixge_init (vlib_main_t * vm)
+{
+  ixge_main_t *xm = &ixge_main;
+  clib_error_t *error;
+
+  xm->vlib_main = vm;
+  memset (&xm->tx_descriptor_template, 0,
+	  sizeof (xm->tx_descriptor_template));
+  memset (&xm->tx_descriptor_template_mask, 0,
+	  sizeof (xm->tx_descriptor_template_mask));
+  xm->tx_descriptor_template.status0 =
+    (IXGE_TX_DESCRIPTOR_STATUS0_ADVANCED |
+     IXGE_TX_DESCRIPTOR_STATUS0_IS_ADVANCED |
+     IXGE_TX_DESCRIPTOR_STATUS0_INSERT_FCS);
+  xm->tx_descriptor_template_mask.status0 = 0xffff;
+  xm->tx_descriptor_template_mask.status1 = 0x00003fff;
+
+  xm->tx_descriptor_template_mask.status0 &=
+    ~(IXGE_TX_DESCRIPTOR_STATUS0_IS_END_OF_PACKET
+      | IXGE_TX_DESCRIPTOR_STATUS0_REPORT_STATUS);
+  xm->tx_descriptor_template_mask.status1 &=
+    ~(IXGE_TX_DESCRIPTOR_STATUS1_DONE);
+
+  error = vlib_call_init_function (vm, pci_bus_init);
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (ixge_init);
+
+
+static void
+ixge_pci_intr_handler (vlib_pci_device_t * dev)
+{
+  ixge_main_t *xm = &ixge_main;
+  vlib_main_t *vm = xm->vlib_main;
+
+  vlib_node_set_interrupt_pending (vm, ixge_input_node.index);
+
+  /* Let node know which device is interrupting. */
+  {
+    vlib_node_runtime_t *rt =
+      vlib_node_get_runtime (vm, ixge_input_node.index);
+    rt->runtime_data[0] |= 1 << dev->private_data;
+  }
+}
+
+static clib_error_t *
+ixge_pci_init (vlib_main_t * vm, vlib_pci_device_t * dev)
+{
+  ixge_main_t *xm = &ixge_main;
+  clib_error_t *error;
+  void *r;
+  ixge_device_t *xd;
+
+  /* Device found: make sure we have dma memory. */
+  if (unix_physmem_is_fake (vm))
+    return clib_error_return (0, "no physical memory available");
+
+  error = vlib_pci_map_resource (dev, 0, &r);
+  if (error)
+    return error;
+
+  vec_add2 (xm->devices, xd, 1);
+
+  if (vec_len (xm->devices) == 1)
+    {
+      ixge_input_node.function = ixge_input_multiarch_select ();
+    }
+
+  xd->pci_device = dev[0];
+  xd->device_id = xd->pci_device.config0.header.device_id;
+  xd->regs = r;
+  xd->device_index = xd - xm->devices;
+  xd->pci_function = dev->bus_address.function;
+  xd->per_interface_next_index = ~0;
+
+
+  /* Chip found so enable node. */
+  {
+    vlib_node_set_state (vm, ixge_input_node.index,
+			 (IXGE_ALWAYS_POLL
+			  ? VLIB_NODE_STATE_POLLING
+			  : VLIB_NODE_STATE_INTERRUPT));
+
+    dev->private_data = xd->device_index;
+  }
+
+  if (vec_len (xm->devices) == 1)
+    {
+      vlib_register_node (vm, &ixge_process_node);
+      xm->process_node_index = ixge_process_node.index;
+    }
+
+  error = vlib_pci_bus_master_enable (dev);
+
+  if (error)
+    return error;
+
+  return vlib_pci_intr_enable (dev);
+}
+
+/* *INDENT-OFF* */
+PCI_REGISTER_DEVICE (ixge_pci_device_registration,static) = {
+  .init_function = ixge_pci_init,
+  .interrupt_handler = ixge_pci_intr_handler,
+  .supported_devices = {
+#define _(t,i) { .vendor_id = PCI_VENDOR_ID_INTEL, .device_id = i, },
+    foreach_ixge_pci_device_id
+#undef _
+    { 0 },
+  },
+};
+/* *INDENT-ON* */
+
+void
+ixge_set_next_node (ixge_rx_next_t next, char *name)
+{
+  vlib_node_registration_t *r = &ixge_input_node;
+
+  switch (next)
+    {
+    case IXGE_RX_NEXT_IP4_INPUT:
+    case IXGE_RX_NEXT_IP6_INPUT:
+    case IXGE_RX_NEXT_ETHERNET_INPUT:
+      r->next_nodes[next] = name;
+      break;
+
+    default:
+      clib_warning ("%s: illegal next %d\n", __FUNCTION__, next);
+      break;
+    }
+}
+#endif
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .default_disabled = 1,
+};
+
+/* *INDENT-ON* */
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/ixge/ixge.h b/src/plugins/ixge/ixge.h
new file mode 100644
index 00000000..779603b3
--- /dev/null
+++ b/src/plugins/ixge/ixge.h
@@ -0,0 +1,1293 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_ixge_h
+#define included_ixge_h
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+typedef volatile struct
+{
+  /* [31:7] 128 byte aligned. */
+  u32 descriptor_address[2];
+  u32 n_descriptor_bytes;
+
+  /* [5] rx/tx descriptor dca enable
+     [6] rx packet head dca enable
+     [7] rx packet tail dca enable
+     [9] rx/tx descriptor relaxed order
+     [11] rx/tx descriptor write back relaxed order
+     [13] rx/tx data write/read relaxed order
+     [15] rx head data write relaxed order
+     [31:24] apic id for cpu's cache. */
+  u32 dca_control;
+
+  u32 head_index;
+
+  /* [4:0] tail buffer size (in 1k byte units)
+     [13:8] head buffer size (in 64 byte units)
+     [24:22] lo free descriptors threshold (units of 64 descriptors)
+     [27:25] descriptor type 0 = legacy, 1 = advanced one buffer (e.g. tail),
+     2 = advanced header splitting (head + tail), 5 = advanced header
+     splitting (head only).
+     [28] drop if no descriptors available. */
+  u32 rx_split_control;
+
+  u32 tail_index;
+    CLIB_PAD_FROM_TO (0x1c, 0x28);
+
+  /* [7:0] rx/tx prefetch threshold
+     [15:8] rx/tx host threshold
+     [24:16] rx/tx write back threshold
+     [25] rx/tx enable
+     [26] tx descriptor writeback flush
+     [30] rx strip vlan enable */
+  u32 control;
+
+  u32 rx_coallesce_control;
+
+  union
+  {
+    struct
+    {
+      /* packets bytes lo hi */
+      u32 stats[3];
+
+      u32 unused;
+    } rx;
+
+    struct
+    {
+      u32 unused[2];
+
+      /* [0] enables head write back. */
+      u32 head_index_write_back_address[2];
+    } tx;
+  };
+} ixge_dma_regs_t;
+
+/* Only advanced descriptors are supported. */
+typedef struct
+{
+  u64 tail_address;
+  u64 head_address;
+} ixge_rx_to_hw_descriptor_t;
+
+typedef struct
+{
+  u32 status[3];
+  u16 n_packet_bytes_this_descriptor;
+  u16 vlan_tag;
+} ixge_rx_from_hw_descriptor_t;
+
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_LAYER2 (1 << (4 + 11))
+/* Valid if not layer2. */
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4 (1 << (4 + 0))
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4_EXT (1 << (4 + 1))
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6 (1 << (4 + 2))
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6_EXT (1 << (4 + 3))
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_TCP (1 << (4 + 4))
+#define IXGE_RX_DESCRIPTOR_STATUS0_IS_UDP (1 << (4 + 5))
+#define IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET(s) (((s) >> 21) & 0x3ff)
+
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE (1 << (0 + 0))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET (1 << (0 + 1))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_VLAN (1 << (0 + 3))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED (1 << (0 + 4))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED (1 << (0 + 5))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED (1 << (0 + 6))
+#define IXGE_RX_DESCRIPTOR_STATUS2_NOT_UNICAST (1 << (0 + 7))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IS_DOUBLE_VLAN (1 << (0 + 9))
+#define IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR (1 << (0 + 10))
+#define IXGE_RX_DESCRIPTOR_STATUS2_ETHERNET_ERROR (1 << (20 + 9))
+#define IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR (1 << (20 + 10))
+#define IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR (1 << (20 + 11))
+
+/* For layer2 packets stats0 bottom 3 bits give ether type index from filter. */
+#define IXGE_RX_DESCRIPTOR_STATUS0_LAYER2_ETHERNET_TYPE(s) ((s) & 7)
+
+typedef struct
+{
+  u64 buffer_address;
+  u16 n_bytes_this_buffer;
+  u16 status0;
+  u32 status1;
+#define IXGE_TX_DESCRIPTOR_STATUS0_ADVANCED (3 << 4)
+#define IXGE_TX_DESCRIPTOR_STATUS0_IS_ADVANCED (1 << (8 + 5))
+#define IXGE_TX_DESCRIPTOR_STATUS0_LOG2_REPORT_STATUS (8 + 3)
+#define IXGE_TX_DESCRIPTOR_STATUS0_REPORT_STATUS (1 << IXGE_TX_DESCRIPTOR_STATUS0_LOG2_REPORT_STATUS)
+#define IXGE_TX_DESCRIPTOR_STATUS0_INSERT_FCS (1 << (8 + 1))
+#define IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET (8 + 0)
+#define IXGE_TX_DESCRIPTOR_STATUS0_IS_END_OF_PACKET (1 << IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET)
+#define IXGE_TX_DESCRIPTOR_STATUS1_DONE (1 << 0)
+#define IXGE_TX_DESCRIPTOR_STATUS1_CONTEXT(i) (/* valid */ (1 << 7) | ((i) << 4))
+#define IXGE_TX_DESCRIPTOR_STATUS1_IPSEC_OFFLOAD (1 << (8 + 2))
+#define IXGE_TX_DESCRIPTOR_STATUS1_INSERT_TCP_UDP_CHECKSUM (1 << (8 + 1))
+#define IXGE_TX_DESCRIPTOR_STATUS1_INSERT_IP4_CHECKSUM (1 << (8 + 0))
+#define IXGE_TX_DESCRIPTOR_STATUS0_N_BYTES_THIS_BUFFER(l) ((l) << 0)
+#define IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET(l) ((l) << 14)
+} ixge_tx_descriptor_t;
+
+typedef struct
+{
+  struct
+  {
+    u8 checksum_start_offset;
+    u8 checksum_insert_offset;
+    u16 checksum_end_offset;
+  } ip, tcp;
+  u32 status0;
+
+  u8 status1;
+
+  /* Byte offset after UDP/TCP header. */
+  u8 payload_offset;
+
+  u16 max_tcp_segment_size;
+} __attribute__ ((packed)) ixge_tx_context_descriptor_t;
+
+typedef union
+{
+  ixge_rx_to_hw_descriptor_t rx_to_hw;
+  ixge_rx_from_hw_descriptor_t rx_from_hw;
+  ixge_tx_descriptor_t tx;
+  u32x4 as_u32x4;
+} ixge_descriptor_t;
+
+typedef volatile struct
+{
+  /* [2] pcie master disable
+     [3] mac reset
+     [26] global device reset */
+  u32 control;
+  u32 control_alias;
+  /* [3:2] device id (0 or 1 for dual port chips)
+     [7] link is up
+     [17:10] num vfs
+     [18] io active
+     [19] pcie master enable status */
+  u32 status_read_only;
+    CLIB_PAD_FROM_TO (0xc, 0x18);
+  /* [14] pf reset done
+     [17] relaxed ordering disable
+     [26] extended vlan enable
+     [28] driver loaded */
+  u32 extended_control;
+    CLIB_PAD_FROM_TO (0x1c, 0x20);
+
+  /* software definable pins.
+     sdp_data [7:0]
+     sdp_is_output [15:8]
+     sdp_is_native [23:16]
+     sdp_function [31:24].
+   */
+  u32 sdp_control;
+    CLIB_PAD_FROM_TO (0x24, 0x28);
+
+  /* [0] i2c clock in
+     [1] i2c clock out
+     [2] i2c data in
+     [3] i2c data out */
+  u32 i2c_control;
+    CLIB_PAD_FROM_TO (0x2c, 0x4c);
+  u32 tcp_timer;
+
+    CLIB_PAD_FROM_TO (0x50, 0x200);
+
+  u32 led_control;
+
+    CLIB_PAD_FROM_TO (0x204, 0x600);
+  u32 core_spare;
+    CLIB_PAD_FROM_TO (0x604, 0x700);
+
+  struct
+  {
+    u32 vflr_events_clear[4];
+    u32 mailbox_interrupt_status[4];
+    u32 mailbox_interrupt_enable[4];
+      CLIB_PAD_FROM_TO (0x730, 0x800);
+  } pf_foo;
+
+  struct
+  {
+    u32 status_write_1_to_clear;
+      CLIB_PAD_FROM_TO (0x804, 0x808);
+    u32 status_write_1_to_set;
+      CLIB_PAD_FROM_TO (0x80c, 0x810);
+    u32 status_auto_clear_enable;
+      CLIB_PAD_FROM_TO (0x814, 0x820);
+
+    /* [11:3] minimum inter-interrupt interval
+       (2e-6 units; 20e-6 units for fast ethernet).
+       [15] low-latency interrupt moderation enable
+       [20:16] low-latency interrupt credit
+       [27:21] interval counter
+       [31] write disable for credit and counter (write only). */
+    u32 throttle0[24];
+
+    u32 enable_write_1_to_set;
+      CLIB_PAD_FROM_TO (0x884, 0x888);
+    u32 enable_write_1_to_clear;
+      CLIB_PAD_FROM_TO (0x88c, 0x890);
+    u32 enable_auto_clear;
+    u32 msi_to_eitr_select;
+    /* [3:0] spd 0-3 interrupt detection enable
+       [4] msi-x enable
+       [5] other clear disable (makes other bits in status not clear on read)
+       etc. */
+    u32 control;
+      CLIB_PAD_FROM_TO (0x89c, 0x900);
+
+    /* Defines interrupt mapping for 128 rx + 128 tx queues.
+       64 x 4 8 bit entries.
+       For register [i]:
+       [5:0] bit in interrupt status for rx queue 2*i + 0
+       [7] valid bit
+       [13:8] bit for tx queue 2*i + 0
+       [15] valid bit
+       similar for rx 2*i + 1 and tx 2*i + 1. */
+    u32 queue_mapping[64];
+
+    /* tcp timer [7:0] and other interrupts [15:8] */
+    u32 misc_mapping;
+      CLIB_PAD_FROM_TO (0xa04, 0xa90);
+
+    /* 64 interrupts determined by mappings. */
+    u32 status1_write_1_to_clear[4];
+    u32 enable1_write_1_to_set[4];
+    u32 enable1_write_1_to_clear[4];
+      CLIB_PAD_FROM_TO (0xac0, 0xad0);
+    u32 status1_enable_auto_clear[4];
+      CLIB_PAD_FROM_TO (0xae0, 0x1000);
+  } interrupt;
+
+  ixge_dma_regs_t rx_dma0[64];
+
+    CLIB_PAD_FROM_TO (0x2000, 0x2140);
+  u32 dcb_rx_packet_plane_t4_config[8];
+  u32 dcb_rx_packet_plane_t4_status[8];
+    CLIB_PAD_FROM_TO (0x2180, 0x2300);
+
+  /* reg i defines mapping for 4 rx queues starting at 4*i + 0. */
+  u32 rx_queue_stats_mapping[32];
+  u32 rx_queue_stats_control;
+
+    CLIB_PAD_FROM_TO (0x2384, 0x2410);
+  u32 fc_user_descriptor_ptr[2];
+  u32 fc_buffer_control;
+    CLIB_PAD_FROM_TO (0x241c, 0x2420);
+  u32 fc_rx_dma;
+    CLIB_PAD_FROM_TO (0x2424, 0x2430);
+  u32 dcb_packet_plane_control;
+    CLIB_PAD_FROM_TO (0x2434, 0x2f00);
+
+  u32 rx_dma_control;
+  u32 pf_queue_drop_enable;
+    CLIB_PAD_FROM_TO (0x2f08, 0x2f20);
+  u32 rx_dma_descriptor_cache_config;
+    CLIB_PAD_FROM_TO (0x2f24, 0x3000);
+
+  /* 1 bit. */
+  u32 rx_enable;
+    CLIB_PAD_FROM_TO (0x3004, 0x3008);
+  /* [15:0] ether type (little endian)
+     [31:16] opcode (big endian) */
+  u32 flow_control_control;
+    CLIB_PAD_FROM_TO (0x300c, 0x3020);
+  /* 3 bit traffic class for each of 8 priorities. */
+  u32 rx_priority_to_traffic_class;
+    CLIB_PAD_FROM_TO (0x3024, 0x3028);
+  u32 rx_coallesce_data_buffer_control;
+    CLIB_PAD_FROM_TO (0x302c, 0x3190);
+  u32 rx_packet_buffer_flush_detect;
+    CLIB_PAD_FROM_TO (0x3194, 0x3200);
+  u32 flow_control_tx_timers[4];	/* 2 timer values */
+    CLIB_PAD_FROM_TO (0x3210, 0x3220);
+  u32 flow_control_rx_threshold_lo[8];
+    CLIB_PAD_FROM_TO (0x3240, 0x3260);
+  u32 flow_control_rx_threshold_hi[8];
+    CLIB_PAD_FROM_TO (0x3280, 0x32a0);
+  u32 flow_control_refresh_threshold;
+    CLIB_PAD_FROM_TO (0x32a4, 0x3c00);
+  /* For each of 8 traffic classes (units of bytes). */
+  u32 rx_packet_buffer_size[8];
+    CLIB_PAD_FROM_TO (0x3c20, 0x3d00);
+  u32 flow_control_config;
+    CLIB_PAD_FROM_TO (0x3d04, 0x4200);
+
+  struct
+  {
+    u32 pcs_config;
+      CLIB_PAD_FROM_TO (0x4204, 0x4208);
+    u32 link_control;
+    u32 link_status;
+    u32 pcs_debug[2];
+    u32 auto_negotiation;
+    u32 link_partner_ability;
+    u32 auto_negotiation_tx_next_page;
+    u32 auto_negotiation_link_partner_next_page;
+      CLIB_PAD_FROM_TO (0x4228, 0x4240);
+  } gige_mac;
+
+  struct
+  {
+    /* [0] tx crc enable
+       [2] enable frames up to max frame size register [31:16]
+       [10] pad frames < 64 bytes if specified by user
+       [15] loopback enable
+       [16] mdc hi speed
+       [17] turn off mdc between mdio packets */
+    u32 control;
+
+    /* [5] rx symbol error (all bits clear on read)
+       [6] rx illegal symbol
+       [7] rx idle error
+       [8] rx local fault
+       [9] rx remote fault */
+    u32 status;
+
+    u32 pause_and_pace_control;
+      CLIB_PAD_FROM_TO (0x424c, 0x425c);
+    u32 phy_command;
+    u32 phy_data;
+      CLIB_PAD_FROM_TO (0x4264, 0x4268);
+
+    /* [31:16] max frame size in bytes. */
+    u32 rx_max_frame_size;
+      CLIB_PAD_FROM_TO (0x426c, 0x4288);
+
+    /* [0]
+       [2] pcs receive link up? (latch lo)
+       [7] local fault
+       [1]
+       [0] pcs 10g base r capable
+       [1] pcs 10g base x capable
+       [2] pcs 10g base w capable
+       [10] rx local fault
+       [11] tx local fault
+       [15:14] 2 => device present at this address (else not present) */
+    u32 xgxs_status[2];
+
+    u32 base_x_pcs_status;
+
+    /* [0] pass unrecognized flow control frames
+       [1] discard pause frames
+       [2] rx priority flow control enable (only in dcb mode)
+       [3] rx flow control enable. */
+    u32 flow_control;
+
+    /* [3:0] tx lanes change polarity
+       [7:4] rx lanes change polarity
+       [11:8] swizzle tx lanes
+       [15:12] swizzle rx lanes
+       4 x 2 bit tx lane swap
+       4 x 2 bit rx lane swap. */
+    u32 serdes_control;
+
+    u32 fifo_control;
+
+    /* [0] force link up
+       [1] autoneg ack2 bit to transmit
+       [6:2] autoneg selector field to transmit
+       [8:7] 10g pma/pmd type 0 => xaui, 1 kx4, 2 cx4
+       [9] 1g pma/pmd type 0 => sfi, 1 => kx/bx
+       [10] disable 10g on without main power
+       [11] restart autoneg on transition to dx power state
+       [12] restart autoneg
+       [15:13] link mode:
+       0 => 1g no autoneg
+       1 => 10g kx4 parallel link no autoneg
+       2 => 1g bx autoneg
+       3 => 10g sfi serdes
+       4 => kx4/kx/kr
+       5 => xgmii 1g/100m
+       6 => kx4/kx/kr 1g an
+       7 kx4/kx/kr sgmii.
+       [16] kr support
+       [17] fec requested
+       [18] fec ability
+       etc. */
+    u32 auto_negotiation_control;
+
+    /* [0] signal detect 1g/100m
+       [1] fec signal detect
+       [2] 10g serial pcs fec block lock
+       [3] 10g serial high error rate
+       [4] 10g serial pcs block lock
+       [5] kx/kx4/kr autoneg next page received
+       [6] kx/kx4/kr backplane autoneg next page received
+       [7] link status clear to read
+       [11:8] 10g signal detect (4 lanes) (for serial just lane 0)
+       [12] 10g serial signal detect
+       [16:13] 10g parallel lane sync status
+       [17] 10g parallel align status
+       [18] 1g sync status
+       [19] kx/kx4/kr backplane autoneg is idle
+       [20] 1g autoneg enabled
+       [21] 1g pcs enabled for sgmii
+       [22] 10g xgxs enabled
+       [23] 10g serial fec enabled (forward error detection)
+       [24] 10g kr pcs enabled
+       [25] sgmii enabled
+       [27:26] mac link mode
+       0 => 1g
+       1 => 10g parallel
+       2 => 10g serial
+       3 => autoneg
+       [29:28] link speed
+       1 => 100m
+       2 => 1g
+       3 => 10g
+       [30] link is up
+       [31] kx/kx4/kr backplane autoneg completed successfully. */
+    u32 link_status;
+
+    /* [17:16] pma/pmd for 10g serial
+       0 => kr, 2 => sfi
+       [18] disable dme pages */
+    u32 auto_negotiation_control2;
+
+      CLIB_PAD_FROM_TO (0x42ac, 0x42b0);
+    u32 link_partner_ability[2];
+      CLIB_PAD_FROM_TO (0x42b8, 0x42d0);
+    u32 manageability_control;
+    u32 link_partner_next_page[2];
+      CLIB_PAD_FROM_TO (0x42dc, 0x42e0);
+    u32 kr_pcs_control;
+    u32 kr_pcs_status;
+    u32 fec_status[2];
+      CLIB_PAD_FROM_TO (0x42f0, 0x4314);
+    u32 sgmii_control;
+      CLIB_PAD_FROM_TO (0x4318, 0x4324);
+    u32 link_status2;
+      CLIB_PAD_FROM_TO (0x4328, 0x4900);
+  } xge_mac;
+
+  u32 tx_dcb_control;
+  u32 tx_dcb_descriptor_plane_queue_select;
+  u32 tx_dcb_descriptor_plane_t1_config;
+  u32 tx_dcb_descriptor_plane_t1_status;
+    CLIB_PAD_FROM_TO (0x4910, 0x4950);
+
+  /* For each TC in units of 1k bytes. */
+  u32 tx_packet_buffer_thresholds[8];
+    CLIB_PAD_FROM_TO (0x4970, 0x4980);
+  struct
+  {
+    u32 mmw;
+    u32 config;
+    u32 status;
+    u32 rate_drift;
+  } dcb_tx_rate_scheduler;
+    CLIB_PAD_FROM_TO (0x4990, 0x4a80);
+  u32 tx_dma_control;
+    CLIB_PAD_FROM_TO (0x4a84, 0x4a88);
+  u32 tx_dma_tcp_flags_control[2];
+    CLIB_PAD_FROM_TO (0x4a90, 0x4b00);
+  u32 pf_mailbox[64];
+    CLIB_PAD_FROM_TO (0x4c00, 0x5000);
+
+  /* RX */
+  u32 checksum_control;
+    CLIB_PAD_FROM_TO (0x5004, 0x5008);
+  u32 rx_filter_control;
+    CLIB_PAD_FROM_TO (0x500c, 0x5010);
+  u32 management_vlan_tag[8];
+  u32 management_udp_tcp_ports[8];
+    CLIB_PAD_FROM_TO (0x5050, 0x5078);
+  /* little endian. */
+  u32 extended_vlan_ether_type;
+    CLIB_PAD_FROM_TO (0x507c, 0x5080);
+  /* [1] store/dma bad packets
+     [8] accept all multicast
+     [9] accept all unicast
+     [10] accept all broadcast. */
+  u32 filter_control;
+    CLIB_PAD_FROM_TO (0x5084, 0x5088);
+  /* [15:0] vlan ethernet type (0x8100) little endian
+     [28] cfi bit expected
+     [29] drop packets with unexpected cfi bit
+     [30] vlan filter enable. */
+  u32 vlan_control;
+    CLIB_PAD_FROM_TO (0x508c, 0x5090);
+  /* [1:0] hi bit of ethernet address for 12 bit index into multicast table
+     0 => 47, 1 => 46, 2 => 45, 3 => 43.
+     [2] enable multicast filter
+   */
+  u32 multicast_control;
+    CLIB_PAD_FROM_TO (0x5094, 0x5100);
+  u32 fcoe_rx_control;
+    CLIB_PAD_FROM_TO (0x5104, 0x5108);
+  u32 fc_flt_context;
+    CLIB_PAD_FROM_TO (0x510c, 0x5110);
+  u32 fc_filter_control;
+    CLIB_PAD_FROM_TO (0x5114, 0x5120);
+  u32 rx_message_type_lo;
+    CLIB_PAD_FROM_TO (0x5124, 0x5128);
+  /* [15:0] ethernet type (little endian)
+     [18:16] matche pri in vlan tag
+     [19] priority match enable
+     [25:20] virtualization pool
+     [26] pool enable
+     [27] is fcoe
+     [30] ieee 1588 timestamp enable
+     [31] filter enable.
+     (See ethernet_type_queue_select.) */
+  u32 ethernet_type_queue_filter[8];
+    CLIB_PAD_FROM_TO (0x5148, 0x5160);
+  /* [7:0] l2 ethernet type and
+     [15:8] l2 ethernet type or */
+  u32 management_decision_filters1[8];
+  u32 vf_vm_tx_switch_loopback_enable[2];
+  u32 rx_time_sync_control;
+    CLIB_PAD_FROM_TO (0x518c, 0x5190);
+  u32 management_ethernet_type_filters[4];
+  u32 rx_timestamp_attributes_lo;
+  u32 rx_timestamp_hi;
+  u32 rx_timestamp_attributes_hi;
+    CLIB_PAD_FROM_TO (0x51ac, 0x51b0);
+  u32 pf_virtual_control;
+    CLIB_PAD_FROM_TO (0x51b4, 0x51d8);
+  u32 fc_offset_parameter;
+    CLIB_PAD_FROM_TO (0x51dc, 0x51e0);
+  u32 vf_rx_enable[2];
+  u32 rx_timestamp_lo;
+    CLIB_PAD_FROM_TO (0x51ec, 0x5200);
+  /* 12 bits determined by multicast_control
+     lookup bits in this vector. */
+  u32 multicast_enable[128];
+
+  /* [0] ethernet address [31:0]
+     [1] [15:0] ethernet address [47:32]
+     [31] valid bit.
+     Index 0 is read from eeprom after reset. */
+  u32 rx_ethernet_address0[16][2];
+
+    CLIB_PAD_FROM_TO (0x5480, 0x5800);
+  u32 wake_up_control;
+    CLIB_PAD_FROM_TO (0x5804, 0x5808);
+  u32 wake_up_filter_control;
+    CLIB_PAD_FROM_TO (0x580c, 0x5818);
+  u32 multiple_rx_queue_command_82598;
+    CLIB_PAD_FROM_TO (0x581c, 0x5820);
+  u32 management_control;
+  u32 management_filter_control;
+    CLIB_PAD_FROM_TO (0x5828, 0x5838);
+  u32 wake_up_ip4_address_valid;
+    CLIB_PAD_FROM_TO (0x583c, 0x5840);
+  u32 wake_up_ip4_address_table[4];
+  u32 management_control_to_host;
+    CLIB_PAD_FROM_TO (0x5854, 0x5880);
+  u32 wake_up_ip6_address_table[4];
+
+  /* unicast_and broadcast_and vlan_and ip_address_and
+     etc. */
+  u32 management_decision_filters[8];
+
+  u32 management_ip4_or_ip6_address_filters[4][4];
+    CLIB_PAD_FROM_TO (0x58f0, 0x5900);
+  u32 wake_up_packet_length;
+    CLIB_PAD_FROM_TO (0x5904, 0x5910);
+  u32 management_ethernet_address_filters[4][2];
+    CLIB_PAD_FROM_TO (0x5930, 0x5a00);
+  u32 wake_up_packet_memory[32];
+    CLIB_PAD_FROM_TO (0x5a80, 0x5c00);
+  u32 redirection_table_82598[32];
+  u32 rss_random_keys_82598[10];
+    CLIB_PAD_FROM_TO (0x5ca8, 0x6000);
+
+  ixge_dma_regs_t tx_dma[128];
+
+  u32 pf_vm_vlan_insert[64];
+  u32 tx_dma_tcp_max_alloc_size_requests;
+    CLIB_PAD_FROM_TO (0x8104, 0x8110);
+  u32 vf_tx_enable[2];
+    CLIB_PAD_FROM_TO (0x8118, 0x8120);
+  /* [0] dcb mode enable
+     [1] virtualization mode enable
+     [3:2] number of tcs/qs per pool. */
+  u32 multiple_tx_queues_command;
+    CLIB_PAD_FROM_TO (0x8124, 0x8200);
+  u32 pf_vf_anti_spoof[8];
+  u32 pf_dma_tx_switch_control;
+    CLIB_PAD_FROM_TO (0x8224, 0x82e0);
+  u32 tx_strict_low_latency_queues[4];
+    CLIB_PAD_FROM_TO (0x82f0, 0x8600);
+  u32 tx_queue_stats_mapping_82599[32];
+  u32 tx_queue_packet_counts[32];
+  u32 tx_queue_byte_counts[32][2];
+
+  struct
+  {
+    u32 control;
+    u32 status;
+    u32 buffer_almost_full;
+      CLIB_PAD_FROM_TO (0x880c, 0x8810);
+    u32 buffer_min_ifg;
+      CLIB_PAD_FROM_TO (0x8814, 0x8900);
+  } tx_security;
+
+  struct
+  {
+    u32 index;
+    u32 salt;
+    u32 key[4];
+      CLIB_PAD_FROM_TO (0x8918, 0x8a00);
+  } tx_ipsec;
+
+  struct
+  {
+    u32 capabilities;
+    u32 control;
+    u32 tx_sci[2];
+    u32 sa;
+    u32 sa_pn[2];
+    u32 key[2][4];
+    /* untagged packets, encrypted packets, protected packets,
+       encrypted bytes, protected bytes */
+    u32 stats[5];
+      CLIB_PAD_FROM_TO (0x8a50, 0x8c00);
+  } tx_link_security;
+
+  struct
+  {
+    u32 control;
+    u32 timestamp_value[2];
+    u32 system_time[2];
+    u32 increment_attributes;
+    u32 time_adjustment_offset[2];
+    u32 aux_control;
+    u32 target_time[2][2];
+      CLIB_PAD_FROM_TO (0x8c34, 0x8c3c);
+    u32 aux_time_stamp[2][2];
+      CLIB_PAD_FROM_TO (0x8c4c, 0x8d00);
+  } tx_timesync;
+
+  struct
+  {
+    u32 control;
+    u32 status;
+      CLIB_PAD_FROM_TO (0x8d08, 0x8e00);
+  } rx_security;
+
+  struct
+  {
+    u32 index;
+    u32 ip_address[4];
+    u32 spi;
+    u32 ip_index;
+    u32 key[4];
+    u32 salt;
+    u32 mode;
+      CLIB_PAD_FROM_TO (0x8e34, 0x8f00);
+  } rx_ipsec;
+
+  struct
+  {
+    u32 capabilities;
+    u32 control;
+    u32 sci[2];
+    u32 sa[2];
+    u32 sa_pn[2];
+    u32 key[2][4];
+    /* see datasheet */
+    u32 stats[17];
+      CLIB_PAD_FROM_TO (0x8f84, 0x9000);
+  } rx_link_security;
+
+  /* 4 wake up, 2 management, 2 wake up. */
+  u32 flexible_filters[8][16][4];
+    CLIB_PAD_FROM_TO (0x9800, 0xa000);
+
+  /* 4096 bits. */
+  u32 vlan_filter[128];
+
+  /* [0] ethernet address [31:0]
+     [1] [15:0] ethernet address [47:32]
+     [31] valid bit.
+     Index 0 is read from eeprom after reset. */
+  u32 rx_ethernet_address1[128][2];
+
+  /* select one of 64 pools for each rx address. */
+  u32 rx_ethernet_address_pool_select[128][2];
+    CLIB_PAD_FROM_TO (0xaa00, 0xc800);
+  u32 tx_priority_to_traffic_class;
+    CLIB_PAD_FROM_TO (0xc804, 0xcc00);
+
+  /* In bytes units of 1k.  Total packet buffer is 160k. */
+  u32 tx_packet_buffer_size[8];
+
+    CLIB_PAD_FROM_TO (0xcc20, 0xcd10);
+  u32 tx_manageability_tc_mapping;
+    CLIB_PAD_FROM_TO (0xcd14, 0xcd20);
+  u32 dcb_tx_packet_plane_t2_config[8];
+  u32 dcb_tx_packet_plane_t2_status[8];
+    CLIB_PAD_FROM_TO (0xcd60, 0xce00);
+
+  u32 tx_flow_control_status;
+    CLIB_PAD_FROM_TO (0xce04, 0xd000);
+
+  ixge_dma_regs_t rx_dma1[64];
+
+  struct
+  {
+    /* Bigendian ip4 src/dst address. */
+    u32 src_address[128];
+    u32 dst_address[128];
+
+    /* TCP/UDP ports [15:0] src [31:16] dst; bigendian. */
+    u32 tcp_udp_port[128];
+
+    /* [1:0] protocol tcp, udp, sctp, other
+       [4:2] match priority (highest wins)
+       [13:8] pool
+       [25] src address match disable
+       [26] dst address match disable
+       [27] src port match disable
+       [28] dst port match disable
+       [29] protocol match disable
+       [30] pool match disable
+       [31] enable. */
+    u32 control[128];
+
+    /* [12] size bypass
+       [19:13] must be 0x80
+       [20] low-latency interrupt
+       [27:21] rx queue. */
+    u32 interrupt[128];
+  } ip4_filters;
+
+    CLIB_PAD_FROM_TO (0xea00, 0xeb00);
+  /* 4 bit rss output index indexed by 7 bit hash.
+     128 8 bit fields = 32 registers. */
+  u32 redirection_table_82599[32];
+
+  u32 rss_random_key_82599[10];
+    CLIB_PAD_FROM_TO (0xeba8, 0xec00);
+  /* [15:0] reserved
+     [22:16] rx queue index
+     [29] low-latency interrupt on match
+     [31] enable */
+  u32 ethernet_type_queue_select[8];
+    CLIB_PAD_FROM_TO (0xec20, 0xec30);
+  u32 syn_packet_queue_filter;
+    CLIB_PAD_FROM_TO (0xec34, 0xec60);
+  u32 immediate_interrupt_rx_vlan_priority;
+    CLIB_PAD_FROM_TO (0xec64, 0xec70);
+  u32 rss_queues_per_traffic_class;
+    CLIB_PAD_FROM_TO (0xec74, 0xec90);
+  u32 lli_size_threshold;
+    CLIB_PAD_FROM_TO (0xec94, 0xed00);
+
+  struct
+  {
+    u32 control;
+      CLIB_PAD_FROM_TO (0xed04, 0xed10);
+    u32 table[8];
+      CLIB_PAD_FROM_TO (0xed30, 0xee00);
+  } fcoe_redirection;
+
+  struct
+  {
+    /* [1:0] packet buffer allocation 0 => disabled, else 64k*2^(f-1)
+       [3] packet buffer initialization done
+       [4] perfetch match mode
+       [5] report status in rss field of rx descriptors
+       [7] report status always
+       [14:8] drop queue
+       [20:16] flex 2 byte packet offset (units of 2 bytes)
+       [27:24] max linked list length
+       [31:28] full threshold. */
+    u32 control;
+      CLIB_PAD_FROM_TO (0xee04, 0xee0c);
+
+    u32 data[8];
+
+    /* [1:0] 0 => no action, 1 => add, 2 => remove, 3 => query.
+       [2] valid filter found by query command
+       [3] filter update override
+       [4] ip6 adress table
+       [6:5] l4 protocol reserved, udp, tcp, sctp
+       [7] is ip6
+       [8] clear head/tail
+       [9] packet drop action
+       [10] matched packet generates low-latency interrupt
+       [11] last in linked list
+       [12] collision
+       [15] rx queue enable
+       [22:16] rx queue
+       [29:24] pool. */
+    u32 command;
+
+      CLIB_PAD_FROM_TO (0xee30, 0xee3c);
+    /* ip4 dst/src address, tcp ports, udp ports.
+       set bits mean bit is ignored. */
+    u32 ip4_masks[4];
+    u32 filter_length;
+    u32 usage_stats;
+    u32 failed_usage_stats;
+    u32 filters_match_stats;
+    u32 filters_miss_stats;
+      CLIB_PAD_FROM_TO (0xee60, 0xee68);
+    /* Lookup, signature. */
+    u32 hash_keys[2];
+    /* [15:0] ip6 src address 1 bit per byte
+       [31:16] ip6 dst address. */
+    u32 ip6_mask;
+    /* [0] vlan id
+       [1] vlan priority
+       [2] pool
+       [3] ip protocol
+       [4] flex
+       [5] dst ip6. */
+    u32 other_mask;
+      CLIB_PAD_FROM_TO (0xee78, 0xf000);
+  } flow_director;
+
+  struct
+  {
+    u32 l2_control[64];
+    u32 vlan_pool_filter[64];
+    u32 vlan_pool_filter_bitmap[128];
+    u32 dst_ethernet_address[128];
+    u32 mirror_rule[4];
+    u32 mirror_rule_vlan[8];
+    u32 mirror_rule_pool[8];
+      CLIB_PAD_FROM_TO (0xf650, 0x10010);
+  } pf_bar;
+
+  u32 eeprom_flash_control;
+  /* [0] start
+     [1] done
+     [15:2] address
+     [31:16] read data. */
+  u32 eeprom_read;
+    CLIB_PAD_FROM_TO (0x10018, 0x1001c);
+  u32 flash_access;
+    CLIB_PAD_FROM_TO (0x10020, 0x10114);
+  u32 flash_data;
+  u32 flash_control;
+  u32 flash_read_data;
+    CLIB_PAD_FROM_TO (0x10120, 0x1013c);
+  u32 flash_opcode;
+  u32 software_semaphore;
+    CLIB_PAD_FROM_TO (0x10144, 0x10148);
+  u32 firmware_semaphore;
+    CLIB_PAD_FROM_TO (0x1014c, 0x10160);
+  u32 software_firmware_sync;
+    CLIB_PAD_FROM_TO (0x10164, 0x10200);
+  u32 general_rx_control;
+    CLIB_PAD_FROM_TO (0x10204, 0x11000);
+
+  struct
+  {
+    u32 control;
+      CLIB_PAD_FROM_TO (0x11004, 0x11010);
+    /* [3:0] enable counters
+       [7:4] leaky bucket counter mode
+       [29] reset
+       [30] stop
+       [31] start. */
+    u32 counter_control;
+    /* [7:0],[15:8],[23:16],[31:24] event for counters 0-3.
+       event codes:
+       0x0 bad tlp
+       0x10 reqs that reached timeout
+       etc. */
+    u32 counter_event;
+      CLIB_PAD_FROM_TO (0x11018, 0x11020);
+    u32 counters_clear_on_read[4];
+    u32 counter_config[4];
+    struct
+    {
+      u32 address;
+      u32 data;
+    } indirect_access;
+      CLIB_PAD_FROM_TO (0x11048, 0x11050);
+    u32 extended_control;
+      CLIB_PAD_FROM_TO (0x11054, 0x11064);
+    u32 mirrored_revision_id;
+      CLIB_PAD_FROM_TO (0x11068, 0x11070);
+    u32 dca_requester_id_information;
+
+    /* [0] global disable
+       [4:1] mode: 0 => legacy, 1 => dca 1.0. */
+    u32 dca_control;
+      CLIB_PAD_FROM_TO (0x11078, 0x110b0);
+    /* [0] pci completion abort
+       [1] unsupported i/o address
+       [2] wrong byte enable
+       [3] pci timeout */
+    u32 pcie_interrupt_status;
+      CLIB_PAD_FROM_TO (0x110b4, 0x110b8);
+    u32 pcie_interrupt_enable;
+      CLIB_PAD_FROM_TO (0x110bc, 0x110c0);
+    u32 msi_x_pba_clear[8];
+      CLIB_PAD_FROM_TO (0x110e0, 0x12300);
+  } pcie;
+
+  u32 interrupt_throttle1[128 - 24];
+    CLIB_PAD_FROM_TO (0x124a0, 0x14f00);
+
+  u32 core_analog_config;
+    CLIB_PAD_FROM_TO (0x14f04, 0x14f10);
+  u32 core_common_config;
+    CLIB_PAD_FROM_TO (0x14f14, 0x15f14);
+
+  u32 link_sec_software_firmware_interface;
+} ixge_regs_t;
+
+typedef union
+{
+  struct
+  {
+    /* Addresses bigendian. */
+    union
+    {
+      struct
+      {
+	ip6_address_t src_address;
+	u32 unused[1];
+      } ip6;
+      struct
+      {
+	u32 unused[3];
+	ip4_address_t src_address, dst_address;
+      } ip4;
+    };
+
+    /* [15:0] src port (little endian).
+       [31:16] dst port. */
+    u32 tcp_udp_ports;
+
+    /* [15:0] vlan (cfi bit set to 0).
+       [31:16] flex bytes.  bigendian. */
+    u32 vlan_and_flex_word;
+
+    /* [14:0] hash
+       [15] bucket valid
+       [31:16] signature (signature filers)/sw-index (perfect match). */
+    u32 hash;
+  };
+
+  u32 as_u32[8];
+} ixge_flow_director_key_t;
+
+always_inline void
+ixge_throttle_queue_interrupt (ixge_regs_t * r,
+			       u32 queue_interrupt_index,
+			       f64 inter_interrupt_interval_in_secs)
+{
+  volatile u32 *tr =
+    (queue_interrupt_index < ARRAY_LEN (r->interrupt.throttle0)
+     ? &r->interrupt.throttle0[queue_interrupt_index]
+     : &r->interrupt_throttle1[queue_interrupt_index]);
+  ASSERT (queue_interrupt_index < 128);
+  u32 v;
+  i32 i, mask = (1 << 9) - 1;
+
+  i = flt_round_nearest (inter_interrupt_interval_in_secs / 2e-6);
+  i = i < 1 ? 1 : i;
+  i = i >= mask ? mask : i;
+
+  v = tr[0];
+  v &= ~(mask << 3);
+  v |= i << 3;
+  tr[0] = v;
+}
+
+#define foreach_ixge_counter				\
+  _ (0x40d0, rx_total_packets)				\
+  _64 (0x40c0, rx_total_bytes)				\
+  _ (0x41b0, rx_good_packets_before_filtering)		\
+  _64 (0x41b4, rx_good_bytes_before_filtering)		\
+  _ (0x2f50, rx_dma_good_packets)			\
+  _64 (0x2f54, rx_dma_good_bytes)			\
+  _ (0x2f5c, rx_dma_duplicated_good_packets)		\
+  _64 (0x2f60, rx_dma_duplicated_good_bytes)		\
+  _ (0x2f68, rx_dma_good_loopback_packets)		\
+  _64 (0x2f6c, rx_dma_good_loopback_bytes)		\
+  _ (0x2f74, rx_dma_good_duplicated_loopback_packets)	\
+  _64 (0x2f78, rx_dma_good_duplicated_loopback_bytes)	\
+  _ (0x4074, rx_good_packets)				\
+  _64 (0x4088, rx_good_bytes)				\
+  _ (0x407c, rx_multicast_packets)			\
+  _ (0x4078, rx_broadcast_packets)			\
+  _ (0x405c, rx_64_byte_packets)			\
+  _ (0x4060, rx_65_127_byte_packets)			\
+  _ (0x4064, rx_128_255_byte_packets)			\
+  _ (0x4068, rx_256_511_byte_packets)			\
+  _ (0x406c, rx_512_1023_byte_packets)			\
+  _ (0x4070, rx_gt_1023_byte_packets)			\
+  _ (0x4000, rx_crc_errors)				\
+  _ (0x4120, rx_ip_checksum_errors)			\
+  _ (0x4004, rx_illegal_symbol_errors)			\
+  _ (0x4008, rx_error_symbol_errors)			\
+  _ (0x4034, rx_mac_local_faults)			\
+  _ (0x4038, rx_mac_remote_faults)			\
+  _ (0x4040, rx_length_errors)				\
+  _ (0x41a4, rx_xons)					\
+  _ (0x41a8, rx_xoffs)					\
+  _ (0x40a4, rx_undersize_packets)			\
+  _ (0x40a8, rx_fragments)				\
+  _ (0x40ac, rx_oversize_packets)			\
+  _ (0x40b0, rx_jabbers)				\
+  _ (0x40b4, rx_management_packets)			\
+  _ (0x40b8, rx_management_drops)			\
+  _ (0x3fa0, rx_missed_packets_pool_0)			\
+  _ (0x40d4, tx_total_packets)				\
+  _ (0x4080, tx_good_packets)				\
+  _64 (0x4090, tx_good_bytes)				\
+  _ (0x40f0, tx_multicast_packets)			\
+  _ (0x40f4, tx_broadcast_packets)			\
+  _ (0x87a0, tx_dma_good_packets)			\
+  _64 (0x87a4, tx_dma_good_bytes)			\
+  _ (0x40d8, tx_64_byte_packets)			\
+  _ (0x40dc, tx_65_127_byte_packets)			\
+  _ (0x40e0, tx_128_255_byte_packets)			\
+  _ (0x40e4, tx_256_511_byte_packets)			\
+  _ (0x40e8, tx_512_1023_byte_packets)			\
+  _ (0x40ec, tx_gt_1023_byte_packets)			\
+  _ (0x4010, tx_undersize_drops)			\
+  _ (0x8780, switch_security_violation_packets)		\
+  _ (0x5118, fc_crc_errors)				\
+  _ (0x241c, fc_rx_drops)				\
+  _ (0x2424, fc_last_error_count)			\
+  _ (0x2428, fcoe_rx_packets)				\
+  _ (0x242c, fcoe_rx_dwords)				\
+  _ (0x8784, fcoe_tx_packets)				\
+  _ (0x8788, fcoe_tx_dwords)				\
+  _ (0x1030, queue_0_rx_count)				\
+  _ (0x1430, queue_0_drop_count)			\
+  _ (0x1070, queue_1_rx_count)				\
+  _ (0x1470, queue_1_drop_count)			\
+  _ (0x10b0, queue_2_rx_count)				\
+  _ (0x14b0, queue_2_drop_count)			\
+  _ (0x10f0, queue_3_rx_count)				\
+  _ (0x14f0, queue_3_drop_count)			\
+  _ (0x1130, queue_4_rx_count)				\
+  _ (0x1530, queue_4_drop_count)			\
+  _ (0x1170, queue_5_rx_count)				\
+  _ (0x1570, queue_5_drop_count)			\
+  _ (0x11b0, queue_6_rx_count)				\
+  _ (0x15b0, queue_6_drop_count)			\
+  _ (0x11f0, queue_7_rx_count)				\
+  _ (0x15f0, queue_7_drop_count)			\
+  _ (0x1230, queue_8_rx_count)				\
+  _ (0x1630, queue_8_drop_count)			\
+  _ (0x1270, queue_9_rx_count)				\
+  _ (0x1270, queue_9_drop_count)
+
+
+
+
+typedef enum
+{
+#define _(a,f) IXGE_COUNTER_##f,
+#define _64(a,f) _(a,f)
+  foreach_ixge_counter
+#undef _
+#undef _64
+    IXGE_N_COUNTER,
+} ixge_counter_type_t;
+
+typedef struct
+{
+  u32 mdio_address;
+
+  /* 32 bit ID read from ID registers. */
+  u32 id;
+} ixge_phy_t;
+
+typedef struct
+{
+  /* Cache aligned descriptors. */
+  ixge_descriptor_t *descriptors;
+
+  /* Number of descriptors in table. */
+  u32 n_descriptors;
+
+  /* Software head and tail pointers into descriptor ring. */
+  u32 head_index, tail_index;
+
+  /* Index into dma_queues vector. */
+  u32 queue_index;
+
+  /* Buffer indices corresponding to each active descriptor. */
+  u32 *descriptor_buffer_indices;
+
+  union
+  {
+    struct
+    {
+      u32 *volatile head_index_write_back;
+
+      u32 n_buffers_on_ring;
+    } tx;
+
+    struct
+    {
+      /* Buffer indices to use to replenish each descriptor. */
+      u32 *replenish_buffer_indices;
+
+      vlib_node_runtime_t *node;
+      u32 next_index;
+
+      u32 saved_start_of_packet_buffer_index;
+
+      u32 saved_start_of_packet_next_index;
+      u32 saved_last_buffer_index;
+
+      u32 is_start_of_packet;
+
+      u32 n_descriptors_done_total;
+
+      u32 n_descriptors_done_this_call;
+
+      u32 n_bytes;
+    } rx;
+  };
+} ixge_dma_queue_t;
+
+#define foreach_ixge_pci_device_id		\
+  _ (82598, 0x10b6)				\
+  _ (82598_bx, 0x1508)				\
+  _ (82598af_dual_port, 0x10c6)			\
+  _ (82598af_single_port, 0x10c7)		\
+  _ (82598at, 0x10c8)				\
+  _ (82598at2, 0x150b)				\
+  _ (82598eb_sfp_lom, 0x10db)			\
+  _ (82598eb_cx4, 0x10dd)			\
+  _ (82598_cx4_dual_port, 0x10ec)		\
+  _ (82598_da_dual_port, 0x10f1)		\
+  _ (82598_sr_dual_port_em, 0x10e1)		\
+  _ (82598eb_xf_lr, 0x10f4)			\
+  _ (82599_kx4, 0x10f7)				\
+  _ (82599_kx4_mezz, 0x1514)			\
+  _ (82599_kr, 0x1517)				\
+  _ (82599_combo_backplane, 0x10f8)		\
+  _ (82599_cx4, 0x10f9)				\
+  _ (82599_sfp, 0x10fb)				\
+  _ (82599_backplane_fcoe, 0x152a)		\
+  _ (82599_sfp_fcoe, 0x1529)			\
+  _ (82599_sfp_em, 0x1507)			\
+  _ (82599_xaui_lom, 0x10fc)			\
+  _ (82599_t3_lom, 0x151c)			\
+  _ (x540t, 0x1528)
+
+typedef enum
+{
+#define _(f,n) IXGE_##f = n,
+  foreach_ixge_pci_device_id
+#undef _
+} ixge_pci_device_id_t;
+
+typedef struct
+{
+  /* registers */
+  ixge_regs_t *regs;
+
+  /* Specific next index when using dynamic redirection */
+  u32 per_interface_next_index;
+
+  /* PCI bus info. */
+  vlib_pci_device_t pci_device;
+
+  /* From PCI config space header. */
+  ixge_pci_device_id_t device_id;
+
+  u16 device_index;
+
+  /* 0 or 1. */
+  u16 pci_function;
+
+  /* VLIB interface for this instance. */
+  u32 vlib_hw_if_index, vlib_sw_if_index;
+
+  ixge_dma_queue_t *dma_queues[VLIB_N_RX_TX];
+
+  /* Phy index (0 or 1) and address on MDI bus. */
+  u32 phy_index;
+  ixge_phy_t phys[2];
+
+  /* Value of link_status register at last link change. */
+  u32 link_status_at_last_link_change;
+
+  i2c_bus_t i2c_bus;
+  sfp_eeprom_t sfp_eeprom;
+
+  /* Counters. */
+  u64 counters[IXGE_N_COUNTER], counters_last_clear[IXGE_N_COUNTER];
+} ixge_device_t;
+
+typedef struct
+{
+  vlib_main_t *vlib_main;
+
+  /* Vector of devices. */
+  ixge_device_t *devices;
+
+  /* Descriptor ring sizes. */
+  u32 n_descriptors[VLIB_N_RX_TX];
+
+  /* RX buffer size.  Must be at least 1k; will be rounded to
+     next largest 1k size. */
+  u32 n_bytes_in_rx_buffer;
+
+  u32 n_descriptors_per_cache_line;
+
+  u32 vlib_buffer_free_list_index;
+
+  u32 process_node_index;
+
+  /* Template and mask for initializing/validating TX descriptors. */
+  ixge_tx_descriptor_t tx_descriptor_template, tx_descriptor_template_mask;
+
+  /* Vector of buffers for which TX is done and can be freed. */
+  u32 *tx_buffers_pending_free;
+
+  u32 *rx_buffers_to_add;
+
+  f64 time_last_stats_update;
+} ixge_main_t;
+
+ixge_main_t ixge_main;
+vnet_device_class_t ixge_device_class;
+
+typedef enum
+{
+  IXGE_RX_NEXT_IP4_INPUT,
+  IXGE_RX_NEXT_IP6_INPUT,
+  IXGE_RX_NEXT_ETHERNET_INPUT,
+  IXGE_RX_NEXT_DROP,
+  IXGE_RX_N_NEXT,
+} ixge_rx_next_t;
+
+void ixge_set_next_node (ixge_rx_next_t, char *);
+
+#endif /* included_ixge_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 6ba82584..a517a597 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -44,6 +44,7 @@
  */
 
 #include 
+#include 
 
 uword
 vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
@@ -583,6 +584,11 @@ alloc_from_free_list (vlib_main_t * vm,
 
   dst = alloc_buffers;
 
+  /* wait with buffer memory allocation as long as possible
+     in case external buffer manager takes over */
+  if (PREDICT_FALSE (vm->os_physmem_alloc_aligned == 0))
+    unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
+
   n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
   if (n_filled == 0)
     return 0;
diff --git a/src/vnet.am b/src/vnet.am
index ca24c9c5..bc9655cc 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -114,14 +114,16 @@ libvnet_la_SOURCES +=				\
  vnet/ethernet/init.c				\
  vnet/ethernet/interface.c			\
  vnet/ethernet/node.c				\
- vnet/ethernet/pg.c
+ vnet/ethernet/pg.c				\
+ vnet/ethernet/sfp.c
 
 nobase_include_HEADERS +=			\
  vnet/ethernet/arp_packet.h			\
  vnet/ethernet/error.def			\
  vnet/ethernet/ethernet.h			\
  vnet/ethernet/packet.h				\
- vnet/ethernet/types.def
+ vnet/ethernet/types.def			\
+ vnet/ethernet/sfp.h
 
 ########################################
 # Layer 2 protocol: Ethernet bridging
@@ -792,14 +794,6 @@ nobase_include_HEADERS +=			\
   vnet/pg/pg.h					\
   vnet/pg/edit.h
 
-if !WITH_DPDK
-libvnet_la_SOURCES +=				\
-  vnet/devices/nic/ixge.c                       \
-  vnet/devices/nic/ixge.h                       \
-  vnet/devices/nic/sfp.c                        \
-  vnet/devices/nic/sfp.h
-endif
-
 ########################################
 # virtio
 ########################################
diff --git a/src/vnet/devices/nic/ixge.c b/src/vnet/devices/nic/ixge.c
deleted file mode 100644
index d4c4c6b7..00000000
--- a/src/vnet/devices/nic/ixge.c
+++ /dev/null
@@ -1,2938 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- *   WARNING!
- *   This driver is not intended for production use and it is unsupported.
- *   It is provided for educational use only.
- *   Please use supported DPDK driver instead.
- */
-
-#if __x86_64__
-#include 
-
-#ifndef CLIB_HAVE_VEC128
-#warning HACK: ixge driver wont really work, missing u32x4
-typedef unsigned long long u32x4;
-#endif
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define IXGE_ALWAYS_POLL 0
-
-#define EVENT_SET_FLAGS 0
-#define IXGE_HWBP_RACE_ELOG 0
-
-#define PCI_VENDOR_ID_INTEL 0x8086
-
-/* 10 GIG E (XGE) PHY IEEE 802.3 clause 45 definitions. */
-#define XGE_PHY_DEV_TYPE_PMA_PMD 1
-#define XGE_PHY_DEV_TYPE_PHY_XS 4
-#define XGE_PHY_ID1 0x2
-#define XGE_PHY_ID2 0x3
-#define XGE_PHY_CONTROL 0x0
-#define XGE_PHY_CONTROL_RESET (1 << 15)
-
-ixge_main_t ixge_main;
-static vlib_node_registration_t ixge_input_node;
-static vlib_node_registration_t ixge_process_node;
-
-static void
-ixge_semaphore_get (ixge_device_t * xd)
-{
-  ixge_main_t *xm = &ixge_main;
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_regs_t *r = xd->regs;
-  u32 i;
-
-  i = 0;
-  while (!(r->software_semaphore & (1 << 0)))
-    {
-      if (i > 0)
-	vlib_process_suspend (vm, 100e-6);
-      i++;
-    }
-  do
-    {
-      r->software_semaphore |= 1 << 1;
-    }
-  while (!(r->software_semaphore & (1 << 1)));
-}
-
-static void
-ixge_semaphore_release (ixge_device_t * xd)
-{
-  ixge_regs_t *r = xd->regs;
-  r->software_semaphore &= ~3;
-}
-
-static void
-ixge_software_firmware_sync (ixge_device_t * xd, u32 sw_mask)
-{
-  ixge_main_t *xm = &ixge_main;
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_regs_t *r = xd->regs;
-  u32 fw_mask = sw_mask << 5;
-  u32 m, done = 0;
-
-  while (!done)
-    {
-      ixge_semaphore_get (xd);
-      m = r->software_firmware_sync;
-      done = (m & fw_mask) == 0;
-      if (done)
-	r->software_firmware_sync = m | sw_mask;
-      ixge_semaphore_release (xd);
-      if (!done)
-	vlib_process_suspend (vm, 10e-3);
-    }
-}
-
-static void
-ixge_software_firmware_sync_release (ixge_device_t * xd, u32 sw_mask)
-{
-  ixge_regs_t *r = xd->regs;
-  ixge_semaphore_get (xd);
-  r->software_firmware_sync &= ~sw_mask;
-  ixge_semaphore_release (xd);
-}
-
-u32
-ixge_read_write_phy_reg (ixge_device_t * xd, u32 dev_type, u32 reg_index,
-			 u32 v, u32 is_read)
-{
-  ixge_regs_t *r = xd->regs;
-  const u32 busy_bit = 1 << 30;
-  u32 x;
-
-  ASSERT (xd->phy_index < 2);
-  ixge_software_firmware_sync (xd, 1 << (1 + xd->phy_index));
-
-  ASSERT (reg_index < (1 << 16));
-  ASSERT (dev_type < (1 << 5));
-  if (!is_read)
-    r->xge_mac.phy_data = v;
-
-  /* Address cycle. */
-  x =
-    reg_index | (dev_type << 16) | (xd->
-				    phys[xd->phy_index].mdio_address << 21);
-  r->xge_mac.phy_command = x | busy_bit;
-  /* Busy wait timed to take 28e-6 secs.  No suspend. */
-  while (r->xge_mac.phy_command & busy_bit)
-    ;
-
-  r->xge_mac.phy_command = x | ((is_read ? 2 : 1) << 26) | busy_bit;
-  while (r->xge_mac.phy_command & busy_bit)
-    ;
-
-  if (is_read)
-    v = r->xge_mac.phy_data >> 16;
-
-  ixge_software_firmware_sync_release (xd, 1 << (1 + xd->phy_index));
-
-  return v;
-}
-
-static u32
-ixge_read_phy_reg (ixge_device_t * xd, u32 dev_type, u32 reg_index)
-{
-  return ixge_read_write_phy_reg (xd, dev_type, reg_index, 0,	/* is_read */
-				  1);
-}
-
-static void
-ixge_write_phy_reg (ixge_device_t * xd, u32 dev_type, u32 reg_index, u32 v)
-{
-  (void) ixge_read_write_phy_reg (xd, dev_type, reg_index, v,	/* is_read */
-				  0);
-}
-
-static void
-ixge_i2c_put_bits (i2c_bus_t * b, int scl, int sda)
-{
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, b->private_data);
-  u32 v;
-
-  v = 0;
-  v |= (sda != 0) << 3;
-  v |= (scl != 0) << 1;
-  xd->regs->i2c_control = v;
-}
-
-static void
-ixge_i2c_get_bits (i2c_bus_t * b, int *scl, int *sda)
-{
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, b->private_data);
-  u32 v;
-
-  v = xd->regs->i2c_control;
-  *sda = (v & (1 << 2)) != 0;
-  *scl = (v & (1 << 0)) != 0;
-}
-
-static u16
-ixge_read_eeprom (ixge_device_t * xd, u32 address)
-{
-  ixge_regs_t *r = xd->regs;
-  u32 v;
-  r->eeprom_read = (( /* start bit */ (1 << 0)) | (address << 2));
-  /* Wait for done bit. */
-  while (!((v = r->eeprom_read) & (1 << 1)))
-    ;
-  return v >> 16;
-}
-
-static void
-ixge_sfp_enable_disable_laser (ixge_device_t * xd, uword enable)
-{
-  u32 tx_disable_bit = 1 << 3;
-  if (enable)
-    xd->regs->sdp_control &= ~tx_disable_bit;
-  else
-    xd->regs->sdp_control |= tx_disable_bit;
-}
-
-static void
-ixge_sfp_enable_disable_10g (ixge_device_t * xd, uword enable)
-{
-  u32 is_10g_bit = 1 << 5;
-  if (enable)
-    xd->regs->sdp_control |= is_10g_bit;
-  else
-    xd->regs->sdp_control &= ~is_10g_bit;
-}
-
-static clib_error_t *
-ixge_sfp_phy_init_from_eeprom (ixge_device_t * xd, u16 sfp_type)
-{
-  u16 a, id, reg_values_addr = 0;
-
-  a = ixge_read_eeprom (xd, 0x2b);
-  if (a == 0 || a == 0xffff)
-    return clib_error_create ("no init sequence in eeprom");
-
-  while (1)
-    {
-      id = ixge_read_eeprom (xd, ++a);
-      if (id == 0xffff)
-	break;
-      reg_values_addr = ixge_read_eeprom (xd, ++a);
-      if (id == sfp_type)
-	break;
-    }
-  if (id != sfp_type)
-    return clib_error_create ("failed to find id 0x%x", sfp_type);
-
-  ixge_software_firmware_sync (xd, 1 << 3);
-  while (1)
-    {
-      u16 v = ixge_read_eeprom (xd, ++reg_values_addr);
-      if (v == 0xffff)
-	break;
-      xd->regs->core_analog_config = v;
-    }
-  ixge_software_firmware_sync_release (xd, 1 << 3);
-
-  /* Make sure laser is off.  We'll turn on the laser when
-     the interface is brought up. */
-  ixge_sfp_enable_disable_laser (xd, /* enable */ 0);
-  ixge_sfp_enable_disable_10g (xd, /* is_10g */ 1);
-
-  return 0;
-}
-
-static void
-ixge_sfp_device_up_down (ixge_device_t * xd, uword is_up)
-{
-  u32 v;
-
-  if (is_up)
-    {
-      /* pma/pmd 10g serial SFI. */
-      xd->regs->xge_mac.auto_negotiation_control2 &= ~(3 << 16);
-      xd->regs->xge_mac.auto_negotiation_control2 |= 2 << 16;
-
-      v = xd->regs->xge_mac.auto_negotiation_control;
-      v &= ~(7 << 13);
-      v |= (0 << 13);
-      /* Restart autoneg. */
-      v |= (1 << 12);
-      xd->regs->xge_mac.auto_negotiation_control = v;
-
-      while (!(xd->regs->xge_mac.link_partner_ability[0] & 0xf0000))
-	;
-
-      v = xd->regs->xge_mac.auto_negotiation_control;
-
-      /* link mode 10g sfi serdes */
-      v &= ~(7 << 13);
-      v |= (3 << 13);
-
-      /* Restart autoneg. */
-      v |= (1 << 12);
-      xd->regs->xge_mac.auto_negotiation_control = v;
-
-      xd->regs->xge_mac.link_status;
-    }
-
-  ixge_sfp_enable_disable_laser (xd, /* enable */ is_up);
-
-  /* Give time for link partner to notice that we're up. */
-  if (is_up && vlib_in_process_context (vlib_get_main ()))
-    {
-      vlib_process_suspend (vlib_get_main (), 300e-3);
-    }
-}
-
-always_inline ixge_dma_regs_t *
-get_dma_regs (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 qi)
-{
-  ixge_regs_t *r = xd->regs;
-  ASSERT (qi < 128);
-  if (rt == VLIB_RX)
-    return qi < 64 ? &r->rx_dma0[qi] : &r->rx_dma1[qi - 64];
-  else
-    return &r->tx_dma[qi];
-}
-
-static clib_error_t *
-ixge_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
-{
-  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
-  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, hif->dev_instance);
-  ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_RX, 0);
-
-  if (is_up)
-    {
-      xd->regs->rx_enable |= 1;
-      xd->regs->tx_dma_control |= 1;
-      dr->control |= 1 << 25;
-      while (!(dr->control & (1 << 25)))
-	;
-    }
-  else
-    {
-      xd->regs->rx_enable &= ~1;
-      xd->regs->tx_dma_control &= ~1;
-    }
-
-  ixge_sfp_device_up_down (xd, is_up);
-
-  return /* no error */ 0;
-}
-
-static void
-ixge_sfp_phy_init (ixge_device_t * xd)
-{
-  ixge_phy_t *phy = xd->phys + xd->phy_index;
-  i2c_bus_t *ib = &xd->i2c_bus;
-
-  ib->private_data = xd->device_index;
-  ib->put_bits = ixge_i2c_put_bits;
-  ib->get_bits = ixge_i2c_get_bits;
-  vlib_i2c_init (ib);
-
-  vlib_i2c_read_eeprom (ib, 0x50, 0, 128, (u8 *) & xd->sfp_eeprom);
-
-  if (vlib_i2c_bus_timed_out (ib) || !sfp_eeprom_is_valid (&xd->sfp_eeprom))
-    xd->sfp_eeprom.id = SFP_ID_unknown;
-  else
-    {
-      /* FIXME 5 => SR/LR eeprom ID. */
-      clib_error_t *e =
-	ixge_sfp_phy_init_from_eeprom (xd, 5 + xd->pci_function);
-      if (e)
-	clib_error_report (e);
-    }
-
-  phy->mdio_address = ~0;
-}
-
-static void
-ixge_phy_init (ixge_device_t * xd)
-{
-  ixge_main_t *xm = &ixge_main;
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_phy_t *phy = xd->phys + xd->phy_index;
-
-  switch (xd->device_id)
-    {
-    case IXGE_82599_sfp:
-    case IXGE_82599_sfp_em:
-    case IXGE_82599_sfp_fcoe:
-      /* others? */
-      return ixge_sfp_phy_init (xd);
-
-    default:
-      break;
-    }
-
-  /* Probe address of phy. */
-  {
-    u32 i, v;
-
-    phy->mdio_address = ~0;
-    for (i = 0; i < 32; i++)
-      {
-	phy->mdio_address = i;
-	v = ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PMA_PMD, XGE_PHY_ID1);
-	if (v != 0xffff && v != 0)
-	  break;
-      }
-
-    /* No PHY found? */
-    if (i >= 32)
-      return;
-  }
-
-  phy->id =
-    ((ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PMA_PMD, XGE_PHY_ID1) << 16) |
-     ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PMA_PMD, XGE_PHY_ID2));
-
-  {
-    ELOG_TYPE_DECLARE (e) =
-    {
-    .function = (char *) __FUNCTION__,.format =
-	"ixge %d, phy id 0x%d mdio address %d",.format_args = "i4i4i4",};
-    struct
-    {
-      u32 instance, id, address;
-    } *ed;
-    ed = ELOG_DATA (&vm->elog_main, e);
-    ed->instance = xd->device_index;
-    ed->id = phy->id;
-    ed->address = phy->mdio_address;
-  }
-
-  /* Reset phy. */
-  ixge_write_phy_reg (xd, XGE_PHY_DEV_TYPE_PHY_XS, XGE_PHY_CONTROL,
-		      XGE_PHY_CONTROL_RESET);
-
-  /* Wait for self-clearning reset bit to clear. */
-  do
-    {
-      vlib_process_suspend (vm, 1e-3);
-    }
-  while (ixge_read_phy_reg (xd, XGE_PHY_DEV_TYPE_PHY_XS, XGE_PHY_CONTROL) &
-	 XGE_PHY_CONTROL_RESET);
-}
-
-static u8 *
-format_ixge_rx_from_hw_descriptor (u8 * s, va_list * va)
-{
-  ixge_rx_from_hw_descriptor_t *d =
-    va_arg (*va, ixge_rx_from_hw_descriptor_t *);
-  u32 s0 = d->status[0], s2 = d->status[2];
-  u32 is_ip4, is_ip6, is_ip, is_tcp, is_udp;
-  uword indent = format_get_indent (s);
-
-  s = format (s, "%s-owned",
-	      (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE) ? "sw" :
-	      "hw");
-  s =
-    format (s, ", length this descriptor %d, l3 offset %d",
-	    d->n_packet_bytes_this_descriptor,
-	    IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s0));
-  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET)
-    s = format (s, ", end-of-packet");
-
-  s = format (s, "\n%U", format_white_space, indent);
-
-  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_ETHERNET_ERROR)
-    s = format (s, "layer2 error");
-
-  if (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_LAYER2)
-    {
-      s = format (s, "layer 2 type %d", (s0 & 0x1f));
-      return s;
-    }
-
-  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_VLAN)
-    s = format (s, "vlan header 0x%x\n%U", d->vlan_tag,
-		format_white_space, indent);
-
-  if ((is_ip4 = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4)))
-    {
-      s = format (s, "ip4%s",
-		  (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4_EXT) ? " options" :
-		  "");
-      if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED)
-	s = format (s, " checksum %s",
-		    (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR) ?
-		    "bad" : "ok");
-    }
-  if ((is_ip6 = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6)))
-    s = format (s, "ip6%s",
-		(s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6_EXT) ? " extended" :
-		"");
-  is_tcp = is_udp = 0;
-  if ((is_ip = (is_ip4 | is_ip6)))
-    {
-      is_tcp = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_TCP) != 0;
-      is_udp = (s0 & IXGE_RX_DESCRIPTOR_STATUS0_IS_UDP) != 0;
-      if (is_tcp)
-	s = format (s, ", tcp");
-      if (is_udp)
-	s = format (s, ", udp");
-    }
-
-  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED)
-    s = format (s, ", tcp checksum %s",
-		(s2 & IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR) ? "bad" :
-		"ok");
-  if (s2 & IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED)
-    s = format (s, ", udp checksum %s",
-		(s2 & IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR) ? "bad" :
-		"ok");
-
-  return s;
-}
-
-static u8 *
-format_ixge_tx_descriptor (u8 * s, va_list * va)
-{
-  ixge_tx_descriptor_t *d = va_arg (*va, ixge_tx_descriptor_t *);
-  u32 s0 = d->status0, s1 = d->status1;
-  uword indent = format_get_indent (s);
-  u32 v;
-
-  s = format (s, "buffer 0x%Lx, %d packet bytes, %d bytes this buffer",
-	      d->buffer_address, s1 >> 14, d->n_bytes_this_buffer);
-
-  s = format (s, "\n%U", format_white_space, indent);
-
-  if ((v = (s0 >> 0) & 3))
-    s = format (s, "reserved 0x%x, ", v);
-
-  if ((v = (s0 >> 2) & 3))
-    s = format (s, "mac 0x%x, ", v);
-
-  if ((v = (s0 >> 4) & 0xf) != 3)
-    s = format (s, "type 0x%x, ", v);
-
-  s = format (s, "%s%s%s%s%s%s%s%s",
-	      (s0 & (1 << 8)) ? "eop, " : "",
-	      (s0 & (1 << 9)) ? "insert-fcs, " : "",
-	      (s0 & (1 << 10)) ? "reserved26, " : "",
-	      (s0 & (1 << 11)) ? "report-status, " : "",
-	      (s0 & (1 << 12)) ? "reserved28, " : "",
-	      (s0 & (1 << 13)) ? "is-advanced, " : "",
-	      (s0 & (1 << 14)) ? "vlan-enable, " : "",
-	      (s0 & (1 << 15)) ? "tx-segmentation, " : "");
-
-  if ((v = s1 & 0xf) != 0)
-    s = format (s, "status 0x%x, ", v);
-
-  if ((v = (s1 >> 4) & 0xf))
-    s = format (s, "context 0x%x, ", v);
-
-  if ((v = (s1 >> 8) & 0x3f))
-    s = format (s, "options 0x%x, ", v);
-
-  return s;
-}
-
-typedef struct
-{
-  ixge_descriptor_t before, after;
-
-  u32 buffer_index;
-
-  u16 device_index;
-
-  u8 queue_index;
-
-  u8 is_start_of_packet;
-
-  /* Copy of VLIB buffer; packet data stored in pre_data. */
-  vlib_buffer_t buffer;
-} ixge_rx_dma_trace_t;
-
-static u8 *
-format_ixge_rx_dma_trace (u8 * s, va_list * va)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
-  vlib_node_t *node = va_arg (*va, vlib_node_t *);
-  vnet_main_t *vnm = vnet_get_main ();
-  ixge_rx_dma_trace_t *t = va_arg (*va, ixge_rx_dma_trace_t *);
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, t->device_index);
-  format_function_t *f;
-  uword indent = format_get_indent (s);
-
-  {
-    vnet_sw_interface_t *sw =
-      vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
-    s =
-      format (s, "%U rx queue %d", format_vnet_sw_interface_name, vnm, sw,
-	      t->queue_index);
-  }
-
-  s = format (s, "\n%Ubefore: %U",
-	      format_white_space, indent,
-	      format_ixge_rx_from_hw_descriptor, &t->before);
-  s = format (s, "\n%Uafter : head/tail address 0x%Lx/0x%Lx",
-	      format_white_space, indent,
-	      t->after.rx_to_hw.head_address, t->after.rx_to_hw.tail_address);
-
-  s = format (s, "\n%Ubuffer 0x%x: %U",
-	      format_white_space, indent,
-	      t->buffer_index, format_vlib_buffer, &t->buffer);
-
-  s = format (s, "\n%U", format_white_space, indent);
-
-  f = node->format_buffer;
-  if (!f || !t->is_start_of_packet)
-    f = format_hex_bytes;
-  s = format (s, "%U", f, t->buffer.pre_data, sizeof (t->buffer.pre_data));
-
-  return s;
-}
-
-#define foreach_ixge_error					\
-  _ (none, "no error")						\
-  _ (tx_full_drops, "tx ring full drops")			\
-  _ (ip4_checksum_error, "ip4 checksum errors")			\
-  _ (rx_alloc_fail, "rx buf alloc from free list failed")	\
-  _ (rx_alloc_no_physmem, "rx buf alloc failed no physmem")
-
-typedef enum
-{
-#define _(f,s) IXGE_ERROR_##f,
-  foreach_ixge_error
-#undef _
-    IXGE_N_ERROR,
-} ixge_error_t;
-
-always_inline void
-ixge_rx_next_and_error_from_status_x1 (ixge_device_t * xd,
-				       u32 s00, u32 s02,
-				       u8 * next0, u8 * error0, u32 * flags0)
-{
-  u8 is0_ip4, is0_ip6, n0, e0;
-  u32 f0;
-
-  e0 = IXGE_ERROR_none;
-  n0 = IXGE_RX_NEXT_ETHERNET_INPUT;
-
-  is0_ip4 = s02 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED;
-  n0 = is0_ip4 ? IXGE_RX_NEXT_IP4_INPUT : n0;
-
-  e0 = (is0_ip4 && (s02 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR)
-	? IXGE_ERROR_ip4_checksum_error : e0);
-
-  is0_ip6 = s00 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6;
-  n0 = is0_ip6 ? IXGE_RX_NEXT_IP6_INPUT : n0;
-
-  n0 = (xd->per_interface_next_index != ~0) ?
-    xd->per_interface_next_index : n0;
-
-  /* Check for error. */
-  n0 = e0 != IXGE_ERROR_none ? IXGE_RX_NEXT_DROP : n0;
-
-  f0 = ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED
-		| IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED))
-	? IP_BUFFER_L4_CHECKSUM_COMPUTED : 0);
-
-  f0 |= ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR
-		 | IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR))
-	 ? 0 : IP_BUFFER_L4_CHECKSUM_CORRECT);
-
-  *error0 = e0;
-  *next0 = n0;
-  *flags0 = f0;
-}
-
-always_inline void
-ixge_rx_next_and_error_from_status_x2 (ixge_device_t * xd,
-				       u32 s00, u32 s02,
-				       u32 s10, u32 s12,
-				       u8 * next0, u8 * error0, u32 * flags0,
-				       u8 * next1, u8 * error1, u32 * flags1)
-{
-  u8 is0_ip4, is0_ip6, n0, e0;
-  u8 is1_ip4, is1_ip6, n1, e1;
-  u32 f0, f1;
-
-  e0 = e1 = IXGE_ERROR_none;
-  n0 = n1 = IXGE_RX_NEXT_IP4_INPUT;
-
-  is0_ip4 = s02 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED;
-  is1_ip4 = s12 & IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED;
-
-  n0 = is0_ip4 ? IXGE_RX_NEXT_IP4_INPUT : n0;
-  n1 = is1_ip4 ? IXGE_RX_NEXT_IP4_INPUT : n1;
-
-  e0 = (is0_ip4 && (s02 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR)
-	? IXGE_ERROR_ip4_checksum_error : e0);
-  e1 = (is1_ip4 && (s12 & IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR)
-	? IXGE_ERROR_ip4_checksum_error : e1);
-
-  is0_ip6 = s00 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6;
-  is1_ip6 = s10 & IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6;
-
-  n0 = is0_ip6 ? IXGE_RX_NEXT_IP6_INPUT : n0;
-  n1 = is1_ip6 ? IXGE_RX_NEXT_IP6_INPUT : n1;
-
-  n0 = (xd->per_interface_next_index != ~0) ?
-    xd->per_interface_next_index : n0;
-  n1 = (xd->per_interface_next_index != ~0) ?
-    xd->per_interface_next_index : n1;
-
-  /* Check for error. */
-  n0 = e0 != IXGE_ERROR_none ? IXGE_RX_NEXT_DROP : n0;
-  n1 = e1 != IXGE_ERROR_none ? IXGE_RX_NEXT_DROP : n1;
-
-  *error0 = e0;
-  *error1 = e1;
-
-  *next0 = n0;
-  *next1 = n1;
-
-  f0 = ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED
-		| IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED))
-	? IP_BUFFER_L4_CHECKSUM_COMPUTED : 0);
-  f1 = ((s12 & (IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED
-		| IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED))
-	? IP_BUFFER_L4_CHECKSUM_COMPUTED : 0);
-
-  f0 |= ((s02 & (IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR
-		 | IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR))
-	 ? 0 : IP_BUFFER_L4_CHECKSUM_CORRECT);
-  f1 |= ((s12 & (IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR
-		 | IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR))
-	 ? 0 : IP_BUFFER_L4_CHECKSUM_CORRECT);
-
-  *flags0 = f0;
-  *flags1 = f1;
-}
-
-static void
-ixge_rx_trace (ixge_main_t * xm,
-	       ixge_device_t * xd,
-	       ixge_dma_queue_t * dq,
-	       ixge_descriptor_t * before_descriptors,
-	       u32 * before_buffers,
-	       ixge_descriptor_t * after_descriptors, uword n_descriptors)
-{
-  vlib_main_t *vm = xm->vlib_main;
-  vlib_node_runtime_t *node = dq->rx.node;
-  ixge_rx_from_hw_descriptor_t *bd;
-  ixge_rx_to_hw_descriptor_t *ad;
-  u32 *b, n_left, is_sop, next_index_sop;
-
-  n_left = n_descriptors;
-  b = before_buffers;
-  bd = &before_descriptors->rx_from_hw;
-  ad = &after_descriptors->rx_to_hw;
-  is_sop = dq->rx.is_start_of_packet;
-  next_index_sop = dq->rx.saved_start_of_packet_next_index;
-
-  while (n_left >= 2)
-    {
-      u32 bi0, bi1, flags0, flags1;
-      vlib_buffer_t *b0, *b1;
-      ixge_rx_dma_trace_t *t0, *t1;
-      u8 next0, error0, next1, error1;
-
-      bi0 = b[0];
-      bi1 = b[1];
-      n_left -= 2;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      b1 = vlib_get_buffer (vm, bi1);
-
-      ixge_rx_next_and_error_from_status_x2 (xd,
-					     bd[0].status[0], bd[0].status[2],
-					     bd[1].status[0], bd[1].status[2],
-					     &next0, &error0, &flags0,
-					     &next1, &error1, &flags1);
-
-      next_index_sop = is_sop ? next0 : next_index_sop;
-      vlib_trace_buffer (vm, node, next_index_sop, b0, /* follow_chain */ 0);
-      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
-      t0->is_start_of_packet = is_sop;
-      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      next_index_sop = is_sop ? next1 : next_index_sop;
-      vlib_trace_buffer (vm, node, next_index_sop, b1, /* follow_chain */ 0);
-      t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
-      t1->is_start_of_packet = is_sop;
-      is_sop = (b1->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      t0->queue_index = dq->queue_index;
-      t1->queue_index = dq->queue_index;
-      t0->device_index = xd->device_index;
-      t1->device_index = xd->device_index;
-      t0->before.rx_from_hw = bd[0];
-      t1->before.rx_from_hw = bd[1];
-      t0->after.rx_to_hw = ad[0];
-      t1->after.rx_to_hw = ad[1];
-      t0->buffer_index = bi0;
-      t1->buffer_index = bi1;
-      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
-      memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b0->pre_data));
-      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
-	      sizeof (t0->buffer.pre_data));
-      memcpy (t1->buffer.pre_data, b1->data + b1->current_data,
-	      sizeof (t1->buffer.pre_data));
-
-      b += 2;
-      bd += 2;
-      ad += 2;
-    }
-
-  while (n_left >= 1)
-    {
-      u32 bi0, flags0;
-      vlib_buffer_t *b0;
-      ixge_rx_dma_trace_t *t0;
-      u8 next0, error0;
-
-      bi0 = b[0];
-      n_left -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-
-      ixge_rx_next_and_error_from_status_x1 (xd,
-					     bd[0].status[0], bd[0].status[2],
-					     &next0, &error0, &flags0);
-
-      next_index_sop = is_sop ? next0 : next_index_sop;
-      vlib_trace_buffer (vm, node, next_index_sop, b0, /* follow_chain */ 0);
-      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
-      t0->is_start_of_packet = is_sop;
-      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      t0->queue_index = dq->queue_index;
-      t0->device_index = xd->device_index;
-      t0->before.rx_from_hw = bd[0];
-      t0->after.rx_to_hw = ad[0];
-      t0->buffer_index = bi0;
-      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
-      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
-	      sizeof (t0->buffer.pre_data));
-
-      b += 1;
-      bd += 1;
-      ad += 1;
-    }
-}
-
-typedef struct
-{
-  ixge_tx_descriptor_t descriptor;
-
-  u32 buffer_index;
-
-  u16 device_index;
-
-  u8 queue_index;
-
-  u8 is_start_of_packet;
-
-  /* Copy of VLIB buffer; packet data stored in pre_data. */
-  vlib_buffer_t buffer;
-} ixge_tx_dma_trace_t;
-
-static u8 *
-format_ixge_tx_dma_trace (u8 * s, va_list * va)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
-  ixge_tx_dma_trace_t *t = va_arg (*va, ixge_tx_dma_trace_t *);
-  vnet_main_t *vnm = vnet_get_main ();
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, t->device_index);
-  format_function_t *f;
-  uword indent = format_get_indent (s);
-
-  {
-    vnet_sw_interface_t *sw =
-      vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
-    s =
-      format (s, "%U tx queue %d", format_vnet_sw_interface_name, vnm, sw,
-	      t->queue_index);
-  }
-
-  s = format (s, "\n%Udescriptor: %U",
-	      format_white_space, indent,
-	      format_ixge_tx_descriptor, &t->descriptor);
-
-  s = format (s, "\n%Ubuffer 0x%x: %U",
-	      format_white_space, indent,
-	      t->buffer_index, format_vlib_buffer, &t->buffer);
-
-  s = format (s, "\n%U", format_white_space, indent);
-
-  f = format_ethernet_header_with_length;
-  if (!f || !t->is_start_of_packet)
-    f = format_hex_bytes;
-  s = format (s, "%U", f, t->buffer.pre_data, sizeof (t->buffer.pre_data));
-
-  return s;
-}
-
-typedef struct
-{
-  vlib_node_runtime_t *node;
-
-  u32 is_start_of_packet;
-
-  u32 n_bytes_in_packet;
-
-  ixge_tx_descriptor_t *start_of_packet_descriptor;
-} ixge_tx_state_t;
-
-static void
-ixge_tx_trace (ixge_main_t * xm,
-	       ixge_device_t * xd,
-	       ixge_dma_queue_t * dq,
-	       ixge_tx_state_t * tx_state,
-	       ixge_tx_descriptor_t * descriptors,
-	       u32 * buffers, uword n_descriptors)
-{
-  vlib_main_t *vm = xm->vlib_main;
-  vlib_node_runtime_t *node = tx_state->node;
-  ixge_tx_descriptor_t *d;
-  u32 *b, n_left, is_sop;
-
-  n_left = n_descriptors;
-  b = buffers;
-  d = descriptors;
-  is_sop = tx_state->is_start_of_packet;
-
-  while (n_left >= 2)
-    {
-      u32 bi0, bi1;
-      vlib_buffer_t *b0, *b1;
-      ixge_tx_dma_trace_t *t0, *t1;
-
-      bi0 = b[0];
-      bi1 = b[1];
-      n_left -= 2;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      b1 = vlib_get_buffer (vm, bi1);
-
-      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
-      t0->is_start_of_packet = is_sop;
-      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
-      t1->is_start_of_packet = is_sop;
-      is_sop = (b1->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      t0->queue_index = dq->queue_index;
-      t1->queue_index = dq->queue_index;
-      t0->device_index = xd->device_index;
-      t1->device_index = xd->device_index;
-      t0->descriptor = d[0];
-      t1->descriptor = d[1];
-      t0->buffer_index = bi0;
-      t1->buffer_index = bi1;
-      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
-      memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b0->pre_data));
-      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
-	      sizeof (t0->buffer.pre_data));
-      memcpy (t1->buffer.pre_data, b1->data + b1->current_data,
-	      sizeof (t1->buffer.pre_data));
-
-      b += 2;
-      d += 2;
-    }
-
-  while (n_left >= 1)
-    {
-      u32 bi0;
-      vlib_buffer_t *b0;
-      ixge_tx_dma_trace_t *t0;
-
-      bi0 = b[0];
-      n_left -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-
-      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
-      t0->is_start_of_packet = is_sop;
-      is_sop = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      t0->queue_index = dq->queue_index;
-      t0->device_index = xd->device_index;
-      t0->descriptor = d[0];
-      t0->buffer_index = bi0;
-      memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
-      memcpy (t0->buffer.pre_data, b0->data + b0->current_data,
-	      sizeof (t0->buffer.pre_data));
-
-      b += 1;
-      d += 1;
-    }
-}
-
-always_inline uword
-ixge_ring_sub (ixge_dma_queue_t * q, u32 i0, u32 i1)
-{
-  i32 d = i1 - i0;
-  ASSERT (i0 < q->n_descriptors);
-  ASSERT (i1 < q->n_descriptors);
-  return d < 0 ? q->n_descriptors + d : d;
-}
-
-always_inline uword
-ixge_ring_add (ixge_dma_queue_t * q, u32 i0, u32 i1)
-{
-  u32 d = i0 + i1;
-  ASSERT (i0 < q->n_descriptors);
-  ASSERT (i1 < q->n_descriptors);
-  d -= d >= q->n_descriptors ? q->n_descriptors : 0;
-  return d;
-}
-
-always_inline uword
-ixge_tx_descriptor_matches_template (ixge_main_t * xm,
-				     ixge_tx_descriptor_t * d)
-{
-  u32 cmp;
-
-  cmp = ((d->status0 & xm->tx_descriptor_template_mask.status0)
-	 ^ xm->tx_descriptor_template.status0);
-  if (cmp)
-    return 0;
-  cmp = ((d->status1 & xm->tx_descriptor_template_mask.status1)
-	 ^ xm->tx_descriptor_template.status1);
-  if (cmp)
-    return 0;
-
-  return 1;
-}
-
-static uword
-ixge_tx_no_wrap (ixge_main_t * xm,
-		 ixge_device_t * xd,
-		 ixge_dma_queue_t * dq,
-		 u32 * buffers,
-		 u32 start_descriptor_index,
-		 u32 n_descriptors, ixge_tx_state_t * tx_state)
-{
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_tx_descriptor_t *d, *d_sop;
-  u32 n_left = n_descriptors;
-  u32 *to_free = vec_end (xm->tx_buffers_pending_free);
-  u32 *to_tx =
-    vec_elt_at_index (dq->descriptor_buffer_indices, start_descriptor_index);
-  u32 is_sop = tx_state->is_start_of_packet;
-  u32 len_sop = tx_state->n_bytes_in_packet;
-  u16 template_status = xm->tx_descriptor_template.status0;
-  u32 descriptor_prefetch_rotor = 0;
-
-  ASSERT (start_descriptor_index + n_descriptors <= dq->n_descriptors);
-  d = &dq->descriptors[start_descriptor_index].tx;
-  d_sop = is_sop ? d : tx_state->start_of_packet_descriptor;
-
-  while (n_left >= 4)
-    {
-      vlib_buffer_t *b0, *b1;
-      u32 bi0, fi0, len0;
-      u32 bi1, fi1, len1;
-      u8 is_eop0, is_eop1;
-
-      /* Prefetch next iteration. */
-      vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
-      vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
-
-      if ((descriptor_prefetch_rotor & 0x3) == 0)
-	CLIB_PREFETCH (d + 4, CLIB_CACHE_LINE_BYTES, STORE);
-
-      descriptor_prefetch_rotor += 2;
-
-      bi0 = buffers[0];
-      bi1 = buffers[1];
-
-      to_free[0] = fi0 = to_tx[0];
-      to_tx[0] = bi0;
-      to_free += fi0 != 0;
-
-      to_free[0] = fi1 = to_tx[1];
-      to_tx[1] = bi1;
-      to_free += fi1 != 0;
-
-      buffers += 2;
-      n_left -= 2;
-      to_tx += 2;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      b1 = vlib_get_buffer (vm, bi1);
-
-      is_eop0 = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-      is_eop1 = (b1->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      len0 = b0->current_length;
-      len1 = b1->current_length;
-
-      ASSERT (ixge_tx_descriptor_matches_template (xm, d + 0));
-      ASSERT (ixge_tx_descriptor_matches_template (xm, d + 1));
-
-      d[0].buffer_address =
-	vlib_get_buffer_data_physical_address (vm, bi0) + b0->current_data;
-      d[1].buffer_address =
-	vlib_get_buffer_data_physical_address (vm, bi1) + b1->current_data;
-
-      d[0].n_bytes_this_buffer = len0;
-      d[1].n_bytes_this_buffer = len1;
-
-      d[0].status0 =
-	template_status | (is_eop0 <<
-			   IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET);
-      d[1].status0 =
-	template_status | (is_eop1 <<
-			   IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET);
-
-      len_sop = (is_sop ? 0 : len_sop) + len0;
-      d_sop[0].status1 =
-	IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET (len_sop);
-      d += 1;
-      d_sop = is_eop0 ? d : d_sop;
-
-      is_sop = is_eop0;
-
-      len_sop = (is_sop ? 0 : len_sop) + len1;
-      d_sop[0].status1 =
-	IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET (len_sop);
-      d += 1;
-      d_sop = is_eop1 ? d : d_sop;
-
-      is_sop = is_eop1;
-    }
-
-  while (n_left > 0)
-    {
-      vlib_buffer_t *b0;
-      u32 bi0, fi0, len0;
-      u8 is_eop0;
-
-      bi0 = buffers[0];
-
-      to_free[0] = fi0 = to_tx[0];
-      to_tx[0] = bi0;
-      to_free += fi0 != 0;
-
-      buffers += 1;
-      n_left -= 1;
-      to_tx += 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-
-      is_eop0 = (b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0;
-
-      len0 = b0->current_length;
-
-      ASSERT (ixge_tx_descriptor_matches_template (xm, d + 0));
-
-      d[0].buffer_address =
-	vlib_get_buffer_data_physical_address (vm, bi0) + b0->current_data;
-
-      d[0].n_bytes_this_buffer = len0;
-
-      d[0].status0 =
-	template_status | (is_eop0 <<
-			   IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET);
-
-      len_sop = (is_sop ? 0 : len_sop) + len0;
-      d_sop[0].status1 =
-	IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET (len_sop);
-      d += 1;
-      d_sop = is_eop0 ? d : d_sop;
-
-      is_sop = is_eop0;
-    }
-
-  if (tx_state->node->flags & VLIB_NODE_FLAG_TRACE)
-    {
-      to_tx =
-	vec_elt_at_index (dq->descriptor_buffer_indices,
-			  start_descriptor_index);
-      ixge_tx_trace (xm, xd, dq, tx_state,
-		     &dq->descriptors[start_descriptor_index].tx, to_tx,
-		     n_descriptors);
-    }
-
-  _vec_len (xm->tx_buffers_pending_free) =
-    to_free - xm->tx_buffers_pending_free;
-
-  /* When we are done d_sop can point to end of ring.  Wrap it if so. */
-  {
-    ixge_tx_descriptor_t *d_start = &dq->descriptors[0].tx;
-
-    ASSERT (d_sop - d_start <= dq->n_descriptors);
-    d_sop = d_sop - d_start == dq->n_descriptors ? d_start : d_sop;
-  }
-
-  tx_state->is_start_of_packet = is_sop;
-  tx_state->start_of_packet_descriptor = d_sop;
-  tx_state->n_bytes_in_packet = len_sop;
-
-  return n_descriptors;
-}
-
-static uword
-ixge_interface_tx (vlib_main_t * vm,
-		   vlib_node_runtime_t * node, vlib_frame_t * f)
-{
-  ixge_main_t *xm = &ixge_main;
-  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, rd->dev_instance);
-  ixge_dma_queue_t *dq;
-  u32 *from, n_left_tx, n_descriptors_to_tx, n_tail_drop;
-  u32 queue_index = 0;		/* fixme parameter */
-  ixge_tx_state_t tx_state;
-
-  tx_state.node = node;
-  tx_state.is_start_of_packet = 1;
-  tx_state.start_of_packet_descriptor = 0;
-  tx_state.n_bytes_in_packet = 0;
-
-  from = vlib_frame_vector_args (f);
-
-  dq = vec_elt_at_index (xd->dma_queues[VLIB_TX], queue_index);
-
-  dq->head_index = dq->tx.head_index_write_back[0];
-
-  /* Since head == tail means ring is empty we can send up to dq->n_descriptors - 1. */
-  n_left_tx = dq->n_descriptors - 1;
-  n_left_tx -= ixge_ring_sub (dq, dq->head_index, dq->tail_index);
-
-  _vec_len (xm->tx_buffers_pending_free) = 0;
-
-  n_descriptors_to_tx = f->n_vectors;
-  n_tail_drop = 0;
-  if (PREDICT_FALSE (n_descriptors_to_tx > n_left_tx))
-    {
-      i32 i, n_ok, i_eop, i_sop;
-
-      i_sop = i_eop = ~0;
-      for (i = n_left_tx - 1; i >= 0; i--)
-	{
-	  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
-	  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
-	    {
-	      if (i_sop != ~0 && i_eop != ~0)
-		break;
-	      i_eop = i;
-	      i_sop = i + 1;
-	    }
-	}
-      if (i == 0)
-	n_ok = 0;
-      else
-	n_ok = i_eop + 1;
-
-      {
-	ELOG_TYPE_DECLARE (e) =
-	{
-	.function = (char *) __FUNCTION__,.format =
-	    "ixge %d, ring full to tx %d head %d tail %d",.format_args =
-	    "i2i2i2i2",};
-	struct
-	{
-	  u16 instance, to_tx, head, tail;
-	} *ed;
-	ed = ELOG_DATA (&vm->elog_main, e);
-	ed->instance = xd->device_index;
-	ed->to_tx = n_descriptors_to_tx;
-	ed->head = dq->head_index;
-	ed->tail = dq->tail_index;
-      }
-
-      if (n_ok < n_descriptors_to_tx)
-	{
-	  n_tail_drop = n_descriptors_to_tx - n_ok;
-	  vec_add (xm->tx_buffers_pending_free, from + n_ok, n_tail_drop);
-	  vlib_error_count (vm, ixge_input_node.index,
-			    IXGE_ERROR_tx_full_drops, n_tail_drop);
-	}
-
-      n_descriptors_to_tx = n_ok;
-    }
-
-  dq->tx.n_buffers_on_ring += n_descriptors_to_tx;
-
-  /* Process from tail to end of descriptor ring. */
-  if (n_descriptors_to_tx > 0 && dq->tail_index < dq->n_descriptors)
-    {
-      u32 n =
-	clib_min (dq->n_descriptors - dq->tail_index, n_descriptors_to_tx);
-      n = ixge_tx_no_wrap (xm, xd, dq, from, dq->tail_index, n, &tx_state);
-      from += n;
-      n_descriptors_to_tx -= n;
-      dq->tail_index += n;
-      ASSERT (dq->tail_index <= dq->n_descriptors);
-      if (dq->tail_index == dq->n_descriptors)
-	dq->tail_index = 0;
-    }
-
-  if (n_descriptors_to_tx > 0)
-    {
-      u32 n =
-	ixge_tx_no_wrap (xm, xd, dq, from, 0, n_descriptors_to_tx, &tx_state);
-      from += n;
-      ASSERT (n == n_descriptors_to_tx);
-      dq->tail_index += n;
-      ASSERT (dq->tail_index <= dq->n_descriptors);
-      if (dq->tail_index == dq->n_descriptors)
-	dq->tail_index = 0;
-    }
-
-  /* We should only get full packets. */
-  ASSERT (tx_state.is_start_of_packet);
-
-  /* Report status when last descriptor is done. */
-  {
-    u32 i = dq->tail_index == 0 ? dq->n_descriptors - 1 : dq->tail_index - 1;
-    ixge_tx_descriptor_t *d = &dq->descriptors[i].tx;
-    d->status0 |= IXGE_TX_DESCRIPTOR_STATUS0_REPORT_STATUS;
-  }
-
-  /* Give new descriptors to hardware. */
-  {
-    ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_TX, queue_index);
-
-    CLIB_MEMORY_BARRIER ();
-
-    dr->tail_index = dq->tail_index;
-  }
-
-  /* Free any buffers that are done. */
-  {
-    u32 n = _vec_len (xm->tx_buffers_pending_free);
-    if (n > 0)
-      {
-	vlib_buffer_free_no_next (vm, xm->tx_buffers_pending_free, n);
-	_vec_len (xm->tx_buffers_pending_free) = 0;
-	ASSERT (dq->tx.n_buffers_on_ring >= n);
-	dq->tx.n_buffers_on_ring -= (n - n_tail_drop);
-      }
-  }
-
-  return f->n_vectors;
-}
-
-static uword
-ixge_rx_queue_no_wrap (ixge_main_t * xm,
-		       ixge_device_t * xd,
-		       ixge_dma_queue_t * dq,
-		       u32 start_descriptor_index, u32 n_descriptors)
-{
-  vlib_main_t *vm = xm->vlib_main;
-  vlib_node_runtime_t *node = dq->rx.node;
-  ixge_descriptor_t *d;
-  static ixge_descriptor_t *d_trace_save;
-  static u32 *d_trace_buffers;
-  u32 n_descriptors_left = n_descriptors;
-  u32 *to_rx =
-    vec_elt_at_index (dq->descriptor_buffer_indices, start_descriptor_index);
-  u32 *to_add;
-  u32 bi_sop = dq->rx.saved_start_of_packet_buffer_index;
-  u32 bi_last = dq->rx.saved_last_buffer_index;
-  u32 next_index_sop = dq->rx.saved_start_of_packet_next_index;
-  u32 is_sop = dq->rx.is_start_of_packet;
-  u32 next_index, n_left_to_next, *to_next;
-  u32 n_packets = 0;
-  u32 n_bytes = 0;
-  u32 n_trace = vlib_get_trace_count (vm, node);
-  vlib_buffer_t *b_last, b_dummy;
-
-  ASSERT (start_descriptor_index + n_descriptors <= dq->n_descriptors);
-  d = &dq->descriptors[start_descriptor_index];
-
-  b_last = bi_last != ~0 ? vlib_get_buffer (vm, bi_last) : &b_dummy;
-  next_index = dq->rx.next_index;
-
-  if (n_trace > 0)
-    {
-      u32 n = clib_min (n_trace, n_descriptors);
-      if (d_trace_save)
-	{
-	  _vec_len (d_trace_save) = 0;
-	  _vec_len (d_trace_buffers) = 0;
-	}
-      vec_add (d_trace_save, (ixge_descriptor_t *) d, n);
-      vec_add (d_trace_buffers, to_rx, n);
-    }
-
-  {
-    uword l = vec_len (xm->rx_buffers_to_add);
-
-    if (l < n_descriptors_left)
-      {
-	u32 n_to_alloc = 2 * dq->n_descriptors - l;
-	u32 n_allocated;
-
-	vec_resize (xm->rx_buffers_to_add, n_to_alloc);
-
-	_vec_len (xm->rx_buffers_to_add) = l;
-	n_allocated = vlib_buffer_alloc_from_free_list
-	  (vm, xm->rx_buffers_to_add + l, n_to_alloc,
-	   xm->vlib_buffer_free_list_index);
-	_vec_len (xm->rx_buffers_to_add) += n_allocated;
-
-	/* Handle transient allocation failure */
-	if (PREDICT_FALSE (l + n_allocated <= n_descriptors_left))
-	  {
-	    if (n_allocated == 0)
-	      vlib_error_count (vm, ixge_input_node.index,
-				IXGE_ERROR_rx_alloc_no_physmem, 1);
-	    else
-	      vlib_error_count (vm, ixge_input_node.index,
-				IXGE_ERROR_rx_alloc_fail, 1);
-
-	    n_descriptors_left = l + n_allocated;
-	  }
-	n_descriptors = n_descriptors_left;
-      }
-
-    /* Add buffers from end of vector going backwards. */
-    to_add = vec_end (xm->rx_buffers_to_add) - 1;
-  }
-
-  while (n_descriptors_left > 0)
-    {
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_descriptors_left >= 4 && n_left_to_next >= 2)
-	{
-	  vlib_buffer_t *b0, *b1;
-	  u32 bi0, fi0, len0, l3_offset0, s20, s00, flags0;
-	  u32 bi1, fi1, len1, l3_offset1, s21, s01, flags1;
-	  u8 is_eop0, error0, next0;
-	  u8 is_eop1, error1, next1;
-	  ixge_descriptor_t d0, d1;
-
-	  vlib_prefetch_buffer_with_index (vm, to_rx[2], STORE);
-	  vlib_prefetch_buffer_with_index (vm, to_rx[3], STORE);
-
-	  CLIB_PREFETCH (d + 2, 32, STORE);
-
-	  d0.as_u32x4 = d[0].as_u32x4;
-	  d1.as_u32x4 = d[1].as_u32x4;
-
-	  s20 = d0.rx_from_hw.status[2];
-	  s21 = d1.rx_from_hw.status[2];
-
-	  s00 = d0.rx_from_hw.status[0];
-	  s01 = d1.rx_from_hw.status[0];
-
-	  if (!
-	      ((s20 & s21) & IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE))
-	    goto found_hw_owned_descriptor_x2;
-
-	  bi0 = to_rx[0];
-	  bi1 = to_rx[1];
-
-	  ASSERT (to_add - 1 >= xm->rx_buffers_to_add);
-	  fi0 = to_add[0];
-	  fi1 = to_add[-1];
-
-	  to_rx[0] = fi0;
-	  to_rx[1] = fi1;
-	  to_rx += 2;
-	  to_add -= 2;
-
-	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
-		  vlib_buffer_is_known (vm, bi0));
-	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
-		  vlib_buffer_is_known (vm, bi1));
-	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
-		  vlib_buffer_is_known (vm, fi0));
-	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
-		  vlib_buffer_is_known (vm, fi1));
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-	  /*
-	   * Turn this on if you run into
-	   * "bad monkey" contexts, and you want to know exactly
-	   * which nodes they've visited... See main.c...
-	   */
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
-
-	  CLIB_PREFETCH (b0->data, CLIB_CACHE_LINE_BYTES, LOAD);
-	  CLIB_PREFETCH (b1->data, CLIB_CACHE_LINE_BYTES, LOAD);
-
-	  is_eop0 = (s20 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET) != 0;
-	  is_eop1 = (s21 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET) != 0;
-
-	  ixge_rx_next_and_error_from_status_x2 (xd, s00, s20, s01, s21,
-						 &next0, &error0, &flags0,
-						 &next1, &error1, &flags1);
-
-	  next0 = is_sop ? next0 : next_index_sop;
-	  next1 = is_eop0 ? next1 : next0;
-	  next_index_sop = next1;
-
-	  b0->flags |= flags0 | (!is_eop0 << VLIB_BUFFER_LOG2_NEXT_PRESENT);
-	  b1->flags |= flags1 | (!is_eop1 << VLIB_BUFFER_LOG2_NEXT_PRESENT);
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b1)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-
-	  b0->error = node->errors[error0];
-	  b1->error = node->errors[error1];
-
-	  len0 = d0.rx_from_hw.n_packet_bytes_this_descriptor;
-	  len1 = d1.rx_from_hw.n_packet_bytes_this_descriptor;
-	  n_bytes += len0 + len1;
-	  n_packets += is_eop0 + is_eop1;
-
-	  /* Give new buffers to hardware. */
-	  d0.rx_to_hw.tail_address =
-	    vlib_get_buffer_data_physical_address (vm, fi0);
-	  d1.rx_to_hw.tail_address =
-	    vlib_get_buffer_data_physical_address (vm, fi1);
-	  d0.rx_to_hw.head_address = d[0].rx_to_hw.tail_address;
-	  d1.rx_to_hw.head_address = d[1].rx_to_hw.tail_address;
-	  d[0].as_u32x4 = d0.as_u32x4;
-	  d[1].as_u32x4 = d1.as_u32x4;
-
-	  d += 2;
-	  n_descriptors_left -= 2;
-
-	  /* Point to either l2 or l3 header depending on next. */
-	  l3_offset0 = (is_sop && (next0 != IXGE_RX_NEXT_ETHERNET_INPUT))
-	    ? IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s00) : 0;
-	  l3_offset1 = (is_eop0 && (next1 != IXGE_RX_NEXT_ETHERNET_INPUT))
-	    ? IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s01) : 0;
-
-	  b0->current_length = len0 - l3_offset0;
-	  b1->current_length = len1 - l3_offset1;
-	  b0->current_data = l3_offset0;
-	  b1->current_data = l3_offset1;
-
-	  b_last->next_buffer = is_sop ? ~0 : bi0;
-	  b0->next_buffer = is_eop0 ? ~0 : bi1;
-	  bi_last = bi1;
-	  b_last = b1;
-
-	  if (CLIB_DEBUG > 0)
-	    {
-	      u32 bi_sop0 = is_sop ? bi0 : bi_sop;
-	      u32 bi_sop1 = is_eop0 ? bi1 : bi_sop0;
-
-	      if (is_eop0)
-		{
-		  u8 *msg = vlib_validate_buffer (vm, bi_sop0,
-						  /* follow_buffer_next */ 1);
-		  ASSERT (!msg);
-		}
-	      if (is_eop1)
-		{
-		  u8 *msg = vlib_validate_buffer (vm, bi_sop1,
-						  /* follow_buffer_next */ 1);
-		  ASSERT (!msg);
-		}
-	    }
-	  if (0)		/* "Dave" version */
-	    {
-	      u32 bi_sop0 = is_sop ? bi0 : bi_sop;
-	      u32 bi_sop1 = is_eop0 ? bi1 : bi_sop0;
-
-	      if (is_eop0)
-		{
-		  to_next[0] = bi_sop0;
-		  to_next++;
-		  n_left_to_next--;
-
-		  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-						   to_next, n_left_to_next,
-						   bi_sop0, next0);
-		}
-	      if (is_eop1)
-		{
-		  to_next[0] = bi_sop1;
-		  to_next++;
-		  n_left_to_next--;
-
-		  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-						   to_next, n_left_to_next,
-						   bi_sop1, next1);
-		}
-	      is_sop = is_eop1;
-	      bi_sop = bi_sop1;
-	    }
-	  if (1)		/* "Eliot" version */
-	    {
-	      /* Speculatively enqueue to cached next. */
-	      u8 saved_is_sop = is_sop;
-	      u32 bi_sop_save = bi_sop;
-
-	      bi_sop = saved_is_sop ? bi0 : bi_sop;
-	      to_next[0] = bi_sop;
-	      to_next += is_eop0;
-	      n_left_to_next -= is_eop0;
-
-	      bi_sop = is_eop0 ? bi1 : bi_sop;
-	      to_next[0] = bi_sop;
-	      to_next += is_eop1;
-	      n_left_to_next -= is_eop1;
-
-	      is_sop = is_eop1;
-
-	      if (PREDICT_FALSE
-		  (!(next0 == next_index && next1 == next_index)))
-		{
-		  /* Undo speculation. */
-		  to_next -= is_eop0 + is_eop1;
-		  n_left_to_next += is_eop0 + is_eop1;
-
-		  /* Re-do both descriptors being careful about where we enqueue. */
-		  bi_sop = saved_is_sop ? bi0 : bi_sop_save;
-		  if (is_eop0)
-		    {
-		      if (next0 != next_index)
-			vlib_set_next_frame_buffer (vm, node, next0, bi_sop);
-		      else
-			{
-			  to_next[0] = bi_sop;
-			  to_next += 1;
-			  n_left_to_next -= 1;
-			}
-		    }
-
-		  bi_sop = is_eop0 ? bi1 : bi_sop;
-		  if (is_eop1)
-		    {
-		      if (next1 != next_index)
-			vlib_set_next_frame_buffer (vm, node, next1, bi_sop);
-		      else
-			{
-			  to_next[0] = bi_sop;
-			  to_next += 1;
-			  n_left_to_next -= 1;
-			}
-		    }
-
-		  /* Switch cached next index when next for both packets is the same. */
-		  if (is_eop0 && is_eop1 && next0 == next1)
-		    {
-		      vlib_put_next_frame (vm, node, next_index,
-					   n_left_to_next);
-		      next_index = next0;
-		      vlib_get_next_frame (vm, node, next_index,
-					   to_next, n_left_to_next);
-		    }
-		}
-	    }
-	}
-
-      /* Bail out of dual loop and proceed with single loop. */
-    found_hw_owned_descriptor_x2:
-
-      while (n_descriptors_left > 0 && n_left_to_next > 0)
-	{
-	  vlib_buffer_t *b0;
-	  u32 bi0, fi0, len0, l3_offset0, s20, s00, flags0;
-	  u8 is_eop0, error0, next0;
-	  ixge_descriptor_t d0;
-
-	  d0.as_u32x4 = d[0].as_u32x4;
-
-	  s20 = d0.rx_from_hw.status[2];
-	  s00 = d0.rx_from_hw.status[0];
-
-	  if (!(s20 & IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE))
-	    goto found_hw_owned_descriptor_x1;
-
-	  bi0 = to_rx[0];
-	  ASSERT (to_add >= xm->rx_buffers_to_add);
-	  fi0 = to_add[0];
-
-	  to_rx[0] = fi0;
-	  to_rx += 1;
-	  to_add -= 1;
-
-	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
-		  vlib_buffer_is_known (vm, bi0));
-	  ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED ==
-		  vlib_buffer_is_known (vm, fi0));
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  /*
-	   * Turn this on if you run into
-	   * "bad monkey" contexts, and you want to know exactly
-	   * which nodes they've visited...
-	   */
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-
-	  is_eop0 = (s20 & IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET) != 0;
-	  ixge_rx_next_and_error_from_status_x1
-	    (xd, s00, s20, &next0, &error0, &flags0);
-
-	  next0 = is_sop ? next0 : next_index_sop;
-	  next_index_sop = next0;
-
-	  b0->flags |= flags0 | (!is_eop0 << VLIB_BUFFER_LOG2_NEXT_PRESENT);
-
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-
-	  b0->error = node->errors[error0];
-
-	  len0 = d0.rx_from_hw.n_packet_bytes_this_descriptor;
-	  n_bytes += len0;
-	  n_packets += is_eop0;
-
-	  /* Give new buffer to hardware. */
-	  d0.rx_to_hw.tail_address =
-	    vlib_get_buffer_data_physical_address (vm, fi0);
-	  d0.rx_to_hw.head_address = d0.rx_to_hw.tail_address;
-	  d[0].as_u32x4 = d0.as_u32x4;
-
-	  d += 1;
-	  n_descriptors_left -= 1;
-
-	  /* Point to either l2 or l3 header depending on next. */
-	  l3_offset0 = (is_sop && (next0 != IXGE_RX_NEXT_ETHERNET_INPUT))
-	    ? IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET (s00) : 0;
-	  b0->current_length = len0 - l3_offset0;
-	  b0->current_data = l3_offset0;
-
-	  b_last->next_buffer = is_sop ? ~0 : bi0;
-	  bi_last = bi0;
-	  b_last = b0;
-
-	  bi_sop = is_sop ? bi0 : bi_sop;
-
-	  if (CLIB_DEBUG > 0 && is_eop0)
-	    {
-	      u8 *msg =
-		vlib_validate_buffer (vm, bi_sop, /* follow_buffer_next */ 1);
-	      ASSERT (!msg);
-	    }
-
-	  if (0)		/* "Dave" version */
-	    {
-	      if (is_eop0)
-		{
-		  to_next[0] = bi_sop;
-		  to_next++;
-		  n_left_to_next--;
-
-		  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-						   to_next, n_left_to_next,
-						   bi_sop, next0);
-		}
-	    }
-	  if (1)		/* "Eliot" version */
-	    {
-	      if (PREDICT_TRUE (next0 == next_index))
-		{
-		  to_next[0] = bi_sop;
-		  to_next += is_eop0;
-		  n_left_to_next -= is_eop0;
-		}
-	      else
-		{
-		  if (next0 != next_index && is_eop0)
-		    vlib_set_next_frame_buffer (vm, node, next0, bi_sop);
-
-		  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-		  next_index = next0;
-		  vlib_get_next_frame (vm, node, next_index,
-				       to_next, n_left_to_next);
-		}
-	    }
-	  is_sop = is_eop0;
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-found_hw_owned_descriptor_x1:
-  if (n_descriptors_left > 0)
-    vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-
-  _vec_len (xm->rx_buffers_to_add) = (to_add + 1) - xm->rx_buffers_to_add;
-
-  {
-    u32 n_done = n_descriptors - n_descriptors_left;
-
-    if (n_trace > 0 && n_done > 0)
-      {
-	u32 n = clib_min (n_trace, n_done);
-	ixge_rx_trace (xm, xd, dq,
-		       d_trace_save,
-		       d_trace_buffers,
-		       &dq->descriptors[start_descriptor_index], n);
-	vlib_set_trace_count (vm, node, n_trace - n);
-      }
-    if (d_trace_save)
-      {
-	_vec_len (d_trace_save) = 0;
-	_vec_len (d_trace_buffers) = 0;
-      }
-
-    /* Don't keep a reference to b_last if we don't have to.
-       Otherwise we can over-write a next_buffer pointer after already haven
-       enqueued a packet. */
-    if (is_sop)
-      {
-	b_last->next_buffer = ~0;
-	bi_last = ~0;
-      }
-
-    dq->rx.n_descriptors_done_this_call = n_done;
-    dq->rx.n_descriptors_done_total += n_done;
-    dq->rx.is_start_of_packet = is_sop;
-    dq->rx.saved_start_of_packet_buffer_index = bi_sop;
-    dq->rx.saved_last_buffer_index = bi_last;
-    dq->rx.saved_start_of_packet_next_index = next_index_sop;
-    dq->rx.next_index = next_index;
-    dq->rx.n_bytes += n_bytes;
-
-    return n_packets;
-  }
-}
-
-static uword
-ixge_rx_queue (ixge_main_t * xm,
-	       ixge_device_t * xd,
-	       vlib_node_runtime_t * node, u32 queue_index)
-{
-  ixge_dma_queue_t *dq =
-    vec_elt_at_index (xd->dma_queues[VLIB_RX], queue_index);
-  ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_RX, dq->queue_index);
-  uword n_packets = 0;
-  u32 hw_head_index, sw_head_index;
-
-  /* One time initialization. */
-  if (!dq->rx.node)
-    {
-      dq->rx.node = node;
-      dq->rx.is_start_of_packet = 1;
-      dq->rx.saved_start_of_packet_buffer_index = ~0;
-      dq->rx.saved_last_buffer_index = ~0;
-    }
-
-  dq->rx.next_index = node->cached_next_index;
-
-  dq->rx.n_descriptors_done_total = 0;
-  dq->rx.n_descriptors_done_this_call = 0;
-  dq->rx.n_bytes = 0;
-
-  /* Fetch head from hardware and compare to where we think we are. */
-  hw_head_index = dr->head_index;
-  sw_head_index = dq->head_index;
-
-  if (hw_head_index == sw_head_index)
-    goto done;
-
-  if (hw_head_index < sw_head_index)
-    {
-      u32 n_tried = dq->n_descriptors - sw_head_index;
-      n_packets += ixge_rx_queue_no_wrap (xm, xd, dq, sw_head_index, n_tried);
-      sw_head_index =
-	ixge_ring_add (dq, sw_head_index,
-		       dq->rx.n_descriptors_done_this_call);
-
-      if (dq->rx.n_descriptors_done_this_call != n_tried)
-	goto done;
-    }
-  if (hw_head_index >= sw_head_index)
-    {
-      u32 n_tried = hw_head_index - sw_head_index;
-      n_packets += ixge_rx_queue_no_wrap (xm, xd, dq, sw_head_index, n_tried);
-      sw_head_index =
-	ixge_ring_add (dq, sw_head_index,
-		       dq->rx.n_descriptors_done_this_call);
-    }
-
-done:
-  dq->head_index = sw_head_index;
-  dq->tail_index =
-    ixge_ring_add (dq, dq->tail_index, dq->rx.n_descriptors_done_total);
-
-  /* Give tail back to hardware. */
-  CLIB_MEMORY_BARRIER ();
-
-  dr->tail_index = dq->tail_index;
-
-  vlib_increment_combined_counter (vnet_main.
-				   interface_main.combined_sw_if_counters +
-				   VNET_INTERFACE_COUNTER_RX,
-				   0 /* cpu_index */ ,
-				   xd->vlib_sw_if_index, n_packets,
-				   dq->rx.n_bytes);
-
-  return n_packets;
-}
-
-static void
-ixge_interrupt (ixge_main_t * xm, ixge_device_t * xd, u32 i)
-{
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_regs_t *r = xd->regs;
-
-  if (i != 20)
-    {
-      ELOG_TYPE_DECLARE (e) =
-      {
-	.function = (char *) __FUNCTION__,.format =
-	  "ixge %d, %s",.format_args = "i1t1",.n_enum_strings =
-	  16,.enum_strings =
-	{
-      "flow director",
-	    "rx miss",
-	    "pci exception",
-	    "mailbox",
-	    "link status change",
-	    "linksec key exchange",
-	    "manageability event",
-	    "reserved23",
-	    "sdp0",
-	    "sdp1",
-	    "sdp2",
-	    "sdp3",
-	    "ecc", "descriptor handler error", "tcp timer", "other",},};
-      struct
-      {
-	u8 instance;
-	u8 index;
-      } *ed;
-      ed = ELOG_DATA (&vm->elog_main, e);
-      ed->instance = xd->device_index;
-      ed->index = i - 16;
-    }
-  else
-    {
-      u32 v = r->xge_mac.link_status;
-      uword is_up = (v & (1 << 30)) != 0;
-
-      ELOG_TYPE_DECLARE (e) =
-      {
-      .function = (char *) __FUNCTION__,.format =
-	  "ixge %d, link status change 0x%x",.format_args = "i4i4",};
-      struct
-      {
-	u32 instance, link_status;
-      } *ed;
-      ed = ELOG_DATA (&vm->elog_main, e);
-      ed->instance = xd->device_index;
-      ed->link_status = v;
-      xd->link_status_at_last_link_change = v;
-
-      vlib_process_signal_event (vm, ixge_process_node.index,
-				 EVENT_SET_FLAGS,
-				 ((is_up << 31) | xd->vlib_hw_if_index));
-    }
-}
-
-always_inline u32
-clean_block (u32 * b, u32 * t, u32 n_left)
-{
-  u32 *t0 = t;
-
-  while (n_left >= 4)
-    {
-      u32 bi0, bi1, bi2, bi3;
-
-      t[0] = bi0 = b[0];
-      b[0] = 0;
-      t += bi0 != 0;
-
-      t[0] = bi1 = b[1];
-      b[1] = 0;
-      t += bi1 != 0;
-
-      t[0] = bi2 = b[2];
-      b[2] = 0;
-      t += bi2 != 0;
-
-      t[0] = bi3 = b[3];
-      b[3] = 0;
-      t += bi3 != 0;
-
-      b += 4;
-      n_left -= 4;
-    }
-
-  while (n_left > 0)
-    {
-      u32 bi0;
-
-      t[0] = bi0 = b[0];
-      b[0] = 0;
-      t += bi0 != 0;
-      b += 1;
-      n_left -= 1;
-    }
-
-  return t - t0;
-}
-
-static void
-ixge_tx_queue (ixge_main_t * xm, ixge_device_t * xd, u32 queue_index)
-{
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_dma_queue_t *dq =
-    vec_elt_at_index (xd->dma_queues[VLIB_TX], queue_index);
-  u32 n_clean, *b, *t, *t0;
-  i32 n_hw_owned_descriptors;
-  i32 first_to_clean, last_to_clean;
-  u64 hwbp_race = 0;
-
-  /* Handle case where head write back pointer update
-   * arrives after the interrupt during high PCI bus loads.
-   */
-  while ((dq->head_index == dq->tx.head_index_write_back[0]) &&
-	 dq->tx.n_buffers_on_ring && (dq->head_index != dq->tail_index))
-    {
-      hwbp_race++;
-      if (IXGE_HWBP_RACE_ELOG && (hwbp_race == 1))
-	{
-	  ELOG_TYPE_DECLARE (e) =
-	  {
-	  .function = (char *) __FUNCTION__,.format =
-	      "ixge %d tx head index race: head %4d, tail %4d, buffs %4d",.format_args
-	      = "i4i4i4i4",};
-	  struct
-	  {
-	    u32 instance, head_index, tail_index, n_buffers_on_ring;
-	  } *ed;
-	  ed = ELOG_DATA (&vm->elog_main, e);
-	  ed->instance = xd->device_index;
-	  ed->head_index = dq->head_index;
-	  ed->tail_index = dq->tail_index;
-	  ed->n_buffers_on_ring = dq->tx.n_buffers_on_ring;
-	}
-    }
-
-  dq->head_index = dq->tx.head_index_write_back[0];
-  n_hw_owned_descriptors = ixge_ring_sub (dq, dq->head_index, dq->tail_index);
-  ASSERT (dq->tx.n_buffers_on_ring >= n_hw_owned_descriptors);
-  n_clean = dq->tx.n_buffers_on_ring - n_hw_owned_descriptors;
-
-  if (IXGE_HWBP_RACE_ELOG && hwbp_race)
-    {
-      ELOG_TYPE_DECLARE (e) =
-      {
-      .function = (char *) __FUNCTION__,.format =
-	  "ixge %d tx head index race: head %4d, hw_owned %4d, n_clean %4d, retries %d",.format_args
-	  = "i4i4i4i4i4",};
-      struct
-      {
-	u32 instance, head_index, n_hw_owned_descriptors, n_clean, retries;
-      } *ed;
-      ed = ELOG_DATA (&vm->elog_main, e);
-      ed->instance = xd->device_index;
-      ed->head_index = dq->head_index;
-      ed->n_hw_owned_descriptors = n_hw_owned_descriptors;
-      ed->n_clean = n_clean;
-      ed->retries = hwbp_race;
-    }
-
-  /*
-   * This function used to wait until hardware owned zero descriptors.
-   * At high PPS rates, that doesn't happen until the TX ring is
-   * completely full of descriptors which need to be cleaned up.
-   * That, in turn, causes TX ring-full drops and/or long RX service
-   * interruptions.
-   */
-  if (n_clean == 0)
-    return;
-
-  /* Clean the n_clean descriptors prior to the reported hardware head */
-  last_to_clean = dq->head_index - 1;
-  last_to_clean = (last_to_clean < 0) ? last_to_clean + dq->n_descriptors :
-    last_to_clean;
-
-  first_to_clean = (last_to_clean) - (n_clean - 1);
-  first_to_clean = (first_to_clean < 0) ? first_to_clean + dq->n_descriptors :
-    first_to_clean;
-
-  vec_resize (xm->tx_buffers_pending_free, dq->n_descriptors - 1);
-  t0 = t = xm->tx_buffers_pending_free;
-  b = dq->descriptor_buffer_indices + first_to_clean;
-
-  /* Wrap case: clean from first to end, then start to last */
-  if (first_to_clean > last_to_clean)
-    {
-      t += clean_block (b, t, (dq->n_descriptors - 1) - first_to_clean);
-      first_to_clean = 0;
-      b = dq->descriptor_buffer_indices;
-    }
-
-  /* Typical case: clean from first to last */
-  if (first_to_clean <= last_to_clean)
-    t += clean_block (b, t, (last_to_clean - first_to_clean) + 1);
-
-  if (t > t0)
-    {
-      u32 n = t - t0;
-      vlib_buffer_free_no_next (vm, t0, n);
-      ASSERT (dq->tx.n_buffers_on_ring >= n);
-      dq->tx.n_buffers_on_ring -= n;
-      _vec_len (xm->tx_buffers_pending_free) = 0;
-    }
-}
-
-/* RX queue interrupts 0 thru 7; TX 8 thru 15. */
-always_inline uword
-ixge_interrupt_is_rx_queue (uword i)
-{
-  return i < 8;
-}
-
-always_inline uword
-ixge_interrupt_is_tx_queue (uword i)
-{
-  return i >= 8 && i < 16;
-}
-
-always_inline uword
-ixge_tx_queue_to_interrupt (uword i)
-{
-  return 8 + i;
-}
-
-always_inline uword
-ixge_rx_queue_to_interrupt (uword i)
-{
-  return 0 + i;
-}
-
-always_inline uword
-ixge_interrupt_rx_queue (uword i)
-{
-  ASSERT (ixge_interrupt_is_rx_queue (i));
-  return i - 0;
-}
-
-always_inline uword
-ixge_interrupt_tx_queue (uword i)
-{
-  ASSERT (ixge_interrupt_is_tx_queue (i));
-  return i - 8;
-}
-
-static uword
-ixge_device_input (ixge_main_t * xm,
-		   ixge_device_t * xd, vlib_node_runtime_t * node)
-{
-  ixge_regs_t *r = xd->regs;
-  u32 i, s;
-  uword n_rx_packets = 0;
-
-  s = r->interrupt.status_write_1_to_set;
-  if (s)
-    r->interrupt.status_write_1_to_clear = s;
-
-  /* *INDENT-OFF* */
-  foreach_set_bit (i, s, ({
-    if (ixge_interrupt_is_rx_queue (i))
-      n_rx_packets += ixge_rx_queue (xm, xd, node, ixge_interrupt_rx_queue (i));
-
-    else if (ixge_interrupt_is_tx_queue (i))
-      ixge_tx_queue (xm, xd, ixge_interrupt_tx_queue (i));
-
-    else
-      ixge_interrupt (xm, xd, i);
-  }));
-  /* *INDENT-ON* */
-
-  return n_rx_packets;
-}
-
-static uword
-ixge_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f)
-{
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd;
-  uword n_rx_packets = 0;
-
-  if (node->state == VLIB_NODE_STATE_INTERRUPT)
-    {
-      uword i;
-
-      /* Loop over devices with interrupts. */
-      /* *INDENT-OFF* */
-      foreach_set_bit (i, node->runtime_data[0], ({
-	xd = vec_elt_at_index (xm->devices, i);
-	n_rx_packets += ixge_device_input (xm, xd, node);
-
-	/* Re-enable interrupts since we're going to stay in interrupt mode. */
-	if (! (node->flags & VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
-	  xd->regs->interrupt.enable_write_1_to_set = ~0;
-      }));
-      /* *INDENT-ON* */
-
-      /* Clear mask of devices with pending interrupts. */
-      node->runtime_data[0] = 0;
-    }
-  else
-    {
-      /* Poll all devices for input/interrupts. */
-      vec_foreach (xd, xm->devices)
-      {
-	n_rx_packets += ixge_device_input (xm, xd, node);
-
-	/* Re-enable interrupts when switching out of polling mode. */
-	if (node->flags &
-	    VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)
-	  xd->regs->interrupt.enable_write_1_to_set = ~0;
-      }
-    }
-
-  return n_rx_packets;
-}
-
-static char *ixge_error_strings[] = {
-#define _(n,s) s,
-  foreach_ixge_error
-#undef _
-};
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (ixge_input_node, static) = {
-  .function = ixge_input,
-  .type = VLIB_NODE_TYPE_INPUT,
-  .name = "ixge-input",
-
-  /* Will be enabled if/when hardware is detected. */
-  .state = VLIB_NODE_STATE_DISABLED,
-
-  .format_buffer = format_ethernet_header_with_length,
-  .format_trace = format_ixge_rx_dma_trace,
-
-  .n_errors = IXGE_N_ERROR,
-  .error_strings = ixge_error_strings,
-
-  .n_next_nodes = IXGE_RX_N_NEXT,
-  .next_nodes = {
-    [IXGE_RX_NEXT_DROP] = "error-drop",
-    [IXGE_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
-    [IXGE_RX_NEXT_IP4_INPUT] = "ip4-input",
-    [IXGE_RX_NEXT_IP6_INPUT] = "ip6-input",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH_CLONE (ixge_input)
-CLIB_MULTIARCH_SELECT_FN (ixge_input)
-/* *INDENT-ON* */
-
-static u8 *
-format_ixge_device_name (u8 * s, va_list * args)
-{
-  u32 i = va_arg (*args, u32);
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, i);
-  return format (s, "TenGigabitEthernet%U",
-		 format_vlib_pci_handle, &xd->pci_device.bus_address);
-}
-
-#define IXGE_COUNTER_IS_64_BIT (1 << 0)
-#define IXGE_COUNTER_NOT_CLEAR_ON_READ (1 << 1)
-
-static u8 ixge_counter_flags[] = {
-#define _(a,f) 0,
-#define _64(a,f) IXGE_COUNTER_IS_64_BIT,
-  foreach_ixge_counter
-#undef _
-#undef _64
-};
-
-static void
-ixge_update_counters (ixge_device_t * xd)
-{
-  /* Byte offset for counter registers. */
-  static u32 reg_offsets[] = {
-#define _(a,f) (a) / sizeof (u32),
-#define _64(a,f) _(a,f)
-    foreach_ixge_counter
-#undef _
-#undef _64
-  };
-  volatile u32 *r = (volatile u32 *) xd->regs;
-  int i;
-
-  for (i = 0; i < ARRAY_LEN (xd->counters); i++)
-    {
-      u32 o = reg_offsets[i];
-      xd->counters[i] += r[o];
-      if (ixge_counter_flags[i] & IXGE_COUNTER_NOT_CLEAR_ON_READ)
-	r[o] = 0;
-      if (ixge_counter_flags[i] & IXGE_COUNTER_IS_64_BIT)
-	xd->counters[i] += (u64) r[o + 1] << (u64) 32;
-    }
-}
-
-static u8 *
-format_ixge_device_id (u8 * s, va_list * args)
-{
-  u32 device_id = va_arg (*args, u32);
-  char *t = 0;
-  switch (device_id)
-    {
-#define _(f,n) case n: t = #f; break;
-      foreach_ixge_pci_device_id;
-#undef _
-    default:
-      t = 0;
-      break;
-    }
-  if (t == 0)
-    s = format (s, "unknown 0x%x", device_id);
-  else
-    s = format (s, "%s", t);
-  return s;
-}
-
-static u8 *
-format_ixge_link_status (u8 * s, va_list * args)
-{
-  ixge_device_t *xd = va_arg (*args, ixge_device_t *);
-  u32 v = xd->link_status_at_last_link_change;
-
-  s = format (s, "%s", (v & (1 << 30)) ? "up" : "down");
-
-  {
-    char *modes[] = {
-      "1g", "10g parallel", "10g serial", "autoneg",
-    };
-    char *speeds[] = {
-      "unknown", "100m", "1g", "10g",
-    };
-    s = format (s, ", mode %s, speed %s",
-		modes[(v >> 26) & 3], speeds[(v >> 28) & 3]);
-  }
-
-  return s;
-}
-
-static u8 *
-format_ixge_device (u8 * s, va_list * args)
-{
-  u32 dev_instance = va_arg (*args, u32);
-  CLIB_UNUSED (int verbose) = va_arg (*args, int);
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, dev_instance);
-  ixge_phy_t *phy = xd->phys + xd->phy_index;
-  uword indent = format_get_indent (s);
-
-  ixge_update_counters (xd);
-  xd->link_status_at_last_link_change = xd->regs->xge_mac.link_status;
-
-  s = format (s, "Intel 8259X: id %U\n%Ulink %U",
-	      format_ixge_device_id, xd->device_id,
-	      format_white_space, indent + 2, format_ixge_link_status, xd);
-
-  {
-
-    s = format (s, "\n%UPCIe %U", format_white_space, indent + 2,
-		format_vlib_pci_link_speed, &xd->pci_device);
-  }
-
-  s = format (s, "\n%U", format_white_space, indent + 2);
-  if (phy->mdio_address != ~0)
-    s = format (s, "PHY address %d, id 0x%x", phy->mdio_address, phy->id);
-  else if (xd->sfp_eeprom.id == SFP_ID_sfp)
-    s = format (s, "SFP %U", format_sfp_eeprom, &xd->sfp_eeprom);
-  else
-    s = format (s, "PHY not found");
-
-  /* FIXME */
-  {
-    ixge_dma_queue_t *dq = vec_elt_at_index (xd->dma_queues[VLIB_RX], 0);
-    ixge_dma_regs_t *dr = get_dma_regs (xd, VLIB_RX, 0);
-    u32 hw_head_index = dr->head_index;
-    u32 sw_head_index = dq->head_index;
-    u32 nitems;
-
-    nitems = ixge_ring_sub (dq, hw_head_index, sw_head_index);
-    s = format (s, "\n%U%d unprocessed, %d total buffers on rx queue 0 ring",
-		format_white_space, indent + 2, nitems, dq->n_descriptors);
-
-    s = format (s, "\n%U%d buffers in driver rx cache",
-		format_white_space, indent + 2,
-		vec_len (xm->rx_buffers_to_add));
-
-    s = format (s, "\n%U%d buffers on tx queue 0 ring",
-		format_white_space, indent + 2,
-		xd->dma_queues[VLIB_TX][0].tx.n_buffers_on_ring);
-  }
-  {
-    u32 i;
-    u64 v;
-    static char *names[] = {
-#define _(a,f) #f,
-#define _64(a,f) _(a,f)
-      foreach_ixge_counter
-#undef _
-#undef _64
-    };
-
-    for (i = 0; i < ARRAY_LEN (names); i++)
-      {
-	v = xd->counters[i] - xd->counters_last_clear[i];
-	if (v != 0)
-	  s = format (s, "\n%U%-40U%16Ld",
-		      format_white_space, indent + 2,
-		      format_c_identifier, names[i], v);
-      }
-  }
-
-  return s;
-}
-
-static void
-ixge_clear_hw_interface_counters (u32 instance)
-{
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, instance);
-  ixge_update_counters (xd);
-  memcpy (xd->counters_last_clear, xd->counters, sizeof (xd->counters));
-}
-
-/*
- * Dynamically redirect all pkts from a specific interface
- * to the specified node
- */
-static void
-ixge_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
-			      u32 node_index)
-{
-  ixge_main_t *xm = &ixge_main;
-  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
-  ixge_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
-
-  /* Shut off redirection */
-  if (node_index == ~0)
-    {
-      xd->per_interface_next_index = node_index;
-      return;
-    }
-
-  xd->per_interface_next_index =
-    vlib_node_add_next (xm->vlib_main, ixge_input_node.index, node_index);
-}
-
-
-/* *INDENT-OFF* */
-VNET_DEVICE_CLASS (ixge_device_class) = {
-  .name = "ixge",
-  .tx_function = ixge_interface_tx,
-  .format_device_name = format_ixge_device_name,
-  .format_device = format_ixge_device,
-  .format_tx_trace = format_ixge_tx_dma_trace,
-  .clear_counters = ixge_clear_hw_interface_counters,
-  .admin_up_down_function = ixge_interface_admin_up_down,
-  .rx_redirect_to_node = ixge_set_interface_next_node,
-  .flatten_output_chains = 1,
-};
-/* *INDENT-ON* */
-
-#define IXGE_N_BYTES_IN_RX_BUFFER  (2048)	// DAW-HACK: Set Rx buffer size so all packets < ETH_MTU_SIZE fit in the buffer (i.e. sop & eop for all descriptors).
-
-static clib_error_t *
-ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index)
-{
-  ixge_main_t *xm = &ixge_main;
-  vlib_main_t *vm = xm->vlib_main;
-  ixge_dma_queue_t *dq;
-  clib_error_t *error = 0;
-
-  vec_validate (xd->dma_queues[rt], queue_index);
-  dq = vec_elt_at_index (xd->dma_queues[rt], queue_index);
-
-  if (!xm->n_descriptors_per_cache_line)
-    xm->n_descriptors_per_cache_line =
-      CLIB_CACHE_LINE_BYTES / sizeof (dq->descriptors[0]);
-
-  if (!xm->n_bytes_in_rx_buffer)
-    xm->n_bytes_in_rx_buffer = IXGE_N_BYTES_IN_RX_BUFFER;
-  xm->n_bytes_in_rx_buffer = round_pow2 (xm->n_bytes_in_rx_buffer, 1024);
-  if (!xm->vlib_buffer_free_list_index)
-    {
-      xm->vlib_buffer_free_list_index =
-	vlib_buffer_get_or_create_free_list (vm, xm->n_bytes_in_rx_buffer,
-					     "ixge rx");
-      ASSERT (xm->vlib_buffer_free_list_index != 0);
-    }
-
-  if (!xm->n_descriptors[rt])
-    xm->n_descriptors[rt] = 4 * VLIB_FRAME_SIZE;
-
-  dq->queue_index = queue_index;
-  dq->n_descriptors =
-    round_pow2 (xm->n_descriptors[rt], xm->n_descriptors_per_cache_line);
-  dq->head_index = dq->tail_index = 0;
-
-  dq->descriptors = vlib_physmem_alloc_aligned (vm, &error,
-						dq->n_descriptors *
-						sizeof (dq->descriptors[0]),
-						128 /* per chip spec */ );
-  if (error)
-    return error;
-
-  memset (dq->descriptors, 0,
-	  dq->n_descriptors * sizeof (dq->descriptors[0]));
-  vec_resize (dq->descriptor_buffer_indices, dq->n_descriptors);
-
-  if (rt == VLIB_RX)
-    {
-      u32 n_alloc, i;
-
-      n_alloc = vlib_buffer_alloc_from_free_list
-	(vm, dq->descriptor_buffer_indices,
-	 vec_len (dq->descriptor_buffer_indices),
-	 xm->vlib_buffer_free_list_index);
-      ASSERT (n_alloc == vec_len (dq->descriptor_buffer_indices));
-      for (i = 0; i < n_alloc; i++)
-	{
-	  vlib_buffer_t *b =
-	    vlib_get_buffer (vm, dq->descriptor_buffer_indices[i]);
-	  dq->descriptors[i].rx_to_hw.tail_address =
-	    vlib_physmem_virtual_to_physical (vm, b->data);
-	}
-    }
-  else
-    {
-      u32 i;
-
-      dq->tx.head_index_write_back =
-	vlib_physmem_alloc (vm, &error, CLIB_CACHE_LINE_BYTES);
-
-      for (i = 0; i < dq->n_descriptors; i++)
-	dq->descriptors[i].tx = xm->tx_descriptor_template;
-
-      vec_validate (xm->tx_buffers_pending_free, dq->n_descriptors - 1);
-    }
-
-  {
-    ixge_dma_regs_t *dr = get_dma_regs (xd, rt, queue_index);
-    u64 a;
-
-    a = vlib_physmem_virtual_to_physical (vm, dq->descriptors);
-    dr->descriptor_address[0] = a & 0xFFFFFFFF;
-    dr->descriptor_address[1] = a >> (u64) 32;
-    dr->n_descriptor_bytes = dq->n_descriptors * sizeof (dq->descriptors[0]);
-    dq->head_index = dq->tail_index = 0;
-
-    if (rt == VLIB_RX)
-      {
-	ASSERT ((xm->n_bytes_in_rx_buffer / 1024) < 32);
-	dr->rx_split_control =
-	  ( /* buffer size */ ((xm->n_bytes_in_rx_buffer / 1024) << 0)
-	   | (			/* lo free descriptor threshold (units of 64 descriptors) */
-	       (1 << 22)) | (	/* descriptor type: advanced one buffer */
-			      (1 << 25)) | (	/* drop if no descriptors available */
-					     (1 << 28)));
-
-	/* Give hardware all but last 16 cache lines' worth of descriptors. */
-	dq->tail_index = dq->n_descriptors -
-	  16 * xm->n_descriptors_per_cache_line;
-      }
-    else
-      {
-	/* Make sure its initialized before hardware can get to it. */
-	dq->tx.head_index_write_back[0] = dq->head_index;
-
-	a =
-	  vlib_physmem_virtual_to_physical (vm, dq->tx.head_index_write_back);
-	dr->tx.head_index_write_back_address[0] = /* enable bit */ 1 | a;
-	dr->tx.head_index_write_back_address[1] = (u64) a >> (u64) 32;
-      }
-
-    /* DMA on 82599 does not work with [13] rx data write relaxed ordering
-       and [12] undocumented set. */
-    if (rt == VLIB_RX)
-      dr->dca_control &= ~((1 << 13) | (1 << 12));
-
-    CLIB_MEMORY_BARRIER ();
-
-    if (rt == VLIB_TX)
-      {
-	xd->regs->tx_dma_control |= (1 << 0);
-	dr->control |= ((32 << 0)	/* prefetch threshold */
-			| (64 << 8)	/* host threshold */
-			| (0 << 16) /* writeback threshold */ );
-      }
-
-    /* Enable this queue and wait for hardware to initialize
-       before adding to tail. */
-    if (rt == VLIB_TX)
-      {
-	dr->control |= 1 << 25;
-	while (!(dr->control & (1 << 25)))
-	  ;
-      }
-
-    /* Set head/tail indices and enable DMA. */
-    dr->head_index = dq->head_index;
-    dr->tail_index = dq->tail_index;
-  }
-
-  return error;
-}
-
-static u32
-ixge_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
-{
-  ixge_device_t *xd;
-  ixge_regs_t *r;
-  u32 old;
-  ixge_main_t *xm = &ixge_main;
-
-  xd = vec_elt_at_index (xm->devices, hw->dev_instance);
-  r = xd->regs;
-
-  old = r->filter_control;
-
-  if (flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL)
-    r->filter_control = old | (1 << 9) /* unicast promiscuous */ ;
-  else
-    r->filter_control = old & ~(1 << 9);
-
-  return old;
-}
-
-static void
-ixge_device_init (ixge_main_t * xm)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  ixge_device_t *xd;
-
-  /* Reset chip(s). */
-  vec_foreach (xd, xm->devices)
-  {
-    ixge_regs_t *r = xd->regs;
-    const u32 reset_bit = (1 << 26) | (1 << 3);
-
-    r->control |= reset_bit;
-
-    /* No need to suspend.  Timed to take ~1e-6 secs */
-    while (r->control & reset_bit)
-      ;
-
-    /* Software loaded. */
-    r->extended_control |= (1 << 28);
-
-    ixge_phy_init (xd);
-
-    /* Register ethernet interface. */
-    {
-      u8 addr8[6];
-      u32 i, addr32[2];
-      clib_error_t *error;
-
-      addr32[0] = r->rx_ethernet_address0[0][0];
-      addr32[1] = r->rx_ethernet_address0[0][1];
-      for (i = 0; i < 6; i++)
-	addr8[i] = addr32[i / 4] >> ((i % 4) * 8);
-
-      error = ethernet_register_interface
-	(vnm, ixge_device_class.index, xd->device_index,
-	 /* ethernet address */ addr8,
-	 &xd->vlib_hw_if_index, ixge_flag_change);
-      if (error)
-	clib_error_report (error);
-    }
-
-    {
-      vnet_sw_interface_t *sw =
-	vnet_get_hw_sw_interface (vnm, xd->vlib_hw_if_index);
-      xd->vlib_sw_if_index = sw->sw_if_index;
-    }
-
-    ixge_dma_init (xd, VLIB_RX, /* queue_index */ 0);
-
-    xm->n_descriptors[VLIB_TX] = 20 * VLIB_FRAME_SIZE;
-
-    ixge_dma_init (xd, VLIB_TX, /* queue_index */ 0);
-
-    /* RX/TX queue 0 gets mapped to interrupt bits 0 & 8. */
-    r->interrupt.queue_mapping[0] = (( /* valid bit */ (1 << 7) |
-				      ixge_rx_queue_to_interrupt (0)) << 0);
-
-    r->interrupt.queue_mapping[0] |= (( /* valid bit */ (1 << 7) |
-				       ixge_tx_queue_to_interrupt (0)) << 8);
-
-    /* No use in getting too many interrupts.
-       Limit them to one every 3/4 ring size at line rate
-       min sized packets.
-       No need for this since kernel/vlib main loop provides adequate interrupt
-       limiting scheme. */
-    if (0)
-      {
-	f64 line_rate_max_pps =
-	  10e9 / (8 * (64 + /* interframe padding */ 20));
-	ixge_throttle_queue_interrupt (r, 0,
-				       .75 * xm->n_descriptors[VLIB_RX] /
-				       line_rate_max_pps);
-      }
-
-    /* Accept all multicast and broadcast packets. Should really add them
-       to the dst_ethernet_address register array. */
-    r->filter_control |= (1 << 10) | (1 << 8);
-
-    /* Enable frames up to size in mac frame size register. */
-    r->xge_mac.control |= 1 << 2;
-    r->xge_mac.rx_max_frame_size = (9216 + 14) << 16;
-
-    /* Enable all interrupts. */
-    if (!IXGE_ALWAYS_POLL)
-      r->interrupt.enable_write_1_to_set = ~0;
-  }
-}
-
-static uword
-ixge_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  ixge_main_t *xm = &ixge_main;
-  ixge_device_t *xd;
-  uword event_type, *event_data = 0;
-  f64 timeout, link_debounce_deadline;
-
-  ixge_device_init (xm);
-
-  /* Clear all counters. */
-  vec_foreach (xd, xm->devices)
-  {
-    ixge_update_counters (xd);
-    memset (xd->counters, 0, sizeof (xd->counters));
-  }
-
-  timeout = 30.0;
-  link_debounce_deadline = 1e70;
-
-  while (1)
-    {
-      /* 36 bit stat counters could overflow in ~50 secs.
-         We poll every 30 secs to be conservative. */
-      vlib_process_wait_for_event_or_clock (vm, timeout);
-
-      event_type = vlib_process_get_events (vm, &event_data);
-
-      switch (event_type)
-	{
-	case EVENT_SET_FLAGS:
-	  /* 1 ms */
-	  link_debounce_deadline = vlib_time_now (vm) + 1e-3;
-	  timeout = 1e-3;
-	  break;
-
-	case ~0:
-	  /* No events found: timer expired. */
-	  if (vlib_time_now (vm) > link_debounce_deadline)
-	    {
-	      vec_foreach (xd, xm->devices)
-	      {
-		ixge_regs_t *r = xd->regs;
-		u32 v = r->xge_mac.link_status;
-		uword is_up = (v & (1 << 30)) != 0;
-
-		vnet_hw_interface_set_flags
-		  (vnm, xd->vlib_hw_if_index,
-		   is_up ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
-	      }
-	      link_debounce_deadline = 1e70;
-	      timeout = 30.0;
-	    }
-	  break;
-
-	default:
-	  ASSERT (0);
-	}
-
-      if (event_data)
-	_vec_len (event_data) = 0;
-
-      /* Query stats every 30 secs. */
-      {
-	f64 now = vlib_time_now (vm);
-	if (now - xm->time_last_stats_update > 30)
-	  {
-	    xm->time_last_stats_update = now;
-	    vec_foreach (xd, xm->devices) ixge_update_counters (xd);
-	  }
-      }
-    }
-
-  return 0;
-}
-
-static vlib_node_registration_t ixge_process_node = {
-  .function = ixge_process,
-  .type = VLIB_NODE_TYPE_PROCESS,
-  .name = "ixge-process",
-};
-
-clib_error_t *
-ixge_init (vlib_main_t * vm)
-{
-  ixge_main_t *xm = &ixge_main;
-  clib_error_t *error;
-
-  xm->vlib_main = vm;
-  memset (&xm->tx_descriptor_template, 0,
-	  sizeof (xm->tx_descriptor_template));
-  memset (&xm->tx_descriptor_template_mask, 0,
-	  sizeof (xm->tx_descriptor_template_mask));
-  xm->tx_descriptor_template.status0 =
-    (IXGE_TX_DESCRIPTOR_STATUS0_ADVANCED |
-     IXGE_TX_DESCRIPTOR_STATUS0_IS_ADVANCED |
-     IXGE_TX_DESCRIPTOR_STATUS0_INSERT_FCS);
-  xm->tx_descriptor_template_mask.status0 = 0xffff;
-  xm->tx_descriptor_template_mask.status1 = 0x00003fff;
-
-  xm->tx_descriptor_template_mask.status0 &=
-    ~(IXGE_TX_DESCRIPTOR_STATUS0_IS_END_OF_PACKET
-      | IXGE_TX_DESCRIPTOR_STATUS0_REPORT_STATUS);
-  xm->tx_descriptor_template_mask.status1 &=
-    ~(IXGE_TX_DESCRIPTOR_STATUS1_DONE);
-
-  error = vlib_call_init_function (vm, pci_bus_init);
-
-  return error;
-}
-
-VLIB_INIT_FUNCTION (ixge_init);
-
-
-static void
-ixge_pci_intr_handler (vlib_pci_device_t * dev)
-{
-  ixge_main_t *xm = &ixge_main;
-  vlib_main_t *vm = xm->vlib_main;
-
-  vlib_node_set_interrupt_pending (vm, ixge_input_node.index);
-
-  /* Let node know which device is interrupting. */
-  {
-    vlib_node_runtime_t *rt =
-      vlib_node_get_runtime (vm, ixge_input_node.index);
-    rt->runtime_data[0] |= 1 << dev->private_data;
-  }
-}
-
-static clib_error_t *
-ixge_pci_init (vlib_main_t * vm, vlib_pci_device_t * dev)
-{
-  ixge_main_t *xm = &ixge_main;
-  clib_error_t *error;
-  void *r;
-  ixge_device_t *xd;
-
-  /* Device found: make sure we have dma memory. */
-  if (unix_physmem_is_fake (vm))
-    return clib_error_return (0, "no physical memory available");
-
-  error = vlib_pci_map_resource (dev, 0, &r);
-  if (error)
-    return error;
-
-  vec_add2 (xm->devices, xd, 1);
-
-  if (vec_len (xm->devices) == 1)
-    {
-      ixge_input_node.function = ixge_input_multiarch_select ();
-    }
-
-  xd->pci_device = dev[0];
-  xd->device_id = xd->pci_device.config0.header.device_id;
-  xd->regs = r;
-  xd->device_index = xd - xm->devices;
-  xd->pci_function = dev->bus_address.function;
-  xd->per_interface_next_index = ~0;
-
-
-  /* Chip found so enable node. */
-  {
-    vlib_node_set_state (vm, ixge_input_node.index,
-			 (IXGE_ALWAYS_POLL
-			  ? VLIB_NODE_STATE_POLLING
-			  : VLIB_NODE_STATE_INTERRUPT));
-
-    dev->private_data = xd->device_index;
-  }
-
-  if (vec_len (xm->devices) == 1)
-    {
-      vlib_register_node (vm, &ixge_process_node);
-      xm->process_node_index = ixge_process_node.index;
-    }
-
-  error = vlib_pci_bus_master_enable (dev);
-
-  if (error)
-    return error;
-
-  return vlib_pci_intr_enable (dev);
-}
-
-/* *INDENT-OFF* */
-PCI_REGISTER_DEVICE (ixge_pci_device_registration,static) = {
-  .init_function = ixge_pci_init,
-  .interrupt_handler = ixge_pci_intr_handler,
-  .supported_devices = {
-#define _(t,i) { .vendor_id = PCI_VENDOR_ID_INTEL, .device_id = i, },
-    foreach_ixge_pci_device_id
-#undef _
-    { 0 },
-  },
-};
-/* *INDENT-ON* */
-
-void
-ixge_set_next_node (ixge_rx_next_t next, char *name)
-{
-  vlib_node_registration_t *r = &ixge_input_node;
-
-  switch (next)
-    {
-    case IXGE_RX_NEXT_IP4_INPUT:
-    case IXGE_RX_NEXT_IP6_INPUT:
-    case IXGE_RX_NEXT_ETHERNET_INPUT:
-      r->next_nodes[next] = name;
-      break;
-
-    default:
-      clib_warning ("%s: illegal next %d\n", __FUNCTION__, next);
-      break;
-    }
-}
-#endif
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/nic/ixge.h b/src/vnet/devices/nic/ixge.h
deleted file mode 100644
index a8e652dc..00000000
--- a/src/vnet/devices/nic/ixge.h
+++ /dev/null
@@ -1,1293 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef included_ixge_h
-#define included_ixge_h
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-typedef volatile struct
-{
-  /* [31:7] 128 byte aligned. */
-  u32 descriptor_address[2];
-  u32 n_descriptor_bytes;
-
-  /* [5] rx/tx descriptor dca enable
-     [6] rx packet head dca enable
-     [7] rx packet tail dca enable
-     [9] rx/tx descriptor relaxed order
-     [11] rx/tx descriptor write back relaxed order
-     [13] rx/tx data write/read relaxed order
-     [15] rx head data write relaxed order
-     [31:24] apic id for cpu's cache. */
-  u32 dca_control;
-
-  u32 head_index;
-
-  /* [4:0] tail buffer size (in 1k byte units)
-     [13:8] head buffer size (in 64 byte units)
-     [24:22] lo free descriptors threshold (units of 64 descriptors)
-     [27:25] descriptor type 0 = legacy, 1 = advanced one buffer (e.g. tail),
-     2 = advanced header splitting (head + tail), 5 = advanced header
-     splitting (head only).
-     [28] drop if no descriptors available. */
-  u32 rx_split_control;
-
-  u32 tail_index;
-    CLIB_PAD_FROM_TO (0x1c, 0x28);
-
-  /* [7:0] rx/tx prefetch threshold
-     [15:8] rx/tx host threshold
-     [24:16] rx/tx write back threshold
-     [25] rx/tx enable
-     [26] tx descriptor writeback flush
-     [30] rx strip vlan enable */
-  u32 control;
-
-  u32 rx_coallesce_control;
-
-  union
-  {
-    struct
-    {
-      /* packets bytes lo hi */
-      u32 stats[3];
-
-      u32 unused;
-    } rx;
-
-    struct
-    {
-      u32 unused[2];
-
-      /* [0] enables head write back. */
-      u32 head_index_write_back_address[2];
-    } tx;
-  };
-} ixge_dma_regs_t;
-
-/* Only advanced descriptors are supported. */
-typedef struct
-{
-  u64 tail_address;
-  u64 head_address;
-} ixge_rx_to_hw_descriptor_t;
-
-typedef struct
-{
-  u32 status[3];
-  u16 n_packet_bytes_this_descriptor;
-  u16 vlan_tag;
-} ixge_rx_from_hw_descriptor_t;
-
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_LAYER2 (1 << (4 + 11))
-/* Valid if not layer2. */
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4 (1 << (4 + 0))
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP4_EXT (1 << (4 + 1))
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6 (1 << (4 + 2))
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_IP6_EXT (1 << (4 + 3))
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_TCP (1 << (4 + 4))
-#define IXGE_RX_DESCRIPTOR_STATUS0_IS_UDP (1 << (4 + 5))
-#define IXGE_RX_DESCRIPTOR_STATUS0_L3_OFFSET(s) (((s) >> 21) & 0x3ff)
-
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_OWNED_BY_SOFTWARE (1 << (0 + 0))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_END_OF_PACKET (1 << (0 + 1))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_VLAN (1 << (0 + 3))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_UDP_CHECKSUMMED (1 << (0 + 4))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_TCP_CHECKSUMMED (1 << (0 + 5))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_IP4_CHECKSUMMED (1 << (0 + 6))
-#define IXGE_RX_DESCRIPTOR_STATUS2_NOT_UNICAST (1 << (0 + 7))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IS_DOUBLE_VLAN (1 << (0 + 9))
-#define IXGE_RX_DESCRIPTOR_STATUS2_UDP_CHECKSUM_ERROR (1 << (0 + 10))
-#define IXGE_RX_DESCRIPTOR_STATUS2_ETHERNET_ERROR (1 << (20 + 9))
-#define IXGE_RX_DESCRIPTOR_STATUS2_TCP_CHECKSUM_ERROR (1 << (20 + 10))
-#define IXGE_RX_DESCRIPTOR_STATUS2_IP4_CHECKSUM_ERROR (1 << (20 + 11))
-
-/* For layer2 packets stats0 bottom 3 bits give ether type index from filter. */
-#define IXGE_RX_DESCRIPTOR_STATUS0_LAYER2_ETHERNET_TYPE(s) ((s) & 7)
-
-typedef struct
-{
-  u64 buffer_address;
-  u16 n_bytes_this_buffer;
-  u16 status0;
-  u32 status1;
-#define IXGE_TX_DESCRIPTOR_STATUS0_ADVANCED (3 << 4)
-#define IXGE_TX_DESCRIPTOR_STATUS0_IS_ADVANCED (1 << (8 + 5))
-#define IXGE_TX_DESCRIPTOR_STATUS0_LOG2_REPORT_STATUS (8 + 3)
-#define IXGE_TX_DESCRIPTOR_STATUS0_REPORT_STATUS (1 << IXGE_TX_DESCRIPTOR_STATUS0_LOG2_REPORT_STATUS)
-#define IXGE_TX_DESCRIPTOR_STATUS0_INSERT_FCS (1 << (8 + 1))
-#define IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET (8 + 0)
-#define IXGE_TX_DESCRIPTOR_STATUS0_IS_END_OF_PACKET (1 << IXGE_TX_DESCRIPTOR_STATUS0_LOG2_IS_END_OF_PACKET)
-#define IXGE_TX_DESCRIPTOR_STATUS1_DONE (1 << 0)
-#define IXGE_TX_DESCRIPTOR_STATUS1_CONTEXT(i) (/* valid */ (1 << 7) | ((i) << 4))
-#define IXGE_TX_DESCRIPTOR_STATUS1_IPSEC_OFFLOAD (1 << (8 + 2))
-#define IXGE_TX_DESCRIPTOR_STATUS1_INSERT_TCP_UDP_CHECKSUM (1 << (8 + 1))
-#define IXGE_TX_DESCRIPTOR_STATUS1_INSERT_IP4_CHECKSUM (1 << (8 + 0))
-#define IXGE_TX_DESCRIPTOR_STATUS0_N_BYTES_THIS_BUFFER(l) ((l) << 0)
-#define IXGE_TX_DESCRIPTOR_STATUS1_N_BYTES_IN_PACKET(l) ((l) << 14)
-} ixge_tx_descriptor_t;
-
-typedef struct
-{
-  struct
-  {
-    u8 checksum_start_offset;
-    u8 checksum_insert_offset;
-    u16 checksum_end_offset;
-  } ip, tcp;
-  u32 status0;
-
-  u8 status1;
-
-  /* Byte offset after UDP/TCP header. */
-  u8 payload_offset;
-
-  u16 max_tcp_segment_size;
-} __attribute__ ((packed)) ixge_tx_context_descriptor_t;
-
-typedef union
-{
-  ixge_rx_to_hw_descriptor_t rx_to_hw;
-  ixge_rx_from_hw_descriptor_t rx_from_hw;
-  ixge_tx_descriptor_t tx;
-  u32x4 as_u32x4;
-} ixge_descriptor_t;
-
-typedef volatile struct
-{
-  /* [2] pcie master disable
-     [3] mac reset
-     [26] global device reset */
-  u32 control;
-  u32 control_alias;
-  /* [3:2] device id (0 or 1 for dual port chips)
-     [7] link is up
-     [17:10] num vfs
-     [18] io active
-     [19] pcie master enable status */
-  u32 status_read_only;
-    CLIB_PAD_FROM_TO (0xc, 0x18);
-  /* [14] pf reset done
-     [17] relaxed ordering disable
-     [26] extended vlan enable
-     [28] driver loaded */
-  u32 extended_control;
-    CLIB_PAD_FROM_TO (0x1c, 0x20);
-
-  /* software definable pins.
-     sdp_data [7:0]
-     sdp_is_output [15:8]
-     sdp_is_native [23:16]
-     sdp_function [31:24].
-   */
-  u32 sdp_control;
-    CLIB_PAD_FROM_TO (0x24, 0x28);
-
-  /* [0] i2c clock in
-     [1] i2c clock out
-     [2] i2c data in
-     [3] i2c data out */
-  u32 i2c_control;
-    CLIB_PAD_FROM_TO (0x2c, 0x4c);
-  u32 tcp_timer;
-
-    CLIB_PAD_FROM_TO (0x50, 0x200);
-
-  u32 led_control;
-
-    CLIB_PAD_FROM_TO (0x204, 0x600);
-  u32 core_spare;
-    CLIB_PAD_FROM_TO (0x604, 0x700);
-
-  struct
-  {
-    u32 vflr_events_clear[4];
-    u32 mailbox_interrupt_status[4];
-    u32 mailbox_interrupt_enable[4];
-      CLIB_PAD_FROM_TO (0x730, 0x800);
-  } pf_foo;
-
-  struct
-  {
-    u32 status_write_1_to_clear;
-      CLIB_PAD_FROM_TO (0x804, 0x808);
-    u32 status_write_1_to_set;
-      CLIB_PAD_FROM_TO (0x80c, 0x810);
-    u32 status_auto_clear_enable;
-      CLIB_PAD_FROM_TO (0x814, 0x820);
-
-    /* [11:3] minimum inter-interrupt interval
-       (2e-6 units; 20e-6 units for fast ethernet).
-       [15] low-latency interrupt moderation enable
-       [20:16] low-latency interrupt credit
-       [27:21] interval counter
-       [31] write disable for credit and counter (write only). */
-    u32 throttle0[24];
-
-    u32 enable_write_1_to_set;
-      CLIB_PAD_FROM_TO (0x884, 0x888);
-    u32 enable_write_1_to_clear;
-      CLIB_PAD_FROM_TO (0x88c, 0x890);
-    u32 enable_auto_clear;
-    u32 msi_to_eitr_select;
-    /* [3:0] spd 0-3 interrupt detection enable
-       [4] msi-x enable
-       [5] other clear disable (makes other bits in status not clear on read)
-       etc. */
-    u32 control;
-      CLIB_PAD_FROM_TO (0x89c, 0x900);
-
-    /* Defines interrupt mapping for 128 rx + 128 tx queues.
-       64 x 4 8 bit entries.
-       For register [i]:
-       [5:0] bit in interrupt status for rx queue 2*i + 0
-       [7] valid bit
-       [13:8] bit for tx queue 2*i + 0
-       [15] valid bit
-       similar for rx 2*i + 1 and tx 2*i + 1. */
-    u32 queue_mapping[64];
-
-    /* tcp timer [7:0] and other interrupts [15:8] */
-    u32 misc_mapping;
-      CLIB_PAD_FROM_TO (0xa04, 0xa90);
-
-    /* 64 interrupts determined by mappings. */
-    u32 status1_write_1_to_clear[4];
-    u32 enable1_write_1_to_set[4];
-    u32 enable1_write_1_to_clear[4];
-      CLIB_PAD_FROM_TO (0xac0, 0xad0);
-    u32 status1_enable_auto_clear[4];
-      CLIB_PAD_FROM_TO (0xae0, 0x1000);
-  } interrupt;
-
-  ixge_dma_regs_t rx_dma0[64];
-
-    CLIB_PAD_FROM_TO (0x2000, 0x2140);
-  u32 dcb_rx_packet_plane_t4_config[8];
-  u32 dcb_rx_packet_plane_t4_status[8];
-    CLIB_PAD_FROM_TO (0x2180, 0x2300);
-
-  /* reg i defines mapping for 4 rx queues starting at 4*i + 0. */
-  u32 rx_queue_stats_mapping[32];
-  u32 rx_queue_stats_control;
-
-    CLIB_PAD_FROM_TO (0x2384, 0x2410);
-  u32 fc_user_descriptor_ptr[2];
-  u32 fc_buffer_control;
-    CLIB_PAD_FROM_TO (0x241c, 0x2420);
-  u32 fc_rx_dma;
-    CLIB_PAD_FROM_TO (0x2424, 0x2430);
-  u32 dcb_packet_plane_control;
-    CLIB_PAD_FROM_TO (0x2434, 0x2f00);
-
-  u32 rx_dma_control;
-  u32 pf_queue_drop_enable;
-    CLIB_PAD_FROM_TO (0x2f08, 0x2f20);
-  u32 rx_dma_descriptor_cache_config;
-    CLIB_PAD_FROM_TO (0x2f24, 0x3000);
-
-  /* 1 bit. */
-  u32 rx_enable;
-    CLIB_PAD_FROM_TO (0x3004, 0x3008);
-  /* [15:0] ether type (little endian)
-     [31:16] opcode (big endian) */
-  u32 flow_control_control;
-    CLIB_PAD_FROM_TO (0x300c, 0x3020);
-  /* 3 bit traffic class for each of 8 priorities. */
-  u32 rx_priority_to_traffic_class;
-    CLIB_PAD_FROM_TO (0x3024, 0x3028);
-  u32 rx_coallesce_data_buffer_control;
-    CLIB_PAD_FROM_TO (0x302c, 0x3190);
-  u32 rx_packet_buffer_flush_detect;
-    CLIB_PAD_FROM_TO (0x3194, 0x3200);
-  u32 flow_control_tx_timers[4];	/* 2 timer values */
-    CLIB_PAD_FROM_TO (0x3210, 0x3220);
-  u32 flow_control_rx_threshold_lo[8];
-    CLIB_PAD_FROM_TO (0x3240, 0x3260);
-  u32 flow_control_rx_threshold_hi[8];
-    CLIB_PAD_FROM_TO (0x3280, 0x32a0);
-  u32 flow_control_refresh_threshold;
-    CLIB_PAD_FROM_TO (0x32a4, 0x3c00);
-  /* For each of 8 traffic classes (units of bytes). */
-  u32 rx_packet_buffer_size[8];
-    CLIB_PAD_FROM_TO (0x3c20, 0x3d00);
-  u32 flow_control_config;
-    CLIB_PAD_FROM_TO (0x3d04, 0x4200);
-
-  struct
-  {
-    u32 pcs_config;
-      CLIB_PAD_FROM_TO (0x4204, 0x4208);
-    u32 link_control;
-    u32 link_status;
-    u32 pcs_debug[2];
-    u32 auto_negotiation;
-    u32 link_partner_ability;
-    u32 auto_negotiation_tx_next_page;
-    u32 auto_negotiation_link_partner_next_page;
-      CLIB_PAD_FROM_TO (0x4228, 0x4240);
-  } gige_mac;
-
-  struct
-  {
-    /* [0] tx crc enable
-       [2] enable frames up to max frame size register [31:16]
-       [10] pad frames < 64 bytes if specified by user
-       [15] loopback enable
-       [16] mdc hi speed
-       [17] turn off mdc between mdio packets */
-    u32 control;
-
-    /* [5] rx symbol error (all bits clear on read)
-       [6] rx illegal symbol
-       [7] rx idle error
-       [8] rx local fault
-       [9] rx remote fault */
-    u32 status;
-
-    u32 pause_and_pace_control;
-      CLIB_PAD_FROM_TO (0x424c, 0x425c);
-    u32 phy_command;
-    u32 phy_data;
-      CLIB_PAD_FROM_TO (0x4264, 0x4268);
-
-    /* [31:16] max frame size in bytes. */
-    u32 rx_max_frame_size;
-      CLIB_PAD_FROM_TO (0x426c, 0x4288);
-
-    /* [0]
-       [2] pcs receive link up? (latch lo)
-       [7] local fault
-       [1]
-       [0] pcs 10g base r capable
-       [1] pcs 10g base x capable
-       [2] pcs 10g base w capable
-       [10] rx local fault
-       [11] tx local fault
-       [15:14] 2 => device present at this address (else not present) */
-    u32 xgxs_status[2];
-
-    u32 base_x_pcs_status;
-
-    /* [0] pass unrecognized flow control frames
-       [1] discard pause frames
-       [2] rx priority flow control enable (only in dcb mode)
-       [3] rx flow control enable. */
-    u32 flow_control;
-
-    /* [3:0] tx lanes change polarity
-       [7:4] rx lanes change polarity
-       [11:8] swizzle tx lanes
-       [15:12] swizzle rx lanes
-       4 x 2 bit tx lane swap
-       4 x 2 bit rx lane swap. */
-    u32 serdes_control;
-
-    u32 fifo_control;
-
-    /* [0] force link up
-       [1] autoneg ack2 bit to transmit
-       [6:2] autoneg selector field to transmit
-       [8:7] 10g pma/pmd type 0 => xaui, 1 kx4, 2 cx4
-       [9] 1g pma/pmd type 0 => sfi, 1 => kx/bx
-       [10] disable 10g on without main power
-       [11] restart autoneg on transition to dx power state
-       [12] restart autoneg
-       [15:13] link mode:
-       0 => 1g no autoneg
-       1 => 10g kx4 parallel link no autoneg
-       2 => 1g bx autoneg
-       3 => 10g sfi serdes
-       4 => kx4/kx/kr
-       5 => xgmii 1g/100m
-       6 => kx4/kx/kr 1g an
-       7 kx4/kx/kr sgmii.
-       [16] kr support
-       [17] fec requested
-       [18] fec ability
-       etc. */
-    u32 auto_negotiation_control;
-
-    /* [0] signal detect 1g/100m
-       [1] fec signal detect
-       [2] 10g serial pcs fec block lock
-       [3] 10g serial high error rate
-       [4] 10g serial pcs block lock
-       [5] kx/kx4/kr autoneg next page received
-       [6] kx/kx4/kr backplane autoneg next page received
-       [7] link status clear to read
-       [11:8] 10g signal detect (4 lanes) (for serial just lane 0)
-       [12] 10g serial signal detect
-       [16:13] 10g parallel lane sync status
-       [17] 10g parallel align status
-       [18] 1g sync status
-       [19] kx/kx4/kr backplane autoneg is idle
-       [20] 1g autoneg enabled
-       [21] 1g pcs enabled for sgmii
-       [22] 10g xgxs enabled
-       [23] 10g serial fec enabled (forward error detection)
-       [24] 10g kr pcs enabled
-       [25] sgmii enabled
-       [27:26] mac link mode
-       0 => 1g
-       1 => 10g parallel
-       2 => 10g serial
-       3 => autoneg
-       [29:28] link speed
-       1 => 100m
-       2 => 1g
-       3 => 10g
-       [30] link is up
-       [31] kx/kx4/kr backplane autoneg completed successfully. */
-    u32 link_status;
-
-    /* [17:16] pma/pmd for 10g serial
-       0 => kr, 2 => sfi
-       [18] disable dme pages */
-    u32 auto_negotiation_control2;
-
-      CLIB_PAD_FROM_TO (0x42ac, 0x42b0);
-    u32 link_partner_ability[2];
-      CLIB_PAD_FROM_TO (0x42b8, 0x42d0);
-    u32 manageability_control;
-    u32 link_partner_next_page[2];
-      CLIB_PAD_FROM_TO (0x42dc, 0x42e0);
-    u32 kr_pcs_control;
-    u32 kr_pcs_status;
-    u32 fec_status[2];
-      CLIB_PAD_FROM_TO (0x42f0, 0x4314);
-    u32 sgmii_control;
-      CLIB_PAD_FROM_TO (0x4318, 0x4324);
-    u32 link_status2;
-      CLIB_PAD_FROM_TO (0x4328, 0x4900);
-  } xge_mac;
-
-  u32 tx_dcb_control;
-  u32 tx_dcb_descriptor_plane_queue_select;
-  u32 tx_dcb_descriptor_plane_t1_config;
-  u32 tx_dcb_descriptor_plane_t1_status;
-    CLIB_PAD_FROM_TO (0x4910, 0x4950);
-
-  /* For each TC in units of 1k bytes. */
-  u32 tx_packet_buffer_thresholds[8];
-    CLIB_PAD_FROM_TO (0x4970, 0x4980);
-  struct
-  {
-    u32 mmw;
-    u32 config;
-    u32 status;
-    u32 rate_drift;
-  } dcb_tx_rate_scheduler;
-    CLIB_PAD_FROM_TO (0x4990, 0x4a80);
-  u32 tx_dma_control;
-    CLIB_PAD_FROM_TO (0x4a84, 0x4a88);
-  u32 tx_dma_tcp_flags_control[2];
-    CLIB_PAD_FROM_TO (0x4a90, 0x4b00);
-  u32 pf_mailbox[64];
-    CLIB_PAD_FROM_TO (0x4c00, 0x5000);
-
-  /* RX */
-  u32 checksum_control;
-    CLIB_PAD_FROM_TO (0x5004, 0x5008);
-  u32 rx_filter_control;
-    CLIB_PAD_FROM_TO (0x500c, 0x5010);
-  u32 management_vlan_tag[8];
-  u32 management_udp_tcp_ports[8];
-    CLIB_PAD_FROM_TO (0x5050, 0x5078);
-  /* little endian. */
-  u32 extended_vlan_ether_type;
-    CLIB_PAD_FROM_TO (0x507c, 0x5080);
-  /* [1] store/dma bad packets
-     [8] accept all multicast
-     [9] accept all unicast
-     [10] accept all broadcast. */
-  u32 filter_control;
-    CLIB_PAD_FROM_TO (0x5084, 0x5088);
-  /* [15:0] vlan ethernet type (0x8100) little endian
-     [28] cfi bit expected
-     [29] drop packets with unexpected cfi bit
-     [30] vlan filter enable. */
-  u32 vlan_control;
-    CLIB_PAD_FROM_TO (0x508c, 0x5090);
-  /* [1:0] hi bit of ethernet address for 12 bit index into multicast table
-     0 => 47, 1 => 46, 2 => 45, 3 => 43.
-     [2] enable multicast filter
-   */
-  u32 multicast_control;
-    CLIB_PAD_FROM_TO (0x5094, 0x5100);
-  u32 fcoe_rx_control;
-    CLIB_PAD_FROM_TO (0x5104, 0x5108);
-  u32 fc_flt_context;
-    CLIB_PAD_FROM_TO (0x510c, 0x5110);
-  u32 fc_filter_control;
-    CLIB_PAD_FROM_TO (0x5114, 0x5120);
-  u32 rx_message_type_lo;
-    CLIB_PAD_FROM_TO (0x5124, 0x5128);
-  /* [15:0] ethernet type (little endian)
-     [18:16] matche pri in vlan tag
-     [19] priority match enable
-     [25:20] virtualization pool
-     [26] pool enable
-     [27] is fcoe
-     [30] ieee 1588 timestamp enable
-     [31] filter enable.
-     (See ethernet_type_queue_select.) */
-  u32 ethernet_type_queue_filter[8];
-    CLIB_PAD_FROM_TO (0x5148, 0x5160);
-  /* [7:0] l2 ethernet type and
-     [15:8] l2 ethernet type or */
-  u32 management_decision_filters1[8];
-  u32 vf_vm_tx_switch_loopback_enable[2];
-  u32 rx_time_sync_control;
-    CLIB_PAD_FROM_TO (0x518c, 0x5190);
-  u32 management_ethernet_type_filters[4];
-  u32 rx_timestamp_attributes_lo;
-  u32 rx_timestamp_hi;
-  u32 rx_timestamp_attributes_hi;
-    CLIB_PAD_FROM_TO (0x51ac, 0x51b0);
-  u32 pf_virtual_control;
-    CLIB_PAD_FROM_TO (0x51b4, 0x51d8);
-  u32 fc_offset_parameter;
-    CLIB_PAD_FROM_TO (0x51dc, 0x51e0);
-  u32 vf_rx_enable[2];
-  u32 rx_timestamp_lo;
-    CLIB_PAD_FROM_TO (0x51ec, 0x5200);
-  /* 12 bits determined by multicast_control
-     lookup bits in this vector. */
-  u32 multicast_enable[128];
-
-  /* [0] ethernet address [31:0]
-     [1] [15:0] ethernet address [47:32]
-     [31] valid bit.
-     Index 0 is read from eeprom after reset. */
-  u32 rx_ethernet_address0[16][2];
-
-    CLIB_PAD_FROM_TO (0x5480, 0x5800);
-  u32 wake_up_control;
-    CLIB_PAD_FROM_TO (0x5804, 0x5808);
-  u32 wake_up_filter_control;
-    CLIB_PAD_FROM_TO (0x580c, 0x5818);
-  u32 multiple_rx_queue_command_82598;
-    CLIB_PAD_FROM_TO (0x581c, 0x5820);
-  u32 management_control;
-  u32 management_filter_control;
-    CLIB_PAD_FROM_TO (0x5828, 0x5838);
-  u32 wake_up_ip4_address_valid;
-    CLIB_PAD_FROM_TO (0x583c, 0x5840);
-  u32 wake_up_ip4_address_table[4];
-  u32 management_control_to_host;
-    CLIB_PAD_FROM_TO (0x5854, 0x5880);
-  u32 wake_up_ip6_address_table[4];
-
-  /* unicast_and broadcast_and vlan_and ip_address_and
-     etc. */
-  u32 management_decision_filters[8];
-
-  u32 management_ip4_or_ip6_address_filters[4][4];
-    CLIB_PAD_FROM_TO (0x58f0, 0x5900);
-  u32 wake_up_packet_length;
-    CLIB_PAD_FROM_TO (0x5904, 0x5910);
-  u32 management_ethernet_address_filters[4][2];
-    CLIB_PAD_FROM_TO (0x5930, 0x5a00);
-  u32 wake_up_packet_memory[32];
-    CLIB_PAD_FROM_TO (0x5a80, 0x5c00);
-  u32 redirection_table_82598[32];
-  u32 rss_random_keys_82598[10];
-    CLIB_PAD_FROM_TO (0x5ca8, 0x6000);
-
-  ixge_dma_regs_t tx_dma[128];
-
-  u32 pf_vm_vlan_insert[64];
-  u32 tx_dma_tcp_max_alloc_size_requests;
-    CLIB_PAD_FROM_TO (0x8104, 0x8110);
-  u32 vf_tx_enable[2];
-    CLIB_PAD_FROM_TO (0x8118, 0x8120);
-  /* [0] dcb mode enable
-     [1] virtualization mode enable
-     [3:2] number of tcs/qs per pool. */
-  u32 multiple_tx_queues_command;
-    CLIB_PAD_FROM_TO (0x8124, 0x8200);
-  u32 pf_vf_anti_spoof[8];
-  u32 pf_dma_tx_switch_control;
-    CLIB_PAD_FROM_TO (0x8224, 0x82e0);
-  u32 tx_strict_low_latency_queues[4];
-    CLIB_PAD_FROM_TO (0x82f0, 0x8600);
-  u32 tx_queue_stats_mapping_82599[32];
-  u32 tx_queue_packet_counts[32];
-  u32 tx_queue_byte_counts[32][2];
-
-  struct
-  {
-    u32 control;
-    u32 status;
-    u32 buffer_almost_full;
-      CLIB_PAD_FROM_TO (0x880c, 0x8810);
-    u32 buffer_min_ifg;
-      CLIB_PAD_FROM_TO (0x8814, 0x8900);
-  } tx_security;
-
-  struct
-  {
-    u32 index;
-    u32 salt;
-    u32 key[4];
-      CLIB_PAD_FROM_TO (0x8918, 0x8a00);
-  } tx_ipsec;
-
-  struct
-  {
-    u32 capabilities;
-    u32 control;
-    u32 tx_sci[2];
-    u32 sa;
-    u32 sa_pn[2];
-    u32 key[2][4];
-    /* untagged packets, encrypted packets, protected packets,
-       encrypted bytes, protected bytes */
-    u32 stats[5];
-      CLIB_PAD_FROM_TO (0x8a50, 0x8c00);
-  } tx_link_security;
-
-  struct
-  {
-    u32 control;
-    u32 timestamp_value[2];
-    u32 system_time[2];
-    u32 increment_attributes;
-    u32 time_adjustment_offset[2];
-    u32 aux_control;
-    u32 target_time[2][2];
-      CLIB_PAD_FROM_TO (0x8c34, 0x8c3c);
-    u32 aux_time_stamp[2][2];
-      CLIB_PAD_FROM_TO (0x8c4c, 0x8d00);
-  } tx_timesync;
-
-  struct
-  {
-    u32 control;
-    u32 status;
-      CLIB_PAD_FROM_TO (0x8d08, 0x8e00);
-  } rx_security;
-
-  struct
-  {
-    u32 index;
-    u32 ip_address[4];
-    u32 spi;
-    u32 ip_index;
-    u32 key[4];
-    u32 salt;
-    u32 mode;
-      CLIB_PAD_FROM_TO (0x8e34, 0x8f00);
-  } rx_ipsec;
-
-  struct
-  {
-    u32 capabilities;
-    u32 control;
-    u32 sci[2];
-    u32 sa[2];
-    u32 sa_pn[2];
-    u32 key[2][4];
-    /* see datasheet */
-    u32 stats[17];
-      CLIB_PAD_FROM_TO (0x8f84, 0x9000);
-  } rx_link_security;
-
-  /* 4 wake up, 2 management, 2 wake up. */
-  u32 flexible_filters[8][16][4];
-    CLIB_PAD_FROM_TO (0x9800, 0xa000);
-
-  /* 4096 bits. */
-  u32 vlan_filter[128];
-
-  /* [0] ethernet address [31:0]
-     [1] [15:0] ethernet address [47:32]
-     [31] valid bit.
-     Index 0 is read from eeprom after reset. */
-  u32 rx_ethernet_address1[128][2];
-
-  /* select one of 64 pools for each rx address. */
-  u32 rx_ethernet_address_pool_select[128][2];
-    CLIB_PAD_FROM_TO (0xaa00, 0xc800);
-  u32 tx_priority_to_traffic_class;
-    CLIB_PAD_FROM_TO (0xc804, 0xcc00);
-
-  /* In bytes units of 1k.  Total packet buffer is 160k. */
-  u32 tx_packet_buffer_size[8];
-
-    CLIB_PAD_FROM_TO (0xcc20, 0xcd10);
-  u32 tx_manageability_tc_mapping;
-    CLIB_PAD_FROM_TO (0xcd14, 0xcd20);
-  u32 dcb_tx_packet_plane_t2_config[8];
-  u32 dcb_tx_packet_plane_t2_status[8];
-    CLIB_PAD_FROM_TO (0xcd60, 0xce00);
-
-  u32 tx_flow_control_status;
-    CLIB_PAD_FROM_TO (0xce04, 0xd000);
-
-  ixge_dma_regs_t rx_dma1[64];
-
-  struct
-  {
-    /* Bigendian ip4 src/dst address. */
-    u32 src_address[128];
-    u32 dst_address[128];
-
-    /* TCP/UDP ports [15:0] src [31:16] dst; bigendian. */
-    u32 tcp_udp_port[128];
-
-    /* [1:0] protocol tcp, udp, sctp, other
-       [4:2] match priority (highest wins)
-       [13:8] pool
-       [25] src address match disable
-       [26] dst address match disable
-       [27] src port match disable
-       [28] dst port match disable
-       [29] protocol match disable
-       [30] pool match disable
-       [31] enable. */
-    u32 control[128];
-
-    /* [12] size bypass
-       [19:13] must be 0x80
-       [20] low-latency interrupt
-       [27:21] rx queue. */
-    u32 interrupt[128];
-  } ip4_filters;
-
-    CLIB_PAD_FROM_TO (0xea00, 0xeb00);
-  /* 4 bit rss output index indexed by 7 bit hash.
-     128 8 bit fields = 32 registers. */
-  u32 redirection_table_82599[32];
-
-  u32 rss_random_key_82599[10];
-    CLIB_PAD_FROM_TO (0xeba8, 0xec00);
-  /* [15:0] reserved
-     [22:16] rx queue index
-     [29] low-latency interrupt on match
-     [31] enable */
-  u32 ethernet_type_queue_select[8];
-    CLIB_PAD_FROM_TO (0xec20, 0xec30);
-  u32 syn_packet_queue_filter;
-    CLIB_PAD_FROM_TO (0xec34, 0xec60);
-  u32 immediate_interrupt_rx_vlan_priority;
-    CLIB_PAD_FROM_TO (0xec64, 0xec70);
-  u32 rss_queues_per_traffic_class;
-    CLIB_PAD_FROM_TO (0xec74, 0xec90);
-  u32 lli_size_threshold;
-    CLIB_PAD_FROM_TO (0xec94, 0xed00);
-
-  struct
-  {
-    u32 control;
-      CLIB_PAD_FROM_TO (0xed04, 0xed10);
-    u32 table[8];
-      CLIB_PAD_FROM_TO (0xed30, 0xee00);
-  } fcoe_redirection;
-
-  struct
-  {
-    /* [1:0] packet buffer allocation 0 => disabled, else 64k*2^(f-1)
-       [3] packet buffer initialization done
-       [4] perfetch match mode
-       [5] report status in rss field of rx descriptors
-       [7] report status always
-       [14:8] drop queue
-       [20:16] flex 2 byte packet offset (units of 2 bytes)
-       [27:24] max linked list length
-       [31:28] full threshold. */
-    u32 control;
-      CLIB_PAD_FROM_TO (0xee04, 0xee0c);
-
-    u32 data[8];
-
-    /* [1:0] 0 => no action, 1 => add, 2 => remove, 3 => query.
-       [2] valid filter found by query command
-       [3] filter update override
-       [4] ip6 adress table
-       [6:5] l4 protocol reserved, udp, tcp, sctp
-       [7] is ip6
-       [8] clear head/tail
-       [9] packet drop action
-       [10] matched packet generates low-latency interrupt
-       [11] last in linked list
-       [12] collision
-       [15] rx queue enable
-       [22:16] rx queue
-       [29:24] pool. */
-    u32 command;
-
-      CLIB_PAD_FROM_TO (0xee30, 0xee3c);
-    /* ip4 dst/src address, tcp ports, udp ports.
-       set bits mean bit is ignored. */
-    u32 ip4_masks[4];
-    u32 filter_length;
-    u32 usage_stats;
-    u32 failed_usage_stats;
-    u32 filters_match_stats;
-    u32 filters_miss_stats;
-      CLIB_PAD_FROM_TO (0xee60, 0xee68);
-    /* Lookup, signature. */
-    u32 hash_keys[2];
-    /* [15:0] ip6 src address 1 bit per byte
-       [31:16] ip6 dst address. */
-    u32 ip6_mask;
-    /* [0] vlan id
-       [1] vlan priority
-       [2] pool
-       [3] ip protocol
-       [4] flex
-       [5] dst ip6. */
-    u32 other_mask;
-      CLIB_PAD_FROM_TO (0xee78, 0xf000);
-  } flow_director;
-
-  struct
-  {
-    u32 l2_control[64];
-    u32 vlan_pool_filter[64];
-    u32 vlan_pool_filter_bitmap[128];
-    u32 dst_ethernet_address[128];
-    u32 mirror_rule[4];
-    u32 mirror_rule_vlan[8];
-    u32 mirror_rule_pool[8];
-      CLIB_PAD_FROM_TO (0xf650, 0x10010);
-  } pf_bar;
-
-  u32 eeprom_flash_control;
-  /* [0] start
-     [1] done
-     [15:2] address
-     [31:16] read data. */
-  u32 eeprom_read;
-    CLIB_PAD_FROM_TO (0x10018, 0x1001c);
-  u32 flash_access;
-    CLIB_PAD_FROM_TO (0x10020, 0x10114);
-  u32 flash_data;
-  u32 flash_control;
-  u32 flash_read_data;
-    CLIB_PAD_FROM_TO (0x10120, 0x1013c);
-  u32 flash_opcode;
-  u32 software_semaphore;
-    CLIB_PAD_FROM_TO (0x10144, 0x10148);
-  u32 firmware_semaphore;
-    CLIB_PAD_FROM_TO (0x1014c, 0x10160);
-  u32 software_firmware_sync;
-    CLIB_PAD_FROM_TO (0x10164, 0x10200);
-  u32 general_rx_control;
-    CLIB_PAD_FROM_TO (0x10204, 0x11000);
-
-  struct
-  {
-    u32 control;
-      CLIB_PAD_FROM_TO (0x11004, 0x11010);
-    /* [3:0] enable counters
-       [7:4] leaky bucket counter mode
-       [29] reset
-       [30] stop
-       [31] start. */
-    u32 counter_control;
-    /* [7:0],[15:8],[23:16],[31:24] event for counters 0-3.
-       event codes:
-       0x0 bad tlp
-       0x10 reqs that reached timeout
-       etc. */
-    u32 counter_event;
-      CLIB_PAD_FROM_TO (0x11018, 0x11020);
-    u32 counters_clear_on_read[4];
-    u32 counter_config[4];
-    struct
-    {
-      u32 address;
-      u32 data;
-    } indirect_access;
-      CLIB_PAD_FROM_TO (0x11048, 0x11050);
-    u32 extended_control;
-      CLIB_PAD_FROM_TO (0x11054, 0x11064);
-    u32 mirrored_revision_id;
-      CLIB_PAD_FROM_TO (0x11068, 0x11070);
-    u32 dca_requester_id_information;
-
-    /* [0] global disable
-       [4:1] mode: 0 => legacy, 1 => dca 1.0. */
-    u32 dca_control;
-      CLIB_PAD_FROM_TO (0x11078, 0x110b0);
-    /* [0] pci completion abort
-       [1] unsupported i/o address
-       [2] wrong byte enable
-       [3] pci timeout */
-    u32 pcie_interrupt_status;
-      CLIB_PAD_FROM_TO (0x110b4, 0x110b8);
-    u32 pcie_interrupt_enable;
-      CLIB_PAD_FROM_TO (0x110bc, 0x110c0);
-    u32 msi_x_pba_clear[8];
-      CLIB_PAD_FROM_TO (0x110e0, 0x12300);
-  } pcie;
-
-  u32 interrupt_throttle1[128 - 24];
-    CLIB_PAD_FROM_TO (0x124a0, 0x14f00);
-
-  u32 core_analog_config;
-    CLIB_PAD_FROM_TO (0x14f04, 0x14f10);
-  u32 core_common_config;
-    CLIB_PAD_FROM_TO (0x14f14, 0x15f14);
-
-  u32 link_sec_software_firmware_interface;
-} ixge_regs_t;
-
-typedef union
-{
-  struct
-  {
-    /* Addresses bigendian. */
-    union
-    {
-      struct
-      {
-	ip6_address_t src_address;
-	u32 unused[1];
-      } ip6;
-      struct
-      {
-	u32 unused[3];
-	ip4_address_t src_address, dst_address;
-      } ip4;
-    };
-
-    /* [15:0] src port (little endian).
-       [31:16] dst port. */
-    u32 tcp_udp_ports;
-
-    /* [15:0] vlan (cfi bit set to 0).
-       [31:16] flex bytes.  bigendian. */
-    u32 vlan_and_flex_word;
-
-    /* [14:0] hash
-       [15] bucket valid
-       [31:16] signature (signature filers)/sw-index (perfect match). */
-    u32 hash;
-  };
-
-  u32 as_u32[8];
-} ixge_flow_director_key_t;
-
-always_inline void
-ixge_throttle_queue_interrupt (ixge_regs_t * r,
-			       u32 queue_interrupt_index,
-			       f64 inter_interrupt_interval_in_secs)
-{
-  volatile u32 *tr =
-    (queue_interrupt_index < ARRAY_LEN (r->interrupt.throttle0)
-     ? &r->interrupt.throttle0[queue_interrupt_index]
-     : &r->interrupt_throttle1[queue_interrupt_index]);
-  ASSERT (queue_interrupt_index < 128);
-  u32 v;
-  i32 i, mask = (1 << 9) - 1;
-
-  i = flt_round_nearest (inter_interrupt_interval_in_secs / 2e-6);
-  i = i < 1 ? 1 : i;
-  i = i >= mask ? mask : i;
-
-  v = tr[0];
-  v &= ~(mask << 3);
-  v |= i << 3;
-  tr[0] = v;
-}
-
-#define foreach_ixge_counter				\
-  _ (0x40d0, rx_total_packets)				\
-  _64 (0x40c0, rx_total_bytes)				\
-  _ (0x41b0, rx_good_packets_before_filtering)		\
-  _64 (0x41b4, rx_good_bytes_before_filtering)		\
-  _ (0x2f50, rx_dma_good_packets)			\
-  _64 (0x2f54, rx_dma_good_bytes)			\
-  _ (0x2f5c, rx_dma_duplicated_good_packets)		\
-  _64 (0x2f60, rx_dma_duplicated_good_bytes)		\
-  _ (0x2f68, rx_dma_good_loopback_packets)		\
-  _64 (0x2f6c, rx_dma_good_loopback_bytes)		\
-  _ (0x2f74, rx_dma_good_duplicated_loopback_packets)	\
-  _64 (0x2f78, rx_dma_good_duplicated_loopback_bytes)	\
-  _ (0x4074, rx_good_packets)				\
-  _64 (0x4088, rx_good_bytes)				\
-  _ (0x407c, rx_multicast_packets)			\
-  _ (0x4078, rx_broadcast_packets)			\
-  _ (0x405c, rx_64_byte_packets)			\
-  _ (0x4060, rx_65_127_byte_packets)			\
-  _ (0x4064, rx_128_255_byte_packets)			\
-  _ (0x4068, rx_256_511_byte_packets)			\
-  _ (0x406c, rx_512_1023_byte_packets)			\
-  _ (0x4070, rx_gt_1023_byte_packets)			\
-  _ (0x4000, rx_crc_errors)				\
-  _ (0x4120, rx_ip_checksum_errors)			\
-  _ (0x4004, rx_illegal_symbol_errors)			\
-  _ (0x4008, rx_error_symbol_errors)			\
-  _ (0x4034, rx_mac_local_faults)			\
-  _ (0x4038, rx_mac_remote_faults)			\
-  _ (0x4040, rx_length_errors)				\
-  _ (0x41a4, rx_xons)					\
-  _ (0x41a8, rx_xoffs)					\
-  _ (0x40a4, rx_undersize_packets)			\
-  _ (0x40a8, rx_fragments)				\
-  _ (0x40ac, rx_oversize_packets)			\
-  _ (0x40b0, rx_jabbers)				\
-  _ (0x40b4, rx_management_packets)			\
-  _ (0x40b8, rx_management_drops)			\
-  _ (0x3fa0, rx_missed_packets_pool_0)			\
-  _ (0x40d4, tx_total_packets)				\
-  _ (0x4080, tx_good_packets)				\
-  _64 (0x4090, tx_good_bytes)				\
-  _ (0x40f0, tx_multicast_packets)			\
-  _ (0x40f4, tx_broadcast_packets)			\
-  _ (0x87a0, tx_dma_good_packets)			\
-  _64 (0x87a4, tx_dma_good_bytes)			\
-  _ (0x40d8, tx_64_byte_packets)			\
-  _ (0x40dc, tx_65_127_byte_packets)			\
-  _ (0x40e0, tx_128_255_byte_packets)			\
-  _ (0x40e4, tx_256_511_byte_packets)			\
-  _ (0x40e8, tx_512_1023_byte_packets)			\
-  _ (0x40ec, tx_gt_1023_byte_packets)			\
-  _ (0x4010, tx_undersize_drops)			\
-  _ (0x8780, switch_security_violation_packets)		\
-  _ (0x5118, fc_crc_errors)				\
-  _ (0x241c, fc_rx_drops)				\
-  _ (0x2424, fc_last_error_count)			\
-  _ (0x2428, fcoe_rx_packets)				\
-  _ (0x242c, fcoe_rx_dwords)				\
-  _ (0x8784, fcoe_tx_packets)				\
-  _ (0x8788, fcoe_tx_dwords)				\
-  _ (0x1030, queue_0_rx_count)				\
-  _ (0x1430, queue_0_drop_count)			\
-  _ (0x1070, queue_1_rx_count)				\
-  _ (0x1470, queue_1_drop_count)			\
-  _ (0x10b0, queue_2_rx_count)				\
-  _ (0x14b0, queue_2_drop_count)			\
-  _ (0x10f0, queue_3_rx_count)				\
-  _ (0x14f0, queue_3_drop_count)			\
-  _ (0x1130, queue_4_rx_count)				\
-  _ (0x1530, queue_4_drop_count)			\
-  _ (0x1170, queue_5_rx_count)				\
-  _ (0x1570, queue_5_drop_count)			\
-  _ (0x11b0, queue_6_rx_count)				\
-  _ (0x15b0, queue_6_drop_count)			\
-  _ (0x11f0, queue_7_rx_count)				\
-  _ (0x15f0, queue_7_drop_count)			\
-  _ (0x1230, queue_8_rx_count)				\
-  _ (0x1630, queue_8_drop_count)			\
-  _ (0x1270, queue_9_rx_count)				\
-  _ (0x1270, queue_9_drop_count)
-
-
-
-
-typedef enum
-{
-#define _(a,f) IXGE_COUNTER_##f,
-#define _64(a,f) _(a,f)
-  foreach_ixge_counter
-#undef _
-#undef _64
-    IXGE_N_COUNTER,
-} ixge_counter_type_t;
-
-typedef struct
-{
-  u32 mdio_address;
-
-  /* 32 bit ID read from ID registers. */
-  u32 id;
-} ixge_phy_t;
-
-typedef struct
-{
-  /* Cache aligned descriptors. */
-  ixge_descriptor_t *descriptors;
-
-  /* Number of descriptors in table. */
-  u32 n_descriptors;
-
-  /* Software head and tail pointers into descriptor ring. */
-  u32 head_index, tail_index;
-
-  /* Index into dma_queues vector. */
-  u32 queue_index;
-
-  /* Buffer indices corresponding to each active descriptor. */
-  u32 *descriptor_buffer_indices;
-
-  union
-  {
-    struct
-    {
-      u32 *volatile head_index_write_back;
-
-      u32 n_buffers_on_ring;
-    } tx;
-
-    struct
-    {
-      /* Buffer indices to use to replenish each descriptor. */
-      u32 *replenish_buffer_indices;
-
-      vlib_node_runtime_t *node;
-      u32 next_index;
-
-      u32 saved_start_of_packet_buffer_index;
-
-      u32 saved_start_of_packet_next_index;
-      u32 saved_last_buffer_index;
-
-      u32 is_start_of_packet;
-
-      u32 n_descriptors_done_total;
-
-      u32 n_descriptors_done_this_call;
-
-      u32 n_bytes;
-    } rx;
-  };
-} ixge_dma_queue_t;
-
-#define foreach_ixge_pci_device_id		\
-  _ (82598, 0x10b6)				\
-  _ (82598_bx, 0x1508)				\
-  _ (82598af_dual_port, 0x10c6)			\
-  _ (82598af_single_port, 0x10c7)		\
-  _ (82598at, 0x10c8)				\
-  _ (82598at2, 0x150b)				\
-  _ (82598eb_sfp_lom, 0x10db)			\
-  _ (82598eb_cx4, 0x10dd)			\
-  _ (82598_cx4_dual_port, 0x10ec)		\
-  _ (82598_da_dual_port, 0x10f1)		\
-  _ (82598_sr_dual_port_em, 0x10e1)		\
-  _ (82598eb_xf_lr, 0x10f4)			\
-  _ (82599_kx4, 0x10f7)				\
-  _ (82599_kx4_mezz, 0x1514)			\
-  _ (82599_kr, 0x1517)				\
-  _ (82599_combo_backplane, 0x10f8)		\
-  _ (82599_cx4, 0x10f9)				\
-  _ (82599_sfp, 0x10fb)				\
-  _ (82599_backplane_fcoe, 0x152a)		\
-  _ (82599_sfp_fcoe, 0x1529)			\
-  _ (82599_sfp_em, 0x1507)			\
-  _ (82599_xaui_lom, 0x10fc)			\
-  _ (82599_t3_lom, 0x151c)			\
-  _ (x540t, 0x1528)
-
-typedef enum
-{
-#define _(f,n) IXGE_##f = n,
-  foreach_ixge_pci_device_id
-#undef _
-} ixge_pci_device_id_t;
-
-typedef struct
-{
-  /* registers */
-  ixge_regs_t *regs;
-
-  /* Specific next index when using dynamic redirection */
-  u32 per_interface_next_index;
-
-  /* PCI bus info. */
-  vlib_pci_device_t pci_device;
-
-  /* From PCI config space header. */
-  ixge_pci_device_id_t device_id;
-
-  u16 device_index;
-
-  /* 0 or 1. */
-  u16 pci_function;
-
-  /* VLIB interface for this instance. */
-  u32 vlib_hw_if_index, vlib_sw_if_index;
-
-  ixge_dma_queue_t *dma_queues[VLIB_N_RX_TX];
-
-  /* Phy index (0 or 1) and address on MDI bus. */
-  u32 phy_index;
-  ixge_phy_t phys[2];
-
-  /* Value of link_status register at last link change. */
-  u32 link_status_at_last_link_change;
-
-  i2c_bus_t i2c_bus;
-  sfp_eeprom_t sfp_eeprom;
-
-  /* Counters. */
-  u64 counters[IXGE_N_COUNTER], counters_last_clear[IXGE_N_COUNTER];
-} ixge_device_t;
-
-typedef struct
-{
-  vlib_main_t *vlib_main;
-
-  /* Vector of devices. */
-  ixge_device_t *devices;
-
-  /* Descriptor ring sizes. */
-  u32 n_descriptors[VLIB_N_RX_TX];
-
-  /* RX buffer size.  Must be at least 1k; will be rounded to
-     next largest 1k size. */
-  u32 n_bytes_in_rx_buffer;
-
-  u32 n_descriptors_per_cache_line;
-
-  u32 vlib_buffer_free_list_index;
-
-  u32 process_node_index;
-
-  /* Template and mask for initializing/validating TX descriptors. */
-  ixge_tx_descriptor_t tx_descriptor_template, tx_descriptor_template_mask;
-
-  /* Vector of buffers for which TX is done and can be freed. */
-  u32 *tx_buffers_pending_free;
-
-  u32 *rx_buffers_to_add;
-
-  f64 time_last_stats_update;
-} ixge_main_t;
-
-ixge_main_t ixge_main;
-vnet_device_class_t ixge_device_class;
-
-typedef enum
-{
-  IXGE_RX_NEXT_IP4_INPUT,
-  IXGE_RX_NEXT_IP6_INPUT,
-  IXGE_RX_NEXT_ETHERNET_INPUT,
-  IXGE_RX_NEXT_DROP,
-  IXGE_RX_N_NEXT,
-} ixge_rx_next_t;
-
-void ixge_set_next_node (ixge_rx_next_t, char *);
-
-#endif /* included_ixge_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/nic/sfp.c b/src/vnet/devices/nic/sfp.c
deleted file mode 100644
index 9e9c008d..00000000
--- a/src/vnet/devices/nic/sfp.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-
-static u8 *
-format_space_terminated (u8 * s, va_list * args)
-{
-  u32 l = va_arg (*args, u32);
-  u8 *v = va_arg (*args, u8 *);
-  u8 *p;
-
-  for (p = v + l - 1; p >= v && p[0] == ' '; p--)
-    ;
-  vec_add (s, v, clib_min (p - v + 1, l));
-  return s;
-}
-
-static u8 *
-format_sfp_id (u8 * s, va_list * args)
-{
-  u32 id = va_arg (*args, u32);
-  char *t = 0;
-  switch (id)
-    {
-#define _(f) case SFP_ID_##f: t = #f; break;
-      foreach_sfp_id
-#undef _
-    default:
-      return format (s, "unknown 0x%x", id);
-    }
-  return format (s, "%s", t);
-}
-
-static u8 *
-format_sfp_compatibility (u8 * s, va_list * args)
-{
-  u32 c = va_arg (*args, u32);
-  char *t = 0;
-  switch (c)
-    {
-#define _(a,b,f) case SFP_COMPATIBILITY_##f: t = #f; break;
-      foreach_sfp_compatibility
-#undef _
-    default:
-      return format (s, "unknown 0x%x", c);
-    }
-  return format (s, "%s", t);
-}
-
-u32
-sfp_is_comatible (sfp_eeprom_t * e, sfp_compatibility_t c)
-{
-  static struct
-  {
-    u8 byte, bit;
-  } t[] =
-  {
-#define _(a,b,f) { .byte = a, .bit = b, },
-    foreach_sfp_compatibility
-#undef _
-  };
-
-  ASSERT (c < ARRAY_LEN (t));
-  return (e->compatibility[t[c].byte] & (1 << t[c].bit)) != 0;
-}
-
-u8 *
-format_sfp_eeprom (u8 * s, va_list * args)
-{
-  sfp_eeprom_t *e = va_arg (*args, sfp_eeprom_t *);
-  uword indent = format_get_indent (s);
-  int i;
-
-  if (e->id != SFP_ID_sfp)
-    s = format (s, "id %U, ", format_sfp_id, e->id);
-
-  s = format (s, "compatibility:");
-  for (i = 0; i < SFP_N_COMPATIBILITY; i++)
-    if (sfp_is_comatible (e, i))
-      s = format (s, " %U", format_sfp_compatibility, i);
-
-  s = format (s, "\n%Uvendor: %U, part %U",
-	      format_white_space, indent,
-	      format_space_terminated, sizeof (e->vendor_name),
-	      e->vendor_name, format_space_terminated,
-	      sizeof (e->vendor_part_number), e->vendor_part_number);
-  s =
-    format (s, "\n%Urevision: %U, serial: %U, date code: %U",
-	    format_white_space, indent, format_space_terminated,
-	    sizeof (e->vendor_revision), e->vendor_revision,
-	    format_space_terminated, sizeof (e->vendor_serial_number),
-	    e->vendor_serial_number, format_space_terminated,
-	    sizeof (e->vendor_date_code), e->vendor_date_code);
-
-  return s;
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/devices/nic/sfp.h b/src/vnet/devices/nic/sfp.h
deleted file mode 100644
index a1ac7997..00000000
--- a/src/vnet/devices/nic/sfp.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef included_vnet_optics_sfp_h
-#define included_vnet_optics_sfp_h
-
-#include 
-
-#define foreach_sfp_id				\
-  _ (unknown)					\
-  _ (gbic)					\
-  _ (on_motherboard)				\
-  _ (sfp)
-
-typedef enum
-{
-#define _(f) SFP_ID_##f,
-  foreach_sfp_id
-#undef _
-} sfp_id_t;
-
-typedef struct
-{
-  u8 id;
-  u8 extended_id;
-  u8 connector_type;
-  u8 compatibility[8];
-  u8 encoding;
-  u8 nominal_bit_rate_100mbits_per_sec;
-  u8 reserved13;
-  u8 link_length[5];
-  u8 reserved19;
-  u8 vendor_name[16];
-  u8 reserved36;
-  u8 vendor_oui[3];
-  u8 vendor_part_number[16];
-  u8 vendor_revision[4];
-  /* 16 bit value network byte order. */
-  u8 laser_wavelength_in_nm[2];
-  u8 reserved62;
-  u8 checksum_0_to_62;
-
-  u8 options[2];
-  u8 max_bit_rate_margin_percent;
-  u8 min_bit_rate_margin_percent;
-  u8 vendor_serial_number[16];
-  u8 vendor_date_code[8];
-  u8 reserved92[3];
-  u8 checksum_63_to_94;
-  u8 vendor_specific[32];
-  u8 reserved128[384];
-
-  /* Vendor specific data follows. */
-  u8 vendor_specific1[0];
-} sfp_eeprom_t;
-
-always_inline uword
-sfp_eeprom_is_valid (sfp_eeprom_t * e)
-{
-  int i;
-  u8 sum = 0;
-  for (i = 0; i < 63; i++)
-    sum += ((u8 *) e)[i];
-  return sum == e->checksum_0_to_62;
-}
-
-/* _ (byte_index, bit_index, name) */
-#define foreach_sfp_compatibility		\
-  _ (0, 4, 10g_base_sr)				\
-  _ (0, 5, 10g_base_lr)				\
-  _ (1, 2, oc48_long_reach)			\
-  _ (1, 1, oc48_intermediate_reach)		\
-  _ (1, 0, oc48_short_reach)			\
-  _ (2, 6, oc12_long_reach)			\
-  _ (2, 5, oc12_intermediate_reach)		\
-  _ (2, 4, oc12_short_reach)			\
-  _ (2, 2, oc3_long_reach)			\
-  _ (2, 1, oc3_intermediate_reach)		\
-  _ (2, 0, oc3_short_reach)			\
-  _ (3, 3, 1g_base_t)				\
-  _ (3, 2, 1g_base_cx)				\
-  _ (3, 1, 1g_base_lx)				\
-  _ (3, 0, 1g_base_sx)
-
-typedef enum
-{
-#define _(a,b,f) SFP_COMPATIBILITY_##f,
-  foreach_sfp_compatibility
-#undef _
-    SFP_N_COMPATIBILITY,
-} sfp_compatibility_t;
-
-u32 sfp_is_comatible (sfp_eeprom_t * e, sfp_compatibility_t c);
-
-format_function_t format_sfp_eeprom;
-
-#endif /* included_vnet_optics_sfp_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/ethernet/sfp.c b/src/vnet/ethernet/sfp.c
new file mode 100644
index 00000000..624740e3
--- /dev/null
+++ b/src/vnet/ethernet/sfp.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+static u8 *
+format_space_terminated (u8 * s, va_list * args)
+{
+  u32 l = va_arg (*args, u32);
+  u8 *v = va_arg (*args, u8 *);
+  u8 *p;
+
+  for (p = v + l - 1; p >= v && p[0] == ' '; p--)
+    ;
+  vec_add (s, v, clib_min (p - v + 1, l));
+  return s;
+}
+
+static u8 *
+format_sfp_id (u8 * s, va_list * args)
+{
+  u32 id = va_arg (*args, u32);
+  char *t = 0;
+  switch (id)
+    {
+#define _(f) case SFP_ID_##f: t = #f; break;
+      foreach_sfp_id
+#undef _
+    default:
+      return format (s, "unknown 0x%x", id);
+    }
+  return format (s, "%s", t);
+}
+
+static u8 *
+format_sfp_compatibility (u8 * s, va_list * args)
+{
+  u32 c = va_arg (*args, u32);
+  char *t = 0;
+  switch (c)
+    {
+#define _(a,b,f) case SFP_COMPATIBILITY_##f: t = #f; break;
+      foreach_sfp_compatibility
+#undef _
+    default:
+      return format (s, "unknown 0x%x", c);
+    }
+  return format (s, "%s", t);
+}
+
+u32
+sfp_is_comatible (sfp_eeprom_t * e, sfp_compatibility_t c)
+{
+  static struct
+  {
+    u8 byte, bit;
+  } t[] =
+  {
+#define _(a,b,f) { .byte = a, .bit = b, },
+    foreach_sfp_compatibility
+#undef _
+  };
+
+  ASSERT (c < ARRAY_LEN (t));
+  return (e->compatibility[t[c].byte] & (1 << t[c].bit)) != 0;
+}
+
+u8 *
+format_sfp_eeprom (u8 * s, va_list * args)
+{
+  sfp_eeprom_t *e = va_arg (*args, sfp_eeprom_t *);
+  uword indent = format_get_indent (s);
+  int i;
+
+  if (e->id != SFP_ID_sfp)
+    s = format (s, "id %U, ", format_sfp_id, e->id);
+
+  s = format (s, "compatibility:");
+  for (i = 0; i < SFP_N_COMPATIBILITY; i++)
+    if (sfp_is_comatible (e, i))
+      s = format (s, " %U", format_sfp_compatibility, i);
+
+  s = format (s, "\n%Uvendor: %U, part %U",
+	      format_white_space, indent,
+	      format_space_terminated, sizeof (e->vendor_name),
+	      e->vendor_name, format_space_terminated,
+	      sizeof (e->vendor_part_number), e->vendor_part_number);
+  s =
+    format (s, "\n%Urevision: %U, serial: %U, date code: %U",
+	    format_white_space, indent, format_space_terminated,
+	    sizeof (e->vendor_revision), e->vendor_revision,
+	    format_space_terminated, sizeof (e->vendor_serial_number),
+	    e->vendor_serial_number, format_space_terminated,
+	    sizeof (e->vendor_date_code), e->vendor_date_code);
+
+  return s;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/ethernet/sfp.h b/src/vnet/ethernet/sfp.h
new file mode 100644
index 00000000..a1ac7997
--- /dev/null
+++ b/src/vnet/ethernet/sfp.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_vnet_optics_sfp_h
+#define included_vnet_optics_sfp_h
+
+#include 
+
+#define foreach_sfp_id				\
+  _ (unknown)					\
+  _ (gbic)					\
+  _ (on_motherboard)				\
+  _ (sfp)
+
+typedef enum
+{
+#define _(f) SFP_ID_##f,
+  foreach_sfp_id
+#undef _
+} sfp_id_t;
+
+typedef struct
+{
+  u8 id;
+  u8 extended_id;
+  u8 connector_type;
+  u8 compatibility[8];
+  u8 encoding;
+  u8 nominal_bit_rate_100mbits_per_sec;
+  u8 reserved13;
+  u8 link_length[5];
+  u8 reserved19;
+  u8 vendor_name[16];
+  u8 reserved36;
+  u8 vendor_oui[3];
+  u8 vendor_part_number[16];
+  u8 vendor_revision[4];
+  /* 16 bit value network byte order. */
+  u8 laser_wavelength_in_nm[2];
+  u8 reserved62;
+  u8 checksum_0_to_62;
+
+  u8 options[2];
+  u8 max_bit_rate_margin_percent;
+  u8 min_bit_rate_margin_percent;
+  u8 vendor_serial_number[16];
+  u8 vendor_date_code[8];
+  u8 reserved92[3];
+  u8 checksum_63_to_94;
+  u8 vendor_specific[32];
+  u8 reserved128[384];
+
+  /* Vendor specific data follows. */
+  u8 vendor_specific1[0];
+} sfp_eeprom_t;
+
+always_inline uword
+sfp_eeprom_is_valid (sfp_eeprom_t * e)
+{
+  int i;
+  u8 sum = 0;
+  for (i = 0; i < 63; i++)
+    sum += ((u8 *) e)[i];
+  return sum == e->checksum_0_to_62;
+}
+
+/* _ (byte_index, bit_index, name) */
+#define foreach_sfp_compatibility		\
+  _ (0, 4, 10g_base_sr)				\
+  _ (0, 5, 10g_base_lr)				\
+  _ (1, 2, oc48_long_reach)			\
+  _ (1, 1, oc48_intermediate_reach)		\
+  _ (1, 0, oc48_short_reach)			\
+  _ (2, 6, oc12_long_reach)			\
+  _ (2, 5, oc12_intermediate_reach)		\
+  _ (2, 4, oc12_short_reach)			\
+  _ (2, 2, oc3_long_reach)			\
+  _ (2, 1, oc3_intermediate_reach)		\
+  _ (2, 0, oc3_short_reach)			\
+  _ (3, 3, 1g_base_t)				\
+  _ (3, 2, 1g_base_cx)				\
+  _ (3, 1, 1g_base_lx)				\
+  _ (3, 0, 1g_base_sx)
+
+typedef enum
+{
+#define _(a,b,f) SFP_COMPATIBILITY_##f,
+  foreach_sfp_compatibility
+#undef _
+    SFP_N_COMPATIBILITY,
+} sfp_compatibility_t;
+
+u32 sfp_is_comatible (sfp_eeprom_t * e, sfp_compatibility_t c);
+
+format_function_t format_sfp_eeprom;
+
+#endif /* included_vnet_optics_sfp_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vpp-api/lua/bench.lua b/src/vpp-api/lua/bench.lua
index 8e5a0b4b..c7231b90 100644
--- a/src/vpp-api/lua/bench.lua
+++ b/src/vpp-api/lua/bench.lua
@@ -53,9 +53,9 @@ function do_bench()
 end
 
 root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
 vpp:init({ pneum_path = pneum_path })
-vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+vpp:json_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api.json")
 
 vpp:connect("lua-bench")
 local n_tests = 10
diff --git a/src/vpp-api/lua/examples/cli/lua-cli.lua b/src/vpp-api/lua/examples/cli/lua-cli.lua
index b3a24d7d..4a27af53 100644
--- a/src/vpp-api/lua/examples/cli/lua-cli.lua
+++ b/src/vpp-api/lua/examples/cli/lua-cli.lua
@@ -557,12 +557,12 @@ end
 
 function init_vpp(vpp)
   local root_dir = "/home/ubuntu/vpp"
-  local pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+  local pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
 
   vpp:init({ pneum_path = pneum_path })
 
   vpp:init({ pneum_path = pneum_path })
-  vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+  vpp:json_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api.json")
 
 
 
diff --git a/src/vpp-api/lua/examples/example-classifier.lua b/src/vpp-api/lua/examples/example-classifier.lua
index ec9c3d3e..b1270757 100644
--- a/src/vpp-api/lua/examples/example-classifier.lua
+++ b/src/vpp-api/lua/examples/example-classifier.lua
@@ -20,12 +20,12 @@ local vpp = require "vpp-lapi"
 local bit = require("bit")
 
 root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
 
 
 vpp:init({ pneum_path = pneum_path })
 
-vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+vpp:json_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api.json")
 
 vpp:connect("aytest")
 
diff --git a/src/vpp-api/lua/examples/example-cli.lua b/src/vpp-api/lua/examples/example-cli.lua
index 8b84989f..85425caf 100644
--- a/src/vpp-api/lua/examples/example-cli.lua
+++ b/src/vpp-api/lua/examples/example-cli.lua
@@ -18,11 +18,11 @@
 vpp = require "vpp-lapi"
 
 root_dir = "/home/ubuntu/vpp"
-pneum_path = root_dir .. "/build-root/install-vpp_lite_debug-native/vpp-api/lib64/libpneum.so"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
 
 vpp:init({ pneum_path = pneum_path })
 
-vpp:json_api(root_dir .. "/build-root/install-vpp_lite_debug-native/vpp/vpp-api/vpe.api.json")
+vpp:json_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api.json")
 
 vpp:connect("aytest")
 
diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c
index a566d956..d6a12325 100644
--- a/src/vpp/vnet/main.c
+++ b/src/vpp/vnet/main.c
@@ -199,9 +199,6 @@ defaulted:
     {
       vm->init_functions_called = hash_create (0, /* value bytes */ 0);
       vpe_main_init (vm);
-#if DPDK == 0
-      unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
-#endif
       return vlib_unix_main (argc, argv);
     }
   else
diff --git a/test/framework.py b/test/framework.py
index 6b1799a5..a0284e37 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -157,7 +157,9 @@ class VppTestCase(unittest.TestCase):
         cls.vpp_cmdline = [cls.vpp_bin, "unix",
                            "{", "nodaemon", debug_cli, coredump_size, "}",
                            "api-trace", "{", "on", "}",
-                           "api-segment", "{", "prefix", cls.shm_prefix, "}"]
+                           "api-segment", "{", "prefix", cls.shm_prefix, "}",
+                           "plugins", "{", "plugin", "dpdk_plugin.so", "{",
+                           "disable", "}", "}"]
         if cls.plugin_path is not None:
             cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path])
         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
-- 
cgit 1.2.3-korg


From eaabe073515e7722ed546b36f99efc6feea305a1 Mon Sep 17 00:00:00 2001
From: Damjan Marion 
Date: Wed, 22 Mar 2017 10:18:13 +0100
Subject: Add memif - packet memory interface for intra-host communication

Change-Id: I94c06b07a39f07ceba87bf3e7fcfc70e43231e8a
Signed-off-by: Damjan Marion 
Co-Authored-By: Milan Lenco 
---
 src/configure.ac                    |    1 +
 src/plugins/Makefile.am             |    4 +
 src/plugins/memif.am                |   32 ++
 src/plugins/memif/cli.c             |  202 +++++++
 src/plugins/memif/device.c          |  330 +++++++++++
 src/plugins/memif/memif.api         |  127 +++++
 src/plugins/memif/memif.c           | 1045 +++++++++++++++++++++++++++++++++++
 src/plugins/memif/memif.h           |  263 +++++++++
 src/plugins/memif/memif_all_api_h.h |   18 +
 src/plugins/memif/memif_api.c       |  332 +++++++++++
 src/plugins/memif/memif_msg_enum.h  |   31 ++
 src/plugins/memif/node.c            |  383 +++++++++++++
 src/vppinfra/clib.h                 |    6 +
 13 files changed, 2774 insertions(+)
 create mode 100644 src/plugins/memif.am
 create mode 100644 src/plugins/memif/cli.c
 create mode 100644 src/plugins/memif/device.c
 create mode 100644 src/plugins/memif/memif.api
 create mode 100644 src/plugins/memif/memif.c
 create mode 100644 src/plugins/memif/memif.h
 create mode 100644 src/plugins/memif/memif_all_api_h.h
 create mode 100644 src/plugins/memif/memif_api.c
 create mode 100644 src/plugins/memif/memif_msg_enum.h
 create mode 100644 src/plugins/memif/node.c

(limited to 'src/plugins/Makefile.am')

diff --git a/src/configure.ac b/src/configure.ac
index 5e02adc1..222dbe6e 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -151,6 +151,7 @@ PLUGIN_ENABLED(ila)
 PLUGIN_ENABLED(ioam)
 PLUGIN_ENABLED(ixge)
 PLUGIN_ENABLED(lb)
+PLUGIN_ENABLED(memif)
 PLUGIN_ENABLED(sixrd)
 PLUGIN_ENABLED(snat)
 PLUGIN_DISABLED(srv6sample)
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 255e644f..6713a9e2 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -57,6 +57,10 @@ if ENABLE_LB_PLUGIN
 include lb.am
 endif
 
+if ENABLE_MEMIF_PLUGIN
+include memif.am
+endif
+
 if ENABLE_SIXRD_PLUGIN
 include sixrd.am
 endif
diff --git a/src/plugins/memif.am b/src/plugins/memif.am
new file mode 100644
index 00000000..d3a3171d
--- /dev/null
+++ b/src/plugins/memif.am
@@ -0,0 +1,32 @@
+# Copyright (c) 2017 Cisco Systems, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppplugins_LTLIBRARIES += memif_plugin.la
+
+memif_plugin_la_SOURCES = memif/memif.c   \
+	memif/memif_api.c                 \
+	memif/cli.c                       \
+	memif/node.c                      \
+	memif/device.c                    \
+	memif/memif_plugin.api.h
+
+noinst_HEADERS += memif/memif.h
+
+nobase_apiinclude_HEADERS +=              \
+		  memif/memif_all_api_h.h \
+		  memif/memif_msg_enum.h  \
+		  memif/memif.api.h
+
+API_FILES += memif/memif.api
+
+# vi:syntax=automake
diff --git a/src/plugins/memif/cli.c b/src/plugins/memif/cli.c
new file mode 100644
index 00000000..ef73693b
--- /dev/null
+++ b/src/plugins/memif/cli.c
@@ -0,0 +1,202 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+
+static clib_error_t *
+memif_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			 vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  int r;
+  u32 ring_size = MEMIF_DEFAULT_RING_SIZE;
+  memif_create_if_args_t args = { 0 };
+  args.buffer_size = MEMIF_DEFAULT_BUFFER_SIZE;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "key 0x%" PRIx64, &args.key))
+	;
+      else if (unformat (line_input, "socket %s", &args.socket_filename))
+	;
+      else if (unformat (line_input, "ring-size %u", &ring_size))
+	;
+      else if (unformat (line_input, "buffer-size %u", &args.buffer_size))
+	;
+      else if (unformat (line_input, "master"))
+	args.is_master = 1;
+      else if (unformat (line_input, "slave"))
+	args.is_master = 0;
+      else if (unformat (line_input, "hw-addr %U",
+			 unformat_ethernet_address, args.hw_addr))
+	args.hw_addr_set = 1;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+  if (!is_pow2 (ring_size))
+    return clib_error_return (0, "ring size must be power of 2");
+
+  args.log2_ring_size = min_log2 (ring_size);
+
+  r = memif_create_if (vm, &args);
+
+  if (r <= VNET_API_ERROR_SYSCALL_ERROR_1
+      && r >= VNET_API_ERROR_SYSCALL_ERROR_10)
+    return clib_error_return (0, "%s (errno %d)", strerror (errno), errno);
+
+  if (r == VNET_API_ERROR_INVALID_INTERFACE)
+    return clib_error_return (0, "Invalid interface name");
+
+  if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS)
+    return clib_error_return (0, "Interface already exists");
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (memif_create_command, static) = {
+  .path = "create memif",
+  .short_help = "create memif [key ] [socket ] "
+                "[ring-size ] [buffer-size ] [hw-addr ] "
+		"",
+  .function = memif_create_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+memif_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			 vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u64 key = 0;
+  u8 key_defined = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "key 0x%" PRIx64, &key))
+	key_defined = 1;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  unformat_free (line_input);
+
+  if (!key_defined)
+    return clib_error_return (0, "missing key");
+
+  memif_delete_if (vm, key);
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (memif_delete_command, static) = {
+  .path = "delete memif",
+  .short_help = "delete memif key ",
+  .function = memif_delete_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+memif_show_command_fn (vlib_main_t * vm, unformat_input_t * input,
+		       vlib_cli_command_t * cmd)
+{
+  memif_main_t *mm = &memif_main;
+  memif_if_t *mif;
+  vnet_main_t *vnm = vnet_get_main ();
+  int i;
+
+  /* *INDENT-OFF* */
+  pool_foreach (mif, mm->interfaces,
+    ({
+       vlib_cli_output (vm, "interface %U", format_vnet_sw_if_index_name,
+			vnm, mif->sw_if_index);
+       vlib_cli_output (vm, "  key 0x%" PRIx64 " file %s", mif->key,
+			mif->socket_filename);
+       vlib_cli_output (vm, "  listener %d conn-fd %d int-fd %d", mif->listener_index,
+			mif->connection.fd, mif->interrupt_line.fd);
+       vlib_cli_output (vm, "  ring-size %u num-c2s-rings %u num-s2c-rings %u buffer_size %u",
+			(1 << mif->log2_ring_size),
+			mif->num_s2m_rings,
+			mif->num_m2s_rings,
+			mif->buffer_size);
+       for (i=0; i < mif->num_s2m_rings; i++)
+         {
+	   memif_ring_t * ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
+	   if (ring)
+	     {
+	       vlib_cli_output (vm, "  slave-to-master ring %u:", i);
+	       vlib_cli_output (vm, "    head %u tail %u", ring->head, ring->tail);
+	     }
+	 }
+       for (i=0; i < mif->num_m2s_rings; i++)
+         {
+	   memif_ring_t * ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
+	   if (ring)
+	     {
+	       vlib_cli_output (vm, "  master-to-slave ring %u:", i);
+	       vlib_cli_output (vm, "    head %u tail %u", ring->head, ring->tail);
+	     }
+	 }
+    }));
+  /* *INDENT-ON* */
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (memif_show_command, static) = {
+  .path = "show memif",
+  .short_help = "show memif",
+  .function = memif_show_command_fn,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+memif_cli_init (vlib_main_t * vm)
+{
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (memif_cli_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/memif/device.c b/src/plugins/memif/device.c
new file mode 100644
index 00000000..446537a3
--- /dev/null
+++ b/src/plugins/memif/device.c
@@ -0,0 +1,330 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#define _GNU_SOURCE
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+
+#define foreach_memif_tx_func_error	       \
+_(NO_FREE_SLOTS, "no free tx slots")           \
+_(PENDING_MSGS, "pending msgs in tx ring")
+
+typedef enum
+{
+#define _(f,s) MEMIF_TX_ERROR_##f,
+  foreach_memif_tx_func_error
+#undef _
+    MEMIF_TX_N_ERROR,
+} memif_tx_func_error_t;
+
+static char *memif_tx_func_error_strings[] = {
+#define _(n,s) s,
+  foreach_memif_tx_func_error
+#undef _
+};
+
+
+static u8 *
+format_memif_device_name (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+
+  s = format (s, "memif%u", i);
+  return s;
+}
+
+static u8 *
+format_memif_device (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  int verbose = va_arg (*args, int);
+  uword indent = format_get_indent (s);
+
+  s = format (s, "MEMIF interface");
+  if (verbose)
+    {
+      s = format (s, "\n%U instance %u", format_white_space, indent + 2,
+		  dev_instance);
+    }
+  return s;
+}
+
+static u8 *
+format_memif_tx_trace (u8 * s, va_list * args)
+{
+  s = format (s, "Unimplemented...");
+  return s;
+}
+
+static_always_inline void
+memif_interface_lock (memif_if_t * mif)
+{
+  if (PREDICT_FALSE (mif->lockp != 0))
+    {
+      while (__sync_lock_test_and_set (mif->lockp, 1))
+	;
+    }
+}
+
+static_always_inline void
+memif_interface_unlock (memif_if_t * mif)
+{
+  if (PREDICT_FALSE (mif->lockp != 0))
+    *mif->lockp = 0;
+}
+
+static_always_inline void
+memif_prefetch_buffer_and_data (vlib_main_t * vm, u32 bi)
+{
+  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
+  vlib_prefetch_buffer_header (b, LOAD);
+  CLIB_PREFETCH (b->data, CLIB_CACHE_LINE_BYTES, LOAD);
+}
+
+static_always_inline uword
+memif_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+			   vlib_frame_t * frame, memif_if_t * mif,
+			   memif_ring_type_t type)
+{
+  u8 rid = 0;
+  memif_ring_t *ring = memif_get_ring (mif, type, rid);
+  u32 *buffers = vlib_frame_args (frame);
+  u32 n_left = frame->n_vectors;
+  u16 ring_size = 1 << mif->log2_ring_size;
+  u16 mask = ring_size - 1;
+  u16 head, tail;
+  u16 free_slots;
+
+  memif_interface_lock (mif);
+
+  /* free consumed buffers */
+
+  head = ring->head;
+  tail = ring->tail;
+
+  if (tail > head)
+    free_slots = tail - head;
+  else
+    free_slots = ring_size - head + tail;
+
+  while (n_left > 5 && free_slots > 1)
+    {
+      if (PREDICT_TRUE (head + 5 < ring_size))
+	{
+	  CLIB_PREFETCH (memif_get_buffer (mif, ring, head + 2),
+			 CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (memif_get_buffer (mif, ring, head + 3),
+			 CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&ring->desc[head + 4], CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&ring->desc[head + 5], CLIB_CACHE_LINE_BYTES, STORE);
+	}
+      else
+	{
+	  CLIB_PREFETCH (memif_get_buffer (mif, ring, (head + 2) % mask),
+			 CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (memif_get_buffer (mif, ring, (head + 3) % mask),
+			 CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&ring->desc[(head + 4) % mask],
+			 CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&ring->desc[(head + 5) % mask],
+			 CLIB_CACHE_LINE_BYTES, STORE);
+	}
+
+      memif_prefetch_buffer_and_data (vm, buffers[2]);
+      memif_prefetch_buffer_and_data (vm, buffers[3]);
+
+      vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[0]);
+      vlib_buffer_t *b1 = vlib_get_buffer (vm, buffers[1]);
+
+      void *mb0 = memif_get_buffer (mif, ring, head);
+      clib_memcpy (mb0, vlib_buffer_get_current (b0), CLIB_CACHE_LINE_BYTES);
+      ring->desc[head].length = b0->current_length;
+      head = (head + 1) & mask;
+
+      void *mb1 = memif_get_buffer (mif, ring, head);
+      clib_memcpy (mb1, vlib_buffer_get_current (b1), CLIB_CACHE_LINE_BYTES);
+      ring->desc[head].length = b1->current_length;
+      head = (head + 1) & mask;
+
+      if (b0->current_length > CLIB_CACHE_LINE_BYTES)
+	{
+	  clib_memcpy (mb0 + CLIB_CACHE_LINE_BYTES,
+		       vlib_buffer_get_current (b0) + CLIB_CACHE_LINE_BYTES,
+		       b0->current_length - CLIB_CACHE_LINE_BYTES);
+	}
+      if (b1->current_length > CLIB_CACHE_LINE_BYTES)
+	{
+	  clib_memcpy (mb1 + CLIB_CACHE_LINE_BYTES,
+		       vlib_buffer_get_current (b1) + CLIB_CACHE_LINE_BYTES,
+		       b1->current_length - CLIB_CACHE_LINE_BYTES);
+	}
+
+
+      buffers += 2;
+      n_left -= 2;
+      free_slots -= 2;
+    }
+
+  while (n_left && free_slots)
+    {
+      vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[0]);
+      void *mb0 = memif_get_buffer (mif, ring, head);
+      clib_memcpy (mb0, vlib_buffer_get_current (b0), CLIB_CACHE_LINE_BYTES);
+
+      if (b0->current_length > CLIB_CACHE_LINE_BYTES)
+	{
+	  clib_memcpy (mb0 + CLIB_CACHE_LINE_BYTES,
+		       vlib_buffer_get_current (b0) + CLIB_CACHE_LINE_BYTES,
+		       b0->current_length - CLIB_CACHE_LINE_BYTES);
+	}
+      ring->desc[head].length = b0->current_length;
+      head = (head + 1) & mask;
+
+      buffers++;
+      n_left--;
+      free_slots--;
+    }
+
+  CLIB_MEMORY_STORE_BARRIER ();
+  ring->head = head;
+
+  memif_interface_unlock (mif);
+
+  if (n_left)
+    {
+      vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
+			n_left);
+      vlib_buffer_free (vm, buffers, n_left);
+    }
+
+  vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
+  if (mif->interrupt_line.fd > 0)
+    {
+      u8 b = rid;
+      CLIB_UNUSED (int r) = write (mif->interrupt_line.fd, &b, sizeof (b));
+    }
+
+  return frame->n_vectors;
+}
+
+static uword
+memif_interface_tx (vlib_main_t * vm,
+		    vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  memif_main_t *nm = &memif_main;
+  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
+  memif_if_t *mif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
+
+  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
+    return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_S2M);
+  else
+    return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_M2S);
+}
+
+static void
+memif_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
+			       u32 node_index)
+{
+  memif_main_t *apm = &memif_main;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  memif_if_t *mif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
+
+  /* Shut off redirection */
+  if (node_index == ~0)
+    {
+      mif->per_interface_next_index = node_index;
+      return;
+    }
+
+  mif->per_interface_next_index =
+    vlib_node_add_next (vlib_get_main (), memif_input_node.index, node_index);
+}
+
+static void
+memif_clear_hw_interface_counters (u32 instance)
+{
+  /* Nothing for now */
+}
+
+static clib_error_t *
+memif_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
+{
+  memif_main_t *apm = &memif_main;
+  memif_msg_t msg;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  memif_if_t *mif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
+
+  if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
+    mif->flags |= MEMIF_IF_FLAG_ADMIN_UP;
+  else
+    {
+      mif->flags &= ~MEMIF_IF_FLAG_ADMIN_UP;
+      if (!(mif->flags & MEMIF_IF_FLAG_DELETING)
+	  && mif->connection.index != ~0)
+	{
+	  msg.version = MEMIF_VERSION;
+	  msg.type = MEMIF_MSG_TYPE_DISCONNECT;
+	  send (mif->connection.fd, &msg, sizeof (msg), 0);
+	}
+    }
+
+  return 0;
+}
+
+static clib_error_t *
+memif_subif_add_del_function (vnet_main_t * vnm,
+			      u32 hw_if_index,
+			      struct vnet_sw_interface_t *st, int is_add)
+{
+  /* Nothing for now */
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VNET_DEVICE_CLASS (memif_device_class) = {
+  .name = "memif",
+  .tx_function = memif_interface_tx,
+  .format_device_name = format_memif_device_name,
+  .format_device = format_memif_device,
+  .format_tx_trace = format_memif_tx_trace,
+  .tx_function_n_errors = MEMIF_TX_N_ERROR,
+  .tx_function_error_strings = memif_tx_func_error_strings,
+  .rx_redirect_to_node = memif_set_interface_next_node,
+  .clear_counters = memif_clear_hw_interface_counters,
+  .admin_up_down_function = memif_interface_admin_up_down,
+  .subif_add_del_function = memif_subif_add_del_function,
+};
+
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH(memif_device_class,
+				  memif_interface_tx)
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/memif/memif.api b/src/plugins/memif/memif.api
new file mode 100644
index 00000000..6f946421
--- /dev/null
+++ b/src/plugins/memif/memif.api
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** \brief Create memory interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param role - role of the interface in the connection (master/slave)
+    @param key - 64bit integer used to authenticate and match opposite sides
+           of the connection
+    @param socket_filename - filename of the socket to be used for connection
+           establishment
+    @param ring_size - the number of entries of RX/TX rings
+    @param buffer_size - size of the buffer allocated for each ring entry
+    @param hw_addr - interface MAC address
+*/
+define memif_create
+{
+  u32 client_index;
+  u32 context;
+
+  u8 role; /* 0 = master, 1 = slave */
+  u64 key; /* optional, default is 0 */
+  u8 socket_filename[128]; /* optional, default is "/var/vpp/memif.sock"
+                              and can be changed in VPP startup config */
+
+  u32 ring_size; /* optional, default is 1024 entries, must be power of 2 */
+  u16 buffer_size; /* optional, default is 2048 bytes */
+  u8 hw_addr[6]; /* optional, randomly generated if not defined */
+};
+
+/** \brief Create memory interface response
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+    @param sw_if_index - software index of the newly created interface
+*/
+define memif_create_reply
+{
+  u32 context;
+  i32 retval;
+  u32 sw_if_index;
+};
+
+/** \brief Delete memory interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface to delete
+*/
+define memif_delete
+{
+  u32 client_index;
+  u32 context;
+
+  u32 sw_if_index;
+};
+
+/** \brief Delete host-interface response
+    @param context - sender context, to match reply w/ request
+    @param retval - return value for request
+*/
+define memif_delete_reply
+{
+  u32 context;
+  i32 retval;
+};
+
+/** \brief Memory interface details structure
+    @param context - sender context, to match reply w/ request (memif_dump)
+    @param sw_if_index - index of the interface
+    @param if_name - name of the interface
+    @param hw_addr - interface MAC address
+    @param key - key associated with the interface
+    @param role - role of the interface in the connection (master/slave)
+    @param socket_filename - name of the socket used by this interface
+           to establish new connections
+    @param ring_size - the number of entries of RX/TX rings
+    @param buffer_size - size of the buffer allocated for each ring entry
+    @param admin_up_down - interface administrative status
+    @param link_up_down - interface link status
+
+*/
+define memif_details
+{
+  u32 context;
+
+  u32 sw_if_index;
+  u8 if_name[64];
+  u8 hw_addr[6];
+
+  /* memif specific parameters */
+  u64 key;
+  u8 role; /* 0 = master, 1 = slave */
+  u8 socket_filename[128];
+  u32 ring_size;
+  u16 buffer_size; /* optional, default is 2048 bytes */
+
+  /* 1 = up, 0 = down */
+  u8 admin_up_down;
+  u8 link_up_down;
+};
+
+/** \brief Dump all memory interfaces
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define memif_dump
+{
+  u32 client_index;
+  u32 context;
+};
+
+/*
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c
new file mode 100644
index 00000000..7ba67c5b
--- /dev/null
+++ b/src/plugins/memif/memif.c
@@ -0,0 +1,1045 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#define _GNU_SOURCE
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define MEMIF_DEBUG 1
+
+#if MEMIF_DEBUG == 1
+#define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#define DEBUG_UNIX_LOG(...) clib_unix_warning(__VA_ARGS__)
+#else
+#define DEBUG_LOG(...)
+#endif
+
+memif_main_t memif_main;
+
+static clib_error_t *memif_conn_fd_read_ready (unix_file_t * uf);
+static clib_error_t *memif_int_fd_read_ready (unix_file_t * uf);
+
+static u32
+memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
+{
+  /* nothing for now */
+  return 0;
+}
+
+static void
+memif_remove_pending_conn (memif_pending_conn_t * pending_conn)
+{
+  memif_main_t *mm = &memif_main;
+
+  unix_file_del (&unix_main,
+		 unix_main.file_pool + pending_conn->connection.index);
+  pool_put (mm->pending_conns, pending_conn);
+}
+
+static void
+memif_connect (vlib_main_t * vm, memif_if_t * mif)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  int num_rings = mif->num_s2m_rings + mif->num_m2s_rings;
+  memif_ring_data_t *rd = NULL;
+
+  vec_validate_aligned (mif->ring_data, num_rings - 1, CLIB_CACHE_LINE_BYTES);
+  vec_foreach (rd, mif->ring_data)
+  {
+    rd->last_head = 0;
+  }
+
+  mif->flags &= ~MEMIF_IF_FLAG_CONNECTING;
+  mif->flags |= MEMIF_IF_FLAG_CONNECTED;
+  vnet_hw_interface_set_flags (vnm, mif->hw_if_index,
+			       VNET_HW_INTERFACE_FLAG_LINK_UP);
+}
+
+static void
+memif_disconnect (vlib_main_t * vm, memif_if_t * mif)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+
+  mif->flags &= ~(MEMIF_IF_FLAG_CONNECTED | MEMIF_IF_FLAG_CONNECTING);
+  if (mif->hw_if_index != ~0)
+    vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0);
+
+  if (mif->interrupt_line.index != ~0)
+    {
+      unix_file_del (&unix_main,
+		     unix_main.file_pool + mif->interrupt_line.index);
+      mif->interrupt_line.index = ~0;
+      mif->interrupt_line.fd = -1;	/* closed in unix_file_del */
+    }
+  if (mif->connection.index != ~0)
+    {
+      unix_file_del (&unix_main, unix_main.file_pool + mif->connection.index);
+      mif->connection.index = ~0;
+      mif->connection.fd = -1;	/* closed in unix_file_del */
+    }
+
+  // TODO: properly munmap + close memif-owned shared memory segments
+  vec_free (mif->regions);
+}
+
+static clib_error_t *
+memif_process_connect_req (memif_pending_conn_t * pending_conn,
+			   memif_msg_t * req, struct ucred *slave_cr,
+			   int shm_fd, int int_fd)
+{
+  memif_main_t *mm = &memif_main;
+  vlib_main_t *vm = vlib_get_main ();
+  int fd = pending_conn->connection.fd;
+  unix_file_t *uf = 0;
+  memif_if_t *mif = 0;
+  memif_msg_t resp = { 0 };
+  unix_file_t template = { 0 };
+  void *shm;
+  uword *p;
+  u8 retval = 0;
+  static clib_error_t *error = 0;
+
+  if (shm_fd == -1)
+    {
+      DEBUG_LOG
+	("Connection request is missing shared memory file descriptor");
+      retval = 1;
+      goto response;
+    }
+
+  if (int_fd == -1)
+    {
+      DEBUG_LOG
+	("Connection request is missing interrupt line file descriptor");
+      retval = 2;
+      goto response;
+    }
+
+  if (slave_cr == NULL)
+    {
+      DEBUG_LOG ("Connection request is missing slave credentials");
+      retval = 3;
+      goto response;
+    }
+
+  p = mhash_get (&mm->if_index_by_key, &req->key);
+  if (!p)
+    {
+      DEBUG_LOG
+	("Connection request with unmatched key (0x%" PRIx64 ")", req->key);
+      retval = 4;
+      goto response;
+    }
+
+  mif = vec_elt_at_index (mm->interfaces, *p);
+  if (mif->listener_index != pending_conn->listener_index)
+    {
+      DEBUG_LOG
+	("Connection request with non-matching listener (%d vs. %d)",
+	 pending_conn->listener_index, mif->listener_index);
+      retval = 5;
+      goto response;
+    }
+
+  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
+    {
+      DEBUG_LOG ("Memif slave does not accept connection requests");
+      retval = 6;
+      goto response;
+    }
+
+  if (mif->connection.fd != -1)
+    {
+      DEBUG_LOG
+	("Memif with key 0x%" PRIx64 " is already connected", mif->key);
+      retval = 7;
+      goto response;
+    }
+
+  if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) == 0)
+    {
+      /* just silently decline the request */
+      retval = 8;
+      goto response;
+    }
+
+  if (req->shared_mem_size < sizeof (memif_shm_t))
+    {
+      DEBUG_LOG
+	("Unexpectedly small shared memory segment received from slave.");
+      retval = 9;
+      goto response;
+    }
+
+  if ((shm =
+       mmap (NULL, req->shared_mem_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+	     shm_fd, 0)) == MAP_FAILED)
+    {
+      DEBUG_UNIX_LOG
+	("Failed to map shared memory segment received from slave memif");
+      error = clib_error_return_unix (0, "mmap fd %d", shm_fd);
+      retval = 10;
+      goto response;
+    }
+
+  if (((memif_shm_t *) shm)->cookie != 0xdeadbeef)
+    {
+      DEBUG_LOG
+	("Possibly corrupted shared memory segment received from slave memif");
+      munmap (shm, req->shared_mem_size);
+      retval = 11;
+      goto response;
+    }
+
+  mif->log2_ring_size = req->log2_ring_size;
+  mif->num_s2m_rings = req->num_s2m_rings;
+  mif->num_m2s_rings = req->num_m2s_rings;
+  mif->buffer_size = req->buffer_size;
+  mif->remote_pid = slave_cr->pid;
+  mif->remote_uid = slave_cr->uid;
+  vec_add1 (mif->regions, shm);
+
+  /* register interrupt line */
+  mif->interrupt_line.fd = int_fd;
+  template.read_function = memif_int_fd_read_ready;
+  template.file_descriptor = int_fd;
+  template.private_data = mif->if_index;
+  mif->interrupt_line.index = unix_file_add (&unix_main, &template);
+
+  /* change context for future messages */
+  uf = vec_elt_at_index (unix_main.file_pool, pending_conn->connection.index);
+  uf->private_data = mif->if_index << 1;
+  mif->connection = pending_conn->connection;
+  pool_put (mm->pending_conns, pending_conn);
+
+  memif_connect (vm, mif);
+
+response:
+  resp.version = MEMIF_VERSION;
+  resp.type = MEMIF_MSG_TYPE_CONNECT_RESP;
+  resp.retval = retval;
+  send (fd, &resp, sizeof (resp), 0);
+  return error;
+}
+
+static clib_error_t *
+memif_process_connect_resp (memif_if_t * mif, memif_msg_t * resp)
+{
+  vlib_main_t *vm = vlib_get_main ();
+
+  if ((mif->flags & MEMIF_IF_FLAG_IS_SLAVE) == 0)
+    {
+      DEBUG_LOG ("Memif master does not accept connection responses");
+      return 0;
+    }
+
+  if ((mif->flags & MEMIF_IF_FLAG_CONNECTING) == 0)
+    {
+      DEBUG_LOG ("Unexpected connection response");
+      return 0;
+    }
+
+  if (resp->retval == 0)
+    memif_connect (vm, mif);
+  else
+    memif_disconnect (vm, mif);
+
+  return 0;
+}
+
+static clib_error_t *
+memif_conn_fd_read_ready (unix_file_t * uf)
+{
+  memif_main_t *mm = &memif_main;
+  vlib_main_t *vm = vlib_get_main ();
+  memif_if_t *mif = 0;
+  memif_pending_conn_t *pending_conn = 0;
+  int fd_array[2] = { -1, -1 };
+  char ctl[CMSG_SPACE (sizeof (fd_array)) +
+	   CMSG_SPACE (sizeof (struct ucred))] = { 0 };
+  struct msghdr mh = { 0 };
+  struct iovec iov[1];
+  struct ucred *cr = 0;
+  memif_msg_t msg = { 0 };
+  struct cmsghdr *cmsg;
+  ssize_t size;
+  static clib_error_t *error = 0;
+
+  iov[0].iov_base = (void *) &msg;
+  iov[0].iov_len = sizeof (memif_msg_t);
+  mh.msg_iov = iov;
+  mh.msg_iovlen = 1;
+  mh.msg_control = ctl;
+  mh.msg_controllen = sizeof (ctl);
+
+  /* grab the appropriate context */
+  if (uf->private_data & 1)
+    pending_conn = vec_elt_at_index (mm->pending_conns,
+				     uf->private_data >> 1);
+  else
+    mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 1);
+
+  /* receive the incoming message */
+  size = recvmsg (uf->file_descriptor, &mh, 0);
+  if (size != sizeof (memif_msg_t))
+    {
+      if (size != 0)
+	{
+	  DEBUG_UNIX_LOG ("Malformed message received on fd %d",
+			  uf->file_descriptor);
+	  error = clib_error_return_unix (0, "recvmsg fd %d",
+					  uf->file_descriptor);
+	}
+      goto disconnect;
+    }
+
+  /* check version of the sender's memif plugin */
+  if (msg.version != MEMIF_VERSION)
+    {
+      DEBUG_LOG ("Memif version mismatch");
+      goto disconnect;
+    }
+
+  /* process the message based on its type */
+  switch (msg.type)
+    {
+    case MEMIF_MSG_TYPE_CONNECT_REQ:
+      if (pending_conn == 0)
+	{
+	  DEBUG_LOG ("Received unexpected connection request");
+	  return 0;
+	}
+
+      /* Read anciliary data */
+      cmsg = CMSG_FIRSTHDR (&mh);
+      while (cmsg)
+	{
+	  if (cmsg->cmsg_level == SOL_SOCKET
+	      && cmsg->cmsg_type == SCM_CREDENTIALS)
+	    {
+	      cr = (struct ucred *) CMSG_DATA (cmsg);
+	    }
+	  else if (cmsg->cmsg_level == SOL_SOCKET
+		   && cmsg->cmsg_type == SCM_RIGHTS)
+	    {
+	      clib_memcpy (fd_array, CMSG_DATA (cmsg), sizeof (fd_array));
+	    }
+	  cmsg = CMSG_NXTHDR (&mh, cmsg);
+	}
+
+      return memif_process_connect_req (pending_conn, &msg, cr,
+					fd_array[0], fd_array[1]);
+
+    case MEMIF_MSG_TYPE_CONNECT_RESP:
+      if (mif == 0)
+	{
+	  DEBUG_LOG ("Received unexpected connection response");
+	  return 0;
+	}
+      return memif_process_connect_resp (mif, &msg);
+
+    case MEMIF_MSG_TYPE_DISCONNECT:
+      goto disconnect;
+
+    default:
+      DEBUG_LOG ("Received unknown message type");
+      goto disconnect;
+    }
+
+  return 0;
+
+disconnect:
+  if (pending_conn)
+    memif_remove_pending_conn (pending_conn);
+  else
+    memif_disconnect (vm, mif);
+  return error;
+}
+
+static clib_error_t *
+memif_int_fd_read_ready (unix_file_t * uf)
+{
+  memif_main_t *mm = &memif_main;
+  vlib_main_t *vm = vlib_get_main ();
+  memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
+  u8 b;
+  ssize_t size;
+
+  size = read (uf->file_descriptor, &b, sizeof (b));
+  if (0 == size)
+    {
+      /* interrupt line was disconnected */
+      unix_file_del (&unix_main,
+		     unix_main.file_pool + mif->interrupt_line.index);
+      mif->interrupt_line.index = ~0;
+      mif->interrupt_line.fd = -1;
+    }
+  vlib_node_set_interrupt_pending (vm, memif_input_node.index);
+  return 0;
+}
+
+static clib_error_t *
+memif_conn_fd_accept_ready (unix_file_t * uf)
+{
+  memif_main_t *mm = &memif_main;
+  memif_listener_t *listener = 0;
+  memif_pending_conn_t *pending_conn = 0;
+  int addr_len;
+  struct sockaddr_un client;
+  int conn_fd;
+  unix_file_t template = { 0 };
+
+  listener = pool_elt_at_index (mm->listeners, uf->private_data);
+
+  addr_len = sizeof (client);
+  conn_fd = accept (uf->file_descriptor,
+		    (struct sockaddr *) &client, (socklen_t *) & addr_len);
+
+  if (conn_fd < 0)
+    return clib_error_return_unix (0, "accept fd %d", uf->file_descriptor);
+
+  pool_get (mm->pending_conns, pending_conn);
+  pending_conn->index = pending_conn - mm->pending_conns;
+  pending_conn->listener_index = listener->index;
+  pending_conn->connection.fd = conn_fd;
+
+  template.read_function = memif_conn_fd_read_ready;
+  template.file_descriptor = conn_fd;
+  template.private_data = (pending_conn->index << 1) | 1;
+  pending_conn->connection.index = unix_file_add (&unix_main, &template);
+
+  return 0;
+}
+
+static void
+memif_connect_master (vlib_main_t * vm, memif_if_t * mif)
+{
+  memif_msg_t msg;
+  struct msghdr mh = { 0 };
+  struct iovec iov[1];
+  struct cmsghdr *cmsg;
+  int mfd = -1;
+  int rv;
+  int fd_array[2] = { -1, -1 };
+  char ctl[CMSG_SPACE (sizeof (fd_array))];
+  memif_ring_t *ring = NULL;
+  int i, j;
+  void *shm = 0;
+  u64 buffer_offset;
+  unix_file_t template = { 0 };
+
+  msg.version = MEMIF_VERSION;
+  msg.type = MEMIF_MSG_TYPE_CONNECT_REQ;
+  msg.key = mif->key;
+  msg.log2_ring_size = mif->log2_ring_size;
+  msg.num_s2m_rings = mif->num_s2m_rings;
+  msg.num_m2s_rings = mif->num_m2s_rings;
+  msg.buffer_size = mif->buffer_size;
+
+  buffer_offset = sizeof (memif_shm_t) +
+    (mif->num_s2m_rings + mif->num_m2s_rings) *
+    (sizeof (memif_ring_t) +
+     sizeof (memif_desc_t) * (1 << mif->log2_ring_size));
+
+  msg.shared_mem_size = buffer_offset +
+    mif->buffer_size * (1 << mif->log2_ring_size) * (mif->num_s2m_rings +
+						     mif->num_m2s_rings);
+
+  if ((mfd = memfd_create ("shared mem", MFD_ALLOW_SEALING)) == -1)
+    {
+      DEBUG_LOG ("Failed to create anonymous file");
+      goto error;
+    }
+
+  if ((fcntl (mfd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
+    {
+      DEBUG_UNIX_LOG ("Failed to seal an anonymous file off from truncating");
+      goto error;
+    }
+
+  if ((ftruncate (mfd, msg.shared_mem_size)) == -1)
+    {
+      DEBUG_UNIX_LOG ("Failed to extend the size of an anonymous file");
+      goto error;
+    }
+
+  if ((shm = mmap (NULL, msg.shared_mem_size, PROT_READ | PROT_WRITE,
+		   MAP_SHARED, mfd, 0)) == MAP_FAILED)
+    {
+      DEBUG_UNIX_LOG ("Failed to map anonymous file into memory");
+      goto error;
+    }
+
+  vec_add1 (mif->regions, shm);
+  ((memif_shm_t *) mif->regions[0])->cookie = 0xdeadbeef;
+
+  for (i = 0; i < mif->num_s2m_rings; i++)
+    {
+      ring = memif_get_ring (mif, MEMIF_RING_S2M, i);
+      ring->head = ring->tail = 0;
+      for (j = 0; j < (1 << mif->log2_ring_size); j++)
+	{
+	  u16 slot = i * (1 << mif->log2_ring_size) + j;
+	  ring->desc[j].region = 0;
+	  ring->desc[j].offset = buffer_offset + (slot * mif->buffer_size);
+	  ring->desc[j].buffer_length = mif->buffer_size;
+	}
+    }
+  for (i = 0; i < mif->num_m2s_rings; i++)
+    {
+      ring = memif_get_ring (mif, MEMIF_RING_M2S, i);
+      ring->head = ring->tail = 0;
+      for (j = 0; j < (1 << mif->log2_ring_size); j++)
+	{
+	  u16 slot =
+	    (i + mif->num_s2m_rings) * (1 << mif->log2_ring_size) + j;
+	  ring->desc[j].region = 0;
+	  ring->desc[j].offset = buffer_offset + (slot * mif->buffer_size);
+	  ring->desc[j].buffer_length = mif->buffer_size;
+	}
+    }
+
+  iov[0].iov_base = (void *) &msg;
+  iov[0].iov_len = sizeof (memif_msg_t);
+  mh.msg_iov = iov;
+  mh.msg_iovlen = 1;
+
+  /* create interrupt socket */
+  if (socketpair (AF_UNIX, SOCK_STREAM, 0, fd_array) < 0)
+    {
+      DEBUG_UNIX_LOG ("Failed to create a pair of connected sockets");
+      goto error;
+    }
+
+  mif->interrupt_line.fd = fd_array[0];
+  template.read_function = memif_int_fd_read_ready;
+  template.file_descriptor = mif->interrupt_line.fd;
+  template.private_data = mif->if_index;
+  mif->interrupt_line.index = unix_file_add (&unix_main, &template);
+
+  memset (&ctl, 0, sizeof (ctl));
+  mh.msg_control = ctl;
+  mh.msg_controllen = sizeof (ctl);
+  cmsg = CMSG_FIRSTHDR (&mh);
+  cmsg->cmsg_len = CMSG_LEN (sizeof (fd_array));
+  cmsg->cmsg_level = SOL_SOCKET;
+  cmsg->cmsg_type = SCM_RIGHTS;
+  fd_array[0] = mfd;
+  clib_memcpy (CMSG_DATA (cmsg), fd_array, sizeof (fd_array));
+
+  mif->flags |= MEMIF_IF_FLAG_CONNECTING;
+  rv = sendmsg (mif->connection.fd, &mh, 0);
+  if (rv < 0)
+    {
+      DEBUG_UNIX_LOG ("Failed to send memif connection request");
+      goto error;
+    }
+
+  /* No need to keep the descriptor open,
+   * mmap creates an extra reference to the underlying file */
+  close (mfd);
+  mfd = -1;
+  /* This FD is given to peer, so we can close it */
+  close (fd_array[1]);
+  fd_array[1] = -1;
+  return;
+
+error:
+  if (mfd > -1)
+    close (mfd);
+  if (fd_array[1] > -1)
+    close (fd_array[1]);
+  memif_disconnect (vm, mif);
+}
+
+static uword
+memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
+{
+  memif_main_t *mm = &memif_main;
+  memif_if_t *mif;
+  struct sockaddr_un sun;
+  int sockfd;
+  uword *event_data = 0, event_type;
+  unix_file_t template = { 0 };
+  u8 enabled = 0;
+  f64 start_time, last_run_duration = 0, now;
+
+  sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
+  sun.sun_family = AF_UNIX;
+  template.read_function = memif_conn_fd_read_ready;
+
+  while (1)
+    {
+      if (enabled)
+	vlib_process_wait_for_event_or_clock (vm,
+					      (f64) 3 - last_run_duration);
+      else
+	vlib_process_wait_for_event (vm);
+
+      event_type = vlib_process_get_events (vm, &event_data);
+      vec_reset_length (event_data);
+
+      switch (event_type)
+	{
+	case ~0:
+	  break;
+	case MEMIF_PROCESS_EVENT_START:
+	  enabled = 1;
+	  break;
+	case MEMIF_PROCESS_EVENT_STOP:
+	  enabled = 0;
+	  continue;
+	default:
+	  ASSERT (0);
+	}
+
+      last_run_duration = start_time = vlib_time_now (vm);
+      /* *INDENT-OFF* */
+      pool_foreach (mif, mm->interfaces,
+        ({
+	  /* Allow no more than 10us without a pause */
+	  now = vlib_time_now (vm);
+	  if (now > start_time + 10e-6)
+	    {
+	      vlib_process_suspend (vm, 100e-6);	/* suspend for 100 us */
+	      start_time = vlib_time_now (vm);
+	    }
+
+	  if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) == 0)
+	    continue;
+
+	  if (mif->flags & MEMIF_IF_FLAG_CONNECTING)
+	    continue;
+
+	  if (mif->flags & MEMIF_IF_FLAG_CONNECTED)
+	    continue;
+
+	  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
+	    {
+	      strncpy (sun.sun_path, (char *) mif->socket_filename,
+		       sizeof (sun.sun_path) - 1);
+
+	      if (connect
+		  (sockfd, (struct sockaddr *) &sun,
+		   sizeof (struct sockaddr_un)) == 0)
+	        {
+		  mif->connection.fd = sockfd;
+		  template.file_descriptor = sockfd;
+		  template.private_data = mif->if_index << 1;
+		  mif->connection.index = unix_file_add (&unix_main, &template);
+		  memif_connect_master (vm, mif);
+
+		  /* grab another fd */
+		  sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
+		  if (sockfd < 0)
+		    {
+		      DEBUG_UNIX_LOG ("socket AF_UNIX");
+		      return 0;
+		    }
+	        }
+	    }
+        }));
+      /* *INDENT-ON* */
+      last_run_duration = vlib_time_now (vm) - last_run_duration;
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (memif_process_node,static) = {
+  .function = memif_process,
+  .type = VLIB_NODE_TYPE_PROCESS,
+  .name = "memif-process",
+};
+/* *INDENT-ON* */
+
+static void
+memif_close_if (memif_main_t * mm, memif_if_t * mif)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  memif_listener_t *listener = 0;
+  memif_pending_conn_t *pending_conn = 0;
+
+  memif_disconnect (vm, mif);
+
+  if (mif->listener_index != (uword) ~ 0)
+    {
+      listener = pool_elt_at_index (mm->listeners, mif->listener_index);
+      if (--listener->usage_counter == 0)
+	{
+	  /* not used anymore -> remove the socket and pending connections */
+
+	  /* *INDENT-OFF* */
+	  pool_foreach (pending_conn, mm->pending_conns,
+	    ({
+	       if (pending_conn->listener_index == mif->listener_index)
+	         {
+		   memif_remove_pending_conn (pending_conn);
+	         }
+	     }));
+          /* *INDENT-ON* */
+
+	  unix_file_del (&unix_main,
+			 unix_main.file_pool + listener->socket.index);
+	  pool_put (mm->listeners, listener);
+	  unlink ((char *) mif->socket_filename);
+	}
+    }
+
+  if (mif->lockp != 0)
+    {
+      clib_mem_free ((void *) mif->lockp);
+      mif->lockp = 0;
+    }
+
+  mhash_unset (&mm->if_index_by_key, &mif->key, &mif->if_index);
+  vec_free (mif->socket_filename);
+  vec_free (mif->ring_data);
+
+  memset (mif, 0, sizeof (*mif));
+  pool_put (mm->interfaces, mif);
+}
+
+int
+memif_worker_thread_enable ()
+{
+  /* if worker threads are enabled, switch to polling mode */
+  foreach_vlib_main ((
+		       {
+		       vlib_node_set_state (this_vlib_main,
+					    memif_input_node.index,
+					    VLIB_NODE_STATE_POLLING);
+		       }));
+
+  return 0;
+}
+
+int
+memif_worker_thread_disable ()
+{
+  foreach_vlib_main ((
+		       {
+		       vlib_node_set_state (this_vlib_main,
+					    memif_input_node.index,
+					    VLIB_NODE_STATE_INTERRUPT);
+		       }));
+
+  return 0;
+}
+
+int
+memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args)
+{
+  memif_main_t *mm = &memif_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  vnet_main_t *vnm = vnet_get_main ();
+  memif_if_t *mif = 0;
+  vnet_sw_interface_t *sw;
+  clib_error_t *error = 0;
+  int ret = 0;
+  uword *p;
+
+  p = mhash_get (&mm->if_index_by_key, &args->key);
+  if (p)
+    return VNET_API_ERROR_SUBIF_ALREADY_EXISTS;
+
+  pool_get (mm->interfaces, mif);
+  memset (mif, 0, sizeof (*mif));
+  mif->key = args->key;
+  mif->if_index = mif - mm->interfaces;
+  mif->sw_if_index = mif->hw_if_index = mif->per_interface_next_index = ~0;
+  mif->listener_index = ~0;
+  mif->connection.index = mif->interrupt_line.index = ~0;
+  mif->connection.fd = mif->interrupt_line.fd = -1;
+
+  if (tm->n_vlib_mains > 1)
+    {
+      mif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+					   CLIB_CACHE_LINE_BYTES);
+      memset ((void *) mif->lockp, 0, CLIB_CACHE_LINE_BYTES);
+    }
+
+  if (!args->hw_addr_set)
+    {
+      f64 now = vlib_time_now (vm);
+      u32 rnd;
+      rnd = (u32) (now * 1e6);
+      rnd = random_u32 (&rnd);
+
+      memcpy (args->hw_addr + 2, &rnd, sizeof (rnd));
+      args->hw_addr[0] = 2;
+      args->hw_addr[1] = 0xfe;
+    }
+
+  error = ethernet_register_interface (vnm, memif_device_class.index,
+				       mif->if_index, args->hw_addr,
+				       &mif->hw_if_index,
+				       memif_eth_flag_change);
+
+  if (error)
+    {
+      clib_error_report (error);
+      ret = VNET_API_ERROR_SYSCALL_ERROR_1;
+      goto error;
+    }
+
+  sw = vnet_get_hw_sw_interface (vnm, mif->hw_if_index);
+  mif->sw_if_index = sw->sw_if_index;
+
+  mif->log2_ring_size = args->log2_ring_size;
+  mif->buffer_size = args->buffer_size;
+
+  /* TODO: make configurable */
+  mif->num_s2m_rings = 1;
+  mif->num_m2s_rings = 1;
+
+  mhash_set_mem (&mm->if_index_by_key, &args->key, &mif->if_index, 0);
+
+  if (args->socket_filename != 0)
+    mif->socket_filename = args->socket_filename;
+  else
+    mif->socket_filename = vec_dup (mm->default_socket_filename);
+
+  args->sw_if_index = mif->sw_if_index;
+
+  if (args->is_master)
+    {
+      struct sockaddr_un un = { 0 };
+      struct stat file_stat;
+      int on = 1;
+      memif_listener_t *listener = 0;
+
+      if (stat ((char *) mif->socket_filename, &file_stat) == 0)
+	{
+	  if (!S_ISSOCK (file_stat.st_mode))
+	    {
+	      errno = ENOTSOCK;
+	      ret = VNET_API_ERROR_SYSCALL_ERROR_2;
+	      goto error;
+	    }
+	  /* *INDENT-OFF* */
+	  pool_foreach (listener, mm->listeners,
+	    ({
+	       if (listener->sock_dev == file_stat.st_dev &&
+		   listener->sock_ino == file_stat.st_ino)
+	         {
+		   /* attach memif to the existing listener */
+		   mif->listener_index = listener->index;
+		   ++listener->usage_counter;
+		   goto signal;
+	         }
+	     }));
+          /* *INDENT-ON* */
+	  unlink ((char *) mif->socket_filename);
+	}
+
+      pool_get (mm->listeners, listener);
+      memset (listener, 0, sizeof (*listener));
+      listener->socket.fd = -1;
+      listener->socket.index = ~0;
+      listener->index = listener - mm->listeners;
+      listener->usage_counter = 1;
+
+      if ((listener->socket.fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
+	{
+	  ret = VNET_API_ERROR_SYSCALL_ERROR_3;
+	  goto error;
+	}
+
+      un.sun_family = AF_UNIX;
+      strncpy ((char *) un.sun_path, (char *) mif->socket_filename,
+	       sizeof (un.sun_path) - 1);
+
+      if (setsockopt (listener->socket.fd, SOL_SOCKET, SO_PASSCRED,
+		      &on, sizeof (on)) < 0)
+	{
+	  ret = VNET_API_ERROR_SYSCALL_ERROR_4;
+	  goto error;
+	}
+      if (bind (listener->socket.fd, (struct sockaddr *) &un,
+		sizeof (un)) == -1)
+	{
+	  ret = VNET_API_ERROR_SYSCALL_ERROR_5;
+	  goto error;
+	}
+      if (listen (listener->socket.fd, 1) == -1)
+	{
+	  ret = VNET_API_ERROR_SYSCALL_ERROR_6;
+	  goto error;
+	}
+
+      if (stat ((char *) mif->socket_filename, &file_stat) == -1)
+	{
+	  ret = VNET_API_ERROR_SYSCALL_ERROR_7;
+	  goto error;
+	}
+
+      listener->sock_dev = file_stat.st_dev;
+      listener->sock_ino = file_stat.st_ino;
+
+      unix_file_t template = { 0 };
+      template.read_function = memif_conn_fd_accept_ready;
+      template.file_descriptor = listener->socket.fd;
+      template.private_data = listener->index;
+      listener->socket.index = unix_file_add (&unix_main, &template);
+
+      mif->listener_index = listener->index;
+    }
+  else
+    {
+      mif->flags |= MEMIF_IF_FLAG_IS_SLAVE;
+    }
+
+#if 0
+  /* use configured or generate random MAC address */
+  if (!args->hw_addr_set &&
+      tm->n_vlib_mains > 1 && pool_elts (mm->interfaces) == 1)
+    memif_worker_thread_enable ();
+#endif
+
+signal:
+  if (pool_elts (mm->interfaces) == 1)
+    {
+      vlib_process_signal_event (vm, memif_process_node.index,
+				 MEMIF_PROCESS_EVENT_START, 0);
+    }
+  return 0;
+
+error:
+  if (mif->hw_if_index != ~0)
+    {
+      ethernet_delete_interface (vnm, mif->hw_if_index);
+      mif->hw_if_index = ~0;
+    }
+  memif_close_if (mm, mif);
+  return ret;
+}
+
+int
+memif_delete_if (vlib_main_t * vm, u64 key)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  memif_main_t *mm = &memif_main;
+  memif_if_t *mif;
+  uword *p;
+
+  p = mhash_get (&mm->if_index_by_key, &key);
+  if (p == NULL)
+    {
+      clib_warning ("Memory interface with key 0x%" PRIx64 " does not exist",
+		    key);
+      return VNET_API_ERROR_SYSCALL_ERROR_1;
+    }
+  mif = pool_elt_at_index (mm->interfaces, p[0]);
+  mif->flags |= MEMIF_IF_FLAG_DELETING;
+
+  /* bring down the interface */
+  vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0);
+  vnet_sw_interface_set_flags (vnm, mif->sw_if_index, 0);
+
+  /* remove the interface */
+  ethernet_delete_interface (vnm, mif->hw_if_index);
+  mif->hw_if_index = ~0;
+  memif_close_if (mm, mif);
+
+  if (pool_elts (mm->interfaces) == 0)
+    {
+      vlib_process_signal_event (vm, memif_process_node.index,
+				 MEMIF_PROCESS_EVENT_STOP, 0);
+    }
+
+#if 0
+  if (tm->n_vlib_mains > 1 && pool_elts (mm->interfaces) == 0)
+    memif_worker_thread_disable ();
+#endif
+
+  return 0;
+}
+
+static clib_error_t *
+memif_init (vlib_main_t * vm)
+{
+  memif_main_t *mm = &memif_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  vlib_thread_registration_t *tr;
+  uword *p;
+
+  memset (mm, 0, sizeof (memif_main_t));
+
+  mm->input_cpu_first_index = 0;
+  mm->input_cpu_count = 1;
+
+  /* initialize binary API */
+  memif_plugin_api_hookup (vm);
+
+  /* find out which cpus will be used for input */
+  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
+
+  if (tr && tr->count > 0)
+    {
+      mm->input_cpu_first_index = tr->first_index;
+      mm->input_cpu_count = tr->count;
+    }
+
+  mhash_init (&mm->if_index_by_key, sizeof (uword), sizeof (u64));
+
+  vec_validate_aligned (mm->rx_buffers, tm->n_vlib_mains - 1,
+			CLIB_CACHE_LINE_BYTES);
+
+  /* set default socket filename */
+  vec_validate (mm->default_socket_filename,
+		strlen (MEMIF_DEFAULT_SOCKET_FILENAME));
+  strncpy ((char *) mm->default_socket_filename,
+	   MEMIF_DEFAULT_SOCKET_FILENAME,
+	   vec_len (mm->default_socket_filename) - 1);
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (memif_init);
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .description = "Packet Memory Interface (experimetal)",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/memif/memif.h b/src/plugins/memif/memif.h
new file mode 100644
index 00000000..a7a88e07
--- /dev/null
+++ b/src/plugins/memif/memif.h
@@ -0,0 +1,263 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+typedef struct
+{
+  u16 version;
+#define MEMIF_VERSION_MAJOR 0
+#define MEMIF_VERSION_MINOR 1
+#define MEMIF_VERSION ((MEMIF_VERSION_MAJOR << 8) | MEMIF_VERSION_MINOR)
+  u8 type;
+#define MEMIF_MSG_TYPE_CONNECT_REQ  0
+#define MEMIF_MSG_TYPE_CONNECT_RESP 1
+#define MEMIF_MSG_TYPE_DISCONNECT   2
+
+  /* Connection-request parameters: */
+  u64 key;
+  u8 log2_ring_size;
+#define MEMIF_DEFAULT_RING_SIZE 1024
+  u16 num_s2m_rings;
+  u16 num_m2s_rings;
+  u16 buffer_size;
+#define MEMIF_DEFAULT_BUFFER_SIZE 2048
+  u32 shared_mem_size;
+
+  /* Connection-response parameters: */
+  u8 retval;
+} memif_msg_t;
+
+typedef struct __attribute__ ((packed))
+{
+  u16 flags;
+#define MEMIF_DESC_FLAG_NEXT (1 << 0)
+  u16 region;
+  u32 buffer_length;
+  u32 length;;
+  u8 reserved[4];
+  u64 offset;
+  u64 metadata;
+} memif_desc_t;
+
+STATIC_ASSERT_SIZEOF (memif_desc_t, 32);
+
+typedef struct
+{
+  u16 head __attribute__ ((aligned (128)));
+  u16 tail __attribute__ ((aligned (128)));
+  memif_desc_t desc[0] __attribute__ ((aligned (128)));
+} memif_ring_t;
+
+typedef struct
+{
+  u32 cookie __attribute__ ((aligned (128)));
+} memif_shm_t;
+
+
+typedef struct
+{
+  u16 last_head;
+  u16 last_tail;
+} memif_ring_data_t;
+
+typedef struct
+{
+  int fd;
+  u32 index;
+} memif_file_t;
+
+typedef struct
+{
+  uword index;
+  dev_t sock_dev;
+  ino_t sock_ino;
+  memif_file_t socket;
+  u16 usage_counter;
+} memif_listener_t;
+
+typedef struct
+{
+  uword index;
+  memif_file_t connection;
+  uword listener_index;
+} memif_pending_conn_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  volatile u32 *lockp;
+  u32 flags;
+#define MEMIF_IF_FLAG_ADMIN_UP   (1 << 0)
+#define MEMIF_IF_FLAG_IS_SLAVE   (1 << 1)
+#define MEMIF_IF_FLAG_CONNECTING (1 << 2)
+#define MEMIF_IF_FLAG_CONNECTED  (1 << 3)
+#define MEMIF_IF_FLAG_DELETING   (1 << 4)
+
+  u64 key;
+  uword if_index;
+  u32 hw_if_index;
+  u32 sw_if_index;
+
+  u32 per_interface_next_index;
+
+  uword listener_index;
+  memif_file_t connection;
+  memif_file_t interrupt_line;
+  u8 *socket_filename;
+
+  void **regions;
+
+  u8 log2_ring_size;
+  u8 num_s2m_rings;
+  u8 num_m2s_rings;
+  u16 buffer_size;
+
+  memif_ring_data_t *ring_data;
+
+  /* remote info */
+  pid_t remote_pid;
+  uid_t remote_uid;
+} memif_if_t;
+
+typedef struct
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+
+  /** API message ID base */
+  u16 msg_id_base;
+
+  /* pool of all memory interfaces */
+  memif_if_t *interfaces;
+
+  /* pool of all listeners */
+  memif_listener_t *listeners;
+
+  /* pool of pending connections */
+  memif_pending_conn_t *pending_conns;
+
+  /* bitmap of pending rx interfaces */
+  uword *pending_input_bitmap;
+
+  /* rx buffer cache */
+  u32 **rx_buffers;
+
+  /* hash of all registered keys */
+  mhash_t if_index_by_key;
+
+  /* first cpu index */
+  u32 input_cpu_first_index;
+
+  /* total cpu count */
+  u32 input_cpu_count;
+
+  /* configuration */
+  u8 *default_socket_filename;
+#define MEMIF_DEFAULT_SOCKET_FILENAME  "/var/vpp/memif.sock"
+} memif_main_t;
+
+extern memif_main_t memif_main;
+extern vnet_device_class_t memif_device_class;
+extern vlib_node_registration_t memif_input_node;
+
+enum
+{
+  MEMIF_PROCESS_EVENT_START = 1,
+  MEMIF_PROCESS_EVENT_STOP = 2,
+} memif_process_event_t;
+
+typedef struct
+{
+  u64 key;
+  u8 *socket_filename;
+  u8 is_master;
+  u8 log2_ring_size;
+  u16 buffer_size;
+  u8 hw_addr_set;
+  u8 hw_addr[6];
+
+  /* return */
+  u32 sw_if_index;
+} memif_create_if_args_t;
+
+int memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args);
+int memif_delete_if (vlib_main_t * vm, u64 key);
+clib_error_t *memif_plugin_api_hookup (vlib_main_t * vm);
+
+#ifndef __NR_memfd_create
+#if defined __x86_64__
+#define __NR_memfd_create 319
+#elif defined __arm__
+#define __NR_memfd_create 385
+#elif defined __aarch64__
+#define __NR_memfd_create 279
+#else
+#error "__NR_memfd_create unknown for this architecture"
+#endif
+#endif
+
+static inline int
+memfd_create (const char *name, unsigned int flags)
+{
+  return syscall (__NR_memfd_create, name, flags);
+}
+
+typedef enum
+{
+  MEMIF_RING_S2M = 0,
+  MEMIF_RING_M2S = 1
+} memif_ring_type_t;
+
+static_always_inline memif_ring_t *
+memif_get_ring (memif_if_t * mif, memif_ring_type_t type, u16 ring_num)
+{
+  if (vec_len (mif->regions) == 0)
+    return NULL;
+  void *p = mif->regions[0];
+  int ring_size =
+    sizeof (memif_ring_t) +
+    sizeof (memif_desc_t) * (1 << mif->log2_ring_size);
+  p += sizeof (memif_shm_t);
+  p += (ring_num + type * mif->num_s2m_rings) * ring_size;
+
+  return (memif_ring_t *) p;
+}
+
+static_always_inline void *
+memif_get_buffer (memif_if_t * mif, memif_ring_t * ring, u16 slot)
+{
+  u16 region = ring->desc[slot].region;
+  return mif->regions[region] + ring->desc[slot].offset;
+}
+
+#ifndef F_LINUX_SPECIFIC_BASE
+#define F_LINUX_SPECIFIC_BASE 1024
+#endif
+#define MFD_ALLOW_SEALING       0x0002U
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
+
+#define F_SEAL_SEAL     0x0001	/* prevent further seals from being set */
+#define F_SEAL_SHRINK   0x0002	/* prevent file from shrinking */
+#define F_SEAL_GROW     0x0004	/* prevent file from growing */
+#define F_SEAL_WRITE    0x0008	/* prevent writes */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/memif/memif_all_api_h.h b/src/plugins/memif/memif_all_api_h.h
new file mode 100644
index 00000000..9729ec16
--- /dev/null
+++ b/src/plugins/memif/memif_all_api_h.h
@@ -0,0 +1,18 @@
+/*
+ * memif_all_api_h.h - plug-in api #include file
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include 
diff --git a/src/plugins/memif/memif_api.c b/src/plugins/memif/memif_api.c
new file mode 100644
index 00000000..1ade317e
--- /dev/null
+++ b/src/plugins/memif/memif_api.c
@@ -0,0 +1,332 @@
+/*
+ *------------------------------------------------------------------
+ * memif_api.c - memif api
+ *
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+/* define message IDs */
+#include 
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+/*
+ * A handy macro to set up a message reply.
+ * Assumes that the following variables are available:
+ * mp - pointer to request message
+ * rmp - pointer to reply message type
+ * rv - return value
+ */
+#define REPLY_MACRO(t)                                          \
+do {                                                            \
+    unix_shared_memory_queue_t * q =                            \
+    vl_api_client_index_to_input_queue (mp->client_index);      \
+    if (!q)                                                     \
+        return;                                                 \
+                                                                \
+    rmp = vl_msg_api_alloc (sizeof (*rmp));                     \
+    rmp->_vl_msg_id = htons ((t)+mm->msg_id_base);              \
+    rmp->context = mp->context;                                 \
+    rmp->retval = htonl (rv);                                   \
+                                                                \
+    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+} while(0);
+
+#define REPLY_MACRO2(t, body)                                   \
+do {                                                            \
+    unix_shared_memory_queue_t * q =                            \
+    vl_api_client_index_to_input_queue (mp->client_index);      \
+    if (!q)                                                     \
+        return;                                                 \
+                                                                \
+    rmp = vl_msg_api_alloc (sizeof (*rmp));                     \
+    rmp->_vl_msg_id = htons ((t)+mm->msg_id_base);              \
+    rmp->context = mp->context;                                 \
+    rmp->retval = htonl (rv);                                   \
+    do {body;} while (0);                                       \
+    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+} while(0);
+
+#define foreach_memif_plugin_api_msg                     \
+_(MEMIF_CREATE, memif_create)                            \
+_(MEMIF_DELETE, memif_delete)                            \
+_(MEMIF_DETAILS, memif_details)                          \
+_(MEMIF_DUMP, memif_dump)                                \
+
+/**
+ * @brief Message handler for memif_create API.
+ * @param mp vl_api_memif_create_t * mp the api message
+ */
+void
+vl_api_memif_create_t_handler (vl_api_memif_create_t * mp)
+{
+  memif_main_t *mm = &memif_main;
+  vlib_main_t *vm = vlib_get_main ();
+  vl_api_memif_create_reply_t *rmp;
+  memif_create_if_args_t args = { 0 };
+  u32 ring_size = MEMIF_DEFAULT_RING_SIZE;
+  static const u8 empty_hw_addr[6];
+  int rv = 0;
+
+  /* key */
+  args.key = clib_net_to_host_u64 (mp->key);
+
+  /* socket filename */
+  mp->socket_filename[ARRAY_LEN (mp->socket_filename) - 1] = 0;
+  if (strlen ((char *) mp->socket_filename) > 0)
+    {
+      vec_validate (args.socket_filename,
+		    strlen ((char *) mp->socket_filename));
+      strncpy ((char *) args.socket_filename, (char *) mp->socket_filename,
+	       vec_len (args.socket_filename));
+    }
+
+  /* role */
+  args.is_master = (mp->role == 0);
+
+  /* ring size */
+  if (mp->ring_size)
+    {
+      ring_size = ntohl (mp->ring_size);
+    }
+  if (!is_pow2 (ring_size))
+    {
+      rv = VNET_API_ERROR_INVALID_ARGUMENT;
+      goto reply;
+    }
+  args.log2_ring_size = min_log2 (ring_size);
+
+  /* buffer size */
+  args.buffer_size = MEMIF_DEFAULT_BUFFER_SIZE;
+  if (mp->buffer_size)
+    {
+      args.buffer_size = ntohs (mp->buffer_size);
+    }
+
+  /* MAC address */
+  if (memcmp (mp->hw_addr, empty_hw_addr, 6) != 0)
+    {
+      memcpy (args.hw_addr, mp->hw_addr, 6);
+      args.hw_addr_set = 1;
+    }
+
+  rv = memif_create_if (vm, &args);
+
+reply:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_MEMIF_CREATE_REPLY,
+    ({
+       rmp->sw_if_index = htonl (args.sw_if_index);
+    }));
+  /* *INDENT-ON* */
+}
+
+/**
+ * @brief Message handler for memif_delete API.
+ * @param mp vl_api_memif_delete_t * mp the api message
+ */
+void
+vl_api_memif_delete_t_handler (vl_api_memif_delete_t * mp)
+{
+  memif_main_t *mm = &memif_main;
+  memif_if_t *mif;
+  vlib_main_t *vm = vlib_get_main ();
+  vl_api_memif_delete_reply_t *rmp;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  pool_foreach (mif, mm->interfaces,
+    ({
+      if (sw_if_index == mif->sw_if_index)
+	{
+	  rv = memif_delete_if (vm, mif->key);
+	  goto reply;
+	}
+    }));
+  /* *INDENT-ON* */
+
+  rv = VNET_API_ERROR_INVALID_SW_IF_INDEX;
+
+reply:
+  REPLY_MACRO (VL_API_MEMIF_DELETE_REPLY);
+}
+
+static void
+send_memif_details (unix_shared_memory_queue_t * q,
+		    memif_if_t * mif,
+		    vnet_sw_interface_t * swif,
+		    u8 * interface_name, u32 context)
+{
+  vl_api_memif_details_t *mp;
+  vnet_main_t *vnm = vnet_get_main ();
+  memif_main_t *mm = &memif_main;
+  vnet_hw_interface_t *hwif;
+
+  hwif = vnet_get_sup_hw_interface (vnm, swif->sw_if_index);
+
+  mp = vl_msg_api_alloc (sizeof (*mp));
+  memset (mp, 0, sizeof (*mp));
+
+  mp->_vl_msg_id = htons (VL_API_MEMIF_DETAILS + mm->msg_id_base);
+  mp->context = context;
+
+  mp->sw_if_index = htonl (swif->sw_if_index);
+  strncpy ((char *) mp->if_name,
+	   (char *) interface_name, ARRAY_LEN (mp->if_name) - 1);
+  memcpy (mp->hw_addr, hwif->hw_address, ARRAY_LEN (mp->hw_addr));
+
+  mp->key = clib_host_to_net_u64 (mif->key);
+  mp->role = (mif->flags & MEMIF_IF_FLAG_IS_SLAVE) ? 1 : 0;
+  strncpy ((char *) mp->socket_filename,
+	   (char *) mif->socket_filename,
+	   ARRAY_LEN (mp->socket_filename) - 1);
+
+  mp->ring_size = htonl (1 << mif->log2_ring_size);
+  mp->buffer_size = htons (mif->buffer_size);
+
+  mp->admin_up_down = (swif->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? 1 : 0;
+  mp->link_up_down = (hwif->flags & VNET_HW_INTERFACE_FLAG_LINK_UP) ? 1 : 0;
+
+  vl_msg_api_send_shmem (q, (u8 *) & mp);
+}
+
+/**
+ * @brief Message handler for memif_dump API.
+ * @param mp vl_api_memif_dump_t * mp the api message
+ */
+void
+vl_api_memif_dump_t_handler (vl_api_memif_dump_t * mp)
+{
+  memif_main_t *mm = &memif_main;
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_sw_interface_t *swif;
+  memif_if_t *mif;
+  u8 *if_name = 0;
+  unix_shared_memory_queue_t *q;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (mif, mm->interfaces,
+    ({
+      swif = vnet_get_sw_interface (vnm, mif->sw_if_index);
+
+      if_name = format (if_name, "%U%c",
+			format_vnet_sw_interface_name,
+			vnm, swif, 0);
+
+      send_memif_details (q, mif, swif, if_name, mp->context);
+      _vec_len (if_name) = 0;
+    }));
+  /* *INDENT-ON* */
+
+  vec_free (if_name);
+}
+
+/**
+ * @brief Message handler for memif_details API.
+ * @param mp vl_api_memif_details_t * mp the api message
+ */
+void
+vl_api_memif_details_t_handler (vl_api_memif_details_t * mp)
+{
+  clib_warning ("BUG");
+}
+
+#define vl_msg_name_crc_list
+#include 
+#undef vl_msg_name_crc_list
+
+static void
+setup_message_id_table (memif_main_t * mm, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + mm->msg_id_base);
+  foreach_vl_msg_name_crc_memif;
+#undef _
+}
+
+/* Set up the API message handling tables */
+clib_error_t *
+memif_plugin_api_hookup (vlib_main_t * vm)
+{
+  memif_main_t *mm = &memif_main;
+  api_main_t *am = &api_main;
+  u8 *name;
+
+  /* Construct the API name */
+  name = format (0, "memif_%08x%c", api_version, 0);
+
+  /* Ask for a correctly-sized block of API message decode slots */
+  mm->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + mm->msg_id_base),     \
+                           #n,                                  \
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_memif_plugin_api_msg;
+#undef _
+
+  /*
+   * Set up the (msg_name, crc, message-id) table
+   */
+  setup_message_id_table (mm, am);
+
+  vec_free (name);
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/memif/memif_msg_enum.h b/src/plugins/memif/memif_msg_enum.h
new file mode 100644
index 00000000..74efee00
--- /dev/null
+++ b/src/plugins/memif/memif_msg_enum.h
@@ -0,0 +1,31 @@
+/*
+ * memif_msg_enum.h - vpp engine plug-in message enumeration
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_memif_msg_enum_h
+#define included_memif_msg_enum_h
+
+#include 
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include 
+  /* We'll want to know how many messages IDs we need... */
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_memif_msg_enum_h */
diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c
new file mode 100644
index 00000000..298472b0
--- /dev/null
+++ b/src/plugins/memif/node.c
@@ -0,0 +1,383 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#define _GNU_SOURCE
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#define foreach_memif_input_error
+
+typedef enum
+{
+#define _(f,s) MEMIF_INPUT_ERROR_##f,
+  foreach_memif_input_error
+#undef _
+    MEMIF_INPUT_N_ERROR,
+} memif_input_error_t;
+
+static char *memif_input_error_strings[] = {
+#define _(n,s) s,
+  foreach_memif_input_error
+#undef _
+};
+
+typedef struct
+{
+  u32 next_index;
+  u32 hw_if_index;
+  u16 ring;
+} memif_input_trace_t;
+
+static u8 *
+format_memif_input_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  memif_input_trace_t *t = va_arg (*args, memif_input_trace_t *);
+  uword indent = format_get_indent (s);
+
+  s = format (s, "memif: hw_if_index %d next-index %d",
+	      t->hw_if_index, t->next_index);
+  s = format (s, "\n%Uslot: ring %u", format_white_space, indent + 2,
+	      t->ring);
+  return s;
+}
+
+static_always_inline void
+memif_prefetch (vlib_main_t * vm, u32 bi)
+{
+  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
+  vlib_prefetch_buffer_header (b, STORE);
+  CLIB_PREFETCH (b->data, CLIB_CACHE_LINE_BYTES, STORE);
+}
+
+static_always_inline uword
+memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+			   vlib_frame_t * frame, memif_if_t * mif,
+			   memif_ring_type_t type)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  u8 rid = 0;			/* Ring id */
+  memif_ring_t *ring = memif_get_ring (mif, type, rid);
+  memif_ring_data_t *rd =
+    vec_elt_at_index (mif->ring_data, rid + type * mif->num_s2m_rings);
+  u16 head;
+
+  u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+  uword n_trace = vlib_get_trace_count (vm, node);
+  memif_main_t *nm = &memif_main;
+  u32 n_rx_packets = 0;
+  u32 n_rx_bytes = 0;
+  u32 *to_next = 0;
+  u32 n_free_bufs;
+  u32 cpu_index = os_get_cpu_number ();
+  u32 bi0, bi1;
+  vlib_buffer_t *b0, *b1;
+  u16 ring_size = 1 << mif->log2_ring_size;
+  u16 mask = ring_size - 1;
+  u16 num_slots;
+  void *mb0, *mb1;
+
+  if (mif->per_interface_next_index != ~0)
+    next_index = mif->per_interface_next_index;
+
+  n_free_bufs = vec_len (nm->rx_buffers[cpu_index]);
+  if (PREDICT_FALSE (n_free_bufs < ring_size))
+    {
+      vec_validate (nm->rx_buffers[cpu_index], ring_size + n_free_bufs - 1);
+      n_free_bufs +=
+	vlib_buffer_alloc (vm, &nm->rx_buffers[cpu_index][n_free_bufs],
+			   ring_size);
+      _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs;
+    }
+
+  head = ring->head;
+  if (head == rd->last_head)
+    return 0;
+
+  if (head > rd->last_head)
+    num_slots = head - rd->last_head;
+  else
+    num_slots = ring_size - rd->last_head + head;
+
+  while (num_slots)
+    {
+      u32 n_left_to_next;
+      u32 next0 = next_index;
+      u32 next1 = next_index;
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (num_slots > 5 && n_left_to_next > 2)
+	{
+	  if (PREDICT_TRUE (rd->last_head + 5 < ring_size))
+	    {
+	      CLIB_PREFETCH (memif_get_buffer (mif, ring, rd->last_head + 2),
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	      CLIB_PREFETCH (memif_get_buffer (mif, ring, rd->last_head + 3),
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	      CLIB_PREFETCH (&ring->desc[rd->last_head + 4],
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	      CLIB_PREFETCH (&ring->desc[rd->last_head + 5],
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	    }
+	  else
+	    {
+	      CLIB_PREFETCH (memif_get_buffer
+			     (mif, ring, (rd->last_head + 2) % mask),
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	      CLIB_PREFETCH (memif_get_buffer
+			     (mif, ring, (rd->last_head + 3) % mask),
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	      CLIB_PREFETCH (&ring->desc[(rd->last_head + 4) % mask],
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	      CLIB_PREFETCH (&ring->desc[(rd->last_head + 5) % mask],
+			     CLIB_CACHE_LINE_BYTES, LOAD);
+	    }
+	  /* get empty buffer */
+	  u32 last_buf = vec_len (nm->rx_buffers[cpu_index]) - 1;
+	  bi0 = nm->rx_buffers[cpu_index][last_buf];
+	  bi1 = nm->rx_buffers[cpu_index][last_buf - 1];
+	  _vec_len (nm->rx_buffers[cpu_index]) -= 2;
+
+	  if (last_buf > 4)
+	    {
+	      memif_prefetch (vm, nm->rx_buffers[cpu_index][last_buf - 2]);
+	      memif_prefetch (vm, nm->rx_buffers[cpu_index][last_buf - 3]);
+	    }
+
+	  /* enqueue buffer */
+	  to_next[0] = bi0;
+	  to_next[1] = bi1;
+	  to_next += 2;
+	  n_left_to_next -= 2;
+
+	  /* fill buffer metadata */
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = mif->sw_if_index;
+	  vnet_buffer (b1)->sw_if_index[VLIB_RX] = mif->sw_if_index;
+
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+	  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+	  /* copy buffer */
+	  mb0 = memif_get_buffer (mif, ring, rd->last_head);
+	  clib_memcpy (vlib_buffer_get_current (b0), mb0,
+		       CLIB_CACHE_LINE_BYTES);
+	  b0->current_length = ring->desc[rd->last_head].length;
+	  rd->last_head = (rd->last_head + 1) & mask;
+
+	  mb1 = memif_get_buffer (mif, ring, rd->last_head);
+	  clib_memcpy (vlib_buffer_get_current (b1), mb1,
+		       CLIB_CACHE_LINE_BYTES);
+	  b1->current_length = ring->desc[rd->last_head].length;
+	  rd->last_head = (rd->last_head + 1) & mask;
+
+	  if (b0->current_length > CLIB_CACHE_LINE_BYTES)
+	    clib_memcpy (vlib_buffer_get_current (b0) + CLIB_CACHE_LINE_BYTES,
+			 mb0 + CLIB_CACHE_LINE_BYTES,
+			 b0->current_length - CLIB_CACHE_LINE_BYTES);
+
+	  if (b1->current_length > CLIB_CACHE_LINE_BYTES)
+	    clib_memcpy (vlib_buffer_get_current (b1) + CLIB_CACHE_LINE_BYTES,
+			 mb1 + CLIB_CACHE_LINE_BYTES,
+			 b1->current_length - CLIB_CACHE_LINE_BYTES);
+
+	  /* trace */
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
+
+	  if (PREDICT_FALSE (n_trace > 0))
+	    {
+	      if (b0)
+		{
+		  memif_input_trace_t *tr;
+		  vlib_trace_buffer (vm, node, next0, b0,
+				     /* follow_chain */ 0);
+		  vlib_set_trace_count (vm, node, --n_trace);
+		  tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  tr->next_index = next0;
+		  tr->hw_if_index = mif->hw_if_index;
+		  tr->ring = rid;
+		}
+
+	      if (n_trace && b1)
+		{
+		  memif_input_trace_t *tr;
+		  vlib_trace_buffer (vm, node, next1, b1,
+				     /* follow_chain */ 0);
+		  vlib_set_trace_count (vm, node, --n_trace);
+		  tr = vlib_add_trace (vm, node, b1, sizeof (*tr));
+		  tr->next_index = next1;
+		  tr->hw_if_index = mif->hw_if_index;
+		  tr->ring = rid;
+		}
+	    }
+
+	  /* redirect if feature path enabled */
+	  vnet_feature_start_device_input_x2 (mif->sw_if_index,
+					      &next0, &next1, b0, b1);
+
+	  /* enqueue */
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
+					   n_left_to_next,
+					   bi0, bi1, next0, next1);
+
+	  /* next packet */
+	  num_slots -= 2;
+	  n_rx_packets += 2;
+	  n_rx_bytes += b0->current_length;
+	  n_rx_bytes += b1->current_length;
+	}
+      while (num_slots && n_left_to_next)
+	{
+	  /* get empty buffer */
+	  u32 last_buf = vec_len (nm->rx_buffers[cpu_index]) - 1;
+	  bi0 = nm->rx_buffers[cpu_index][last_buf];
+	  _vec_len (nm->rx_buffers[cpu_index]) = last_buf;
+
+	  /* enqueue buffer */
+	  to_next[0] = bi0;
+	  to_next += 1;
+	  n_left_to_next--;
+
+	  /* fill buffer metadata */
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b0->current_length = ring->desc[rd->last_head].length;
+	  vnet_buffer (b0)->sw_if_index[VLIB_RX] = mif->sw_if_index;
+	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+	  /* copy buffer */
+	  mb0 = memif_get_buffer (mif, ring, rd->last_head);
+	  clib_memcpy (vlib_buffer_get_current (b0), mb0,
+		       CLIB_CACHE_LINE_BYTES);
+	  if (b0->current_length > CLIB_CACHE_LINE_BYTES)
+	    clib_memcpy (vlib_buffer_get_current (b0) + CLIB_CACHE_LINE_BYTES,
+			 mb0 + CLIB_CACHE_LINE_BYTES,
+			 b0->current_length - CLIB_CACHE_LINE_BYTES);
+
+	  /* trace */
+	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+
+	  if (PREDICT_FALSE (n_trace > 0))
+	    {
+	      if (b0)
+		{
+		  memif_input_trace_t *tr;
+		  vlib_trace_buffer (vm, node, next0, b0,
+				     /* follow_chain */ 0);
+		  vlib_set_trace_count (vm, node, --n_trace);
+		  tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+		  tr->next_index = next0;
+		  tr->hw_if_index = mif->hw_if_index;
+		  tr->ring = rid;
+		}
+	    }
+
+
+	  /* redirect if feature path enabled */
+	  vnet_feature_start_device_input_x1 (mif->sw_if_index, &next0, b0);
+
+	  /* enqueue */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+
+	  /* next packet */
+	  rd->last_head = (rd->last_head + 1) & mask;
+	  num_slots--;
+	  n_rx_packets++;
+	  n_rx_bytes += b0->current_length;
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  CLIB_MEMORY_STORE_BARRIER ();
+  ring->tail = head;
+
+  vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters
+				   + VNET_INTERFACE_COUNTER_RX, cpu_index,
+				   mif->hw_if_index, n_rx_packets,
+				   n_rx_bytes);
+
+  return n_rx_packets;
+}
+
+static uword
+memif_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		vlib_frame_t * frame)
+{
+  u32 n_rx_packets = 0;
+  u32 cpu_index = os_get_cpu_number ();
+  memif_main_t *nm = &memif_main;
+  memif_if_t *mif;
+
+  /* *INDENT-OFF* */
+  pool_foreach (mif, nm->interfaces,
+    ({
+      if (mif->flags & MEMIF_IF_FLAG_ADMIN_UP &&
+	  mif->flags & MEMIF_IF_FLAG_CONNECTED &&
+	  (mif->if_index % nm->input_cpu_count) ==
+	  (cpu_index - nm->input_cpu_first_index))
+	{
+	  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
+	    n_rx_packets +=
+	      memif_device_input_inline (vm, node, frame, mif,
+					 MEMIF_RING_M2S);
+	  else
+	    n_rx_packets +=
+	      memif_device_input_inline (vm, node, frame, mif,
+					 MEMIF_RING_S2M);
+	}
+    }));
+  /* *INDENT-ON* */
+
+  return n_rx_packets;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (memif_input_node) = {
+  .function = memif_input_fn,
+  .name = "memif-input",
+  .sibling_of = "device-input",
+  .format_trace = format_memif_input_trace,
+  .type = VLIB_NODE_TYPE_INPUT,
+  .state = VLIB_NODE_STATE_INTERRUPT,
+  .n_errors = MEMIF_INPUT_N_ERROR,
+  .error_strings = memif_input_error_strings,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (memif_input_node, memif_input_fn)
+/* *INDENT-ON* */
+
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h
index 0386c756..fbb2a21c 100644
--- a/src/vppinfra/clib.h
+++ b/src/vppinfra/clib.h
@@ -100,6 +100,12 @@
 /* Full memory barrier (read and write). */
 #define CLIB_MEMORY_BARRIER() __sync_synchronize ()
 
+#if __x86_64__
+#define CLIB_MEMORY_STORE_BARRIER() __builtin_ia32_sfence ()
+#else
+#define CLIB_MEMORY_STORE_BARRIER() __sync_synchronize ()
+#endif
+
 /* Arranges for function to be called before main. */
 #define INIT_FUNCTION(decl)			\
   decl __attribute ((constructor));		\
-- 
cgit 1.2.3-korg


From 4521afa9a643983c789b6b3a77b14c0a95d5e0b2 Mon Sep 17 00:00:00 2001
From: Pablo Camarillo 
Date: Thu, 16 Mar 2017 10:43:05 +0100
Subject: Bugfixing and documentation for SRv6

- Fixed three coverity issues
- Linked SRv6 docs
- Moved sample plugin to examples folder
- Fixed bug with hash. Now everything is using mhash. Potentially in the future we want to do bihash.

Change-Id: Ie03a13c8fecb1e315e67d0596cbd23220779aaf2
Signed-off-by: Pablo Camarillo 
---
 doxygen/Makefile                                   |   6 +-
 doxygen/user_doc.md                                |  13 +-
 src/configure.ac                                   |   1 -
 src/examples/srv6-sample-localsid/node.c           | 200 +++++++++++++++++++++
 .../srv6-sample-localsid/srv6_localsid_sample.c    | 179 ++++++++++++++++++
 .../srv6-sample-localsid/srv6_localsid_sample.h    |  61 +++++++
 .../srv6_sample_localsid_doc.md                    |  30 ++++
 src/plugins/Makefile.am                            |   4 -
 src/plugins/sample_srv6_localsid.am                |  23 ---
 src/plugins/srv6-localsid/node.c                   | 200 ---------------------
 src/plugins/srv6-localsid/srv6_localsid_sample.c   | 179 ------------------
 src/plugins/srv6-localsid/srv6_localsid_sample.h   |  61 -------
 .../srv6-localsid/srv6_sample_localsid_doc.md      |  38 ----
 src/vnet/sr/sr.h                                   |  30 +---
 src/vnet/sr/sr_doc.md                              | 132 ++------------
 src/vnet/sr/sr_localsid.c                          |  18 +-
 src/vnet/sr/sr_localsid.md                         |  58 ++++++
 src/vnet/sr/sr_policy.md                           |  56 ++++++
 src/vnet/sr/sr_policy_rewrite.c                    |  28 ++-
 src/vnet/sr/sr_steering.c                          |  59 +++---
 src/vnet/sr/sr_steering.md                         |  11 ++
 21 files changed, 672 insertions(+), 715 deletions(-)
 create mode 100644 src/examples/srv6-sample-localsid/node.c
 create mode 100755 src/examples/srv6-sample-localsid/srv6_localsid_sample.c
 create mode 100644 src/examples/srv6-sample-localsid/srv6_localsid_sample.h
 create mode 100644 src/examples/srv6-sample-localsid/srv6_sample_localsid_doc.md
 delete mode 100644 src/plugins/sample_srv6_localsid.am
 delete mode 100644 src/plugins/srv6-localsid/node.c
 delete mode 100755 src/plugins/srv6-localsid/srv6_localsid_sample.c
 delete mode 100644 src/plugins/srv6-localsid/srv6_localsid_sample.h
 delete mode 100644 src/plugins/srv6-localsid/srv6_sample_localsid_doc.md
 create mode 100644 src/vnet/sr/sr_localsid.md
 create mode 100644 src/vnet/sr/sr_policy.md
 create mode 100644 src/vnet/sr/sr_steering.md

(limited to 'src/plugins/Makefile.am')

diff --git a/doxygen/Makefile b/doxygen/Makefile
index 973ca07f..9caa4079 100644
--- a/doxygen/Makefile
+++ b/doxygen/Makefile
@@ -53,7 +53,8 @@ DOXY_SRC_DIRECTORIES = \
 	$(DOXY_SRC)/vlibsocket \
 	$(DOXY_SRC)/vnet \
 	$(DOXY_SRC)/vpp \
-	$(DOXY_SRC)/vpp-api
+	$(DOXY_SRC)/vpp-api \
+	$(DOXY_SRC)/examples
 
 # Input directories and files
 DOXY_INPUT ?= \
@@ -72,9 +73,8 @@ DOXY_INPUT := $(subst $(WS_ROOT)/,,$(DOXY_INPUT))
 # These must be left-anchored paths for the regexp below to work.
 DOXY_EXCLUDE ?= \
 	$(DOXY_SRC)/vlib/vlib/buffer.c \
-	$(DOXY_SRC)/vlib/example \
 	$(DOXY_SRC)/vpp-api/lua \
-	plugins/sample-plugin
+	$(DOXY_SRC)/examples/sample-plugin
 
 # Generate a regexp for filenames to exclude
 DOXY_EXCLUDE_REGEXP = ($(subst .,\.,$(shell echo '$(strip $(DOXY_EXCLUDE))' | sed -e 's/ /|/g')))
diff --git a/doxygen/user_doc.md b/doxygen/user_doc.md
index 40303439..29df6156 100644
--- a/doxygen/user_doc.md
+++ b/doxygen/user_doc.md
@@ -4,13 +4,14 @@ User Documentation    {#user_doc}
 Several modules provide operational, dataplane-user focused documentation.
 
 - [GUI guided user demo](https://wiki.fd.io/view/VPP_Sandbox/vpp-userdemo)
-- @subpage qos_doc
-- @subpage ipsec_gre_doc
-- @subpage dpdk_crypto_ipsec_doc
-- @subpage map_doc
-- @subpage lldp_doc
+- @subpage bfd_doc
 - @subpage ioam_plugin_doc
+- @subpage ipsec_gre_doc
 - @subpage lb_plugin_doc
+- @subpage lldp_doc
+- @subpage map_doc
+- @subpage dpdk_crypto_ipsec_doc
 - @subpage flowperpkt_plugin_doc
+- @subpage qos_doc
 - @subpage span_doc
-- @subpage bfd_doc
+- @subpage srv6_doc
diff --git a/src/configure.ac b/src/configure.ac
index 0ca1af11..581b0968 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -154,7 +154,6 @@ PLUGIN_ENABLED(lb)
 PLUGIN_ENABLED(memif)
 PLUGIN_ENABLED(sixrd)
 PLUGIN_ENABLED(snat)
-PLUGIN_DISABLED(srv6sample)
 
 ###############################################################################
 # Dependency checks
diff --git a/src/examples/srv6-sample-localsid/node.c b/src/examples/srv6-sample-localsid/node.c
new file mode 100644
index 00000000..7bae9cd7
--- /dev/null
+++ b/src/examples/srv6-sample-localsid/node.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+typedef struct {
+  u32 localsid_index;
+} srv6_localsid_sample_trace_t;
+
+/* packet trace format function */
+static u8 * format_srv6_localsid_sample_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  srv6_localsid_sample_trace_t * t = va_arg (*args, srv6_localsid_sample_trace_t *);
+  s = format (s, "SRv6-sample-localsid: localsid_index %d\n",
+              t->localsid_index);
+  return s;
+}
+
+vlib_node_registration_t srv6_localsid_sample_node;
+
+#define foreach_srv6_localsid_counter \
+_(PROCESSED, "srv6-sample-localsid processed packets") \
+_(NO_SRH, "(Error) No SRH.")
+
+typedef enum {
+#define _(sym,str) SRV6_LOCALSID_COUNTER_##sym,
+  foreach_srv6_localsid_counter
+#undef _
+  SRV6_LOCALSID_N_COUNTERS,
+} srv6_localsid_sample_counters;
+
+static char * srv6_localsid_counter_strings[] = {
+#define _(sym,string) string,
+  foreach_srv6_localsid_counter
+#undef _
+};
+
+typedef enum {
+  SRV6_SAMPLE_LOCALSID_NEXT_ERROR,
+  SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP,
+  SRV6_SAMPLE_LOCALSID_N_NEXT,
+} srv6_localsid_sample_next_t;
+
+/**
+ * @brief Function doing End processing. 
+ */
+//Fixme: support OAM (hop-by-hop header) here!
+static_always_inline void
+end_srh_processing (vlib_node_runtime_t * node,
+                    vlib_buffer_t * b0,
+                    ip6_header_t * ip0,
+                    ip6_sr_header_t * sr0,
+                    u32 * next0)
+{
+  ip6_address_t *new_dst0;
+
+  if(PREDICT_TRUE(ip0->protocol == IP_PROTOCOL_IPV6_ROUTE))
+  {
+    if(PREDICT_TRUE(sr0->type == ROUTING_HEADER_TYPE_SR))
+    {
+      if(PREDICT_TRUE(sr0->segments_left != 0))
+      {
+        sr0->segments_left -= 1;
+        new_dst0 = (ip6_address_t *)(sr0->segments);
+        new_dst0 += sr0->segments_left;
+        ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
+        ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
+      }
+      else
+      {
+        *next0 = SRV6_SAMPLE_LOCALSID_NEXT_ERROR;
+        b0->error = node->errors[SRV6_LOCALSID_COUNTER_NO_SRH];
+      }
+    }
+    else
+    {
+      /* Error. Routing header of type != SR */
+      *next0 = SRV6_SAMPLE_LOCALSID_NEXT_ERROR;
+      b0->error = node->errors[SRV6_LOCALSID_COUNTER_NO_SRH];
+    }
+  }
+}
+
+/*
+ * @brief SRv6 Sample Localsid graph node
+ * WARNING: YOU MUST DO THE DUAL LOOP
+ */
+static uword
+srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  u32 next_index;
+  u32 pkts_swapped = 0;
+  
+  ip6_sr_main_t * sm = &sr_main;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+  u32 cpu_index = os_get_cpu_number ();
+
+  while (n_left_from > 0)
+  {
+    u32 n_left_to_next;
+
+    vlib_get_next_frame (vm, node, next_index,
+		   to_next, n_left_to_next);
+
+    while (n_left_from > 0 && n_left_to_next > 0)
+    {
+      u32 bi0;
+      vlib_buffer_t * b0;
+      ip6_header_t * ip0 = 0;
+      ip6_sr_header_t * sr0;
+      u32 next0 = SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP;
+      ip6_sr_localsid_t *ls0;
+      srv6_localsid_sample_per_sid_memory_t *ls0_mem;
+
+      bi0 = from[0];
+      to_next[0] = bi0;
+      from += 1;
+      to_next += 1;
+      n_left_from -= 1;
+      n_left_to_next -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+      ip0 = vlib_buffer_get_current (b0);
+      sr0 = (ip6_sr_header_t *)(ip0+1);
+
+      /* Lookup the SR End behavior based on IP DA (adj) */
+      ls0 = pool_elt_at_index (sm->localsids, vnet_buffer(b0)->ip.adj_index[VLIB_TX]);
+      ls0_mem = ls0->plugin_mem;
+
+      /* SRH processing */
+      end_srh_processing (node, b0, ip0, sr0, &next0);
+
+      /* ==================================================================== */
+      /* INSERT CODE HERE */
+      /* Example starts here */
+      //In this example we are changing the next VRF table by the one in CLI
+      vnet_buffer(b0)->sw_if_index[VLIB_TX] = ls0_mem->fib_table;
+      /* Example finishes here */
+      /* ==================================================================== */
+
+      if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) 
+      {
+        srv6_localsid_sample_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+        tr->localsid_index = ls0 - sm->localsids;
+      }
+
+      /* This increments the SRv6 per LocalSID counters.*/
+      vlib_increment_combined_counter
+        (((next0 == SRV6_SAMPLE_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : &(sm->sr_ls_valid_counters)),
+        cpu_index,
+        ls0 - sm->localsids,
+        1, vlib_buffer_length_in_chain (vm, b0));
+
+      vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, 
+        n_left_to_next, bi0, next0);
+
+      pkts_swapped ++;
+    }
+    vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+
+  }
+
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (srv6_localsid_sample_node) = {
+  .function = srv6_localsid_sample_fn,
+  .name = "srv6-localsid-sample",
+  .vector_size = sizeof (u32),
+  .format_trace = format_srv6_localsid_sample_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = SRV6_LOCALSID_N_COUNTERS,
+  .error_strings = srv6_localsid_counter_strings,
+  .n_next_nodes = SRV6_SAMPLE_LOCALSID_N_NEXT,
+  .next_nodes = {
+        [SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP] = "ip6-lookup",
+        [SRV6_SAMPLE_LOCALSID_NEXT_ERROR] = "error-drop",
+    },
+};
diff --git a/src/examples/srv6-sample-localsid/srv6_localsid_sample.c b/src/examples/srv6-sample-localsid/srv6_localsid_sample.c
new file mode 100755
index 00000000..ec16547e
--- /dev/null
+++ b/src/examples/srv6-sample-localsid/srv6_localsid_sample.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ *------------------------------------------------------------------
+ * srv6_localsid_sample.c - Simple SRv6 LocalSID
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+unsigned char srv6_localsid_name[32] = "Sample-SRv6-LocalSID-plugin";
+unsigned char keyword_str[32] = "new_srv6_localsid";
+unsigned char def_str[64] = "This is a definition of a sample new_srv6_localsid";
+unsigned char params_str[32] = "";
+
+/*****************************************/
+/* SRv6 LocalSID instantiation and removal functions */
+static int
+srv6_localsid_creation_fn (ip6_sr_localsid_t *localsid)
+{
+  /* 
+   * Do you want to do anything fancy upon localsid instantiation?
+   * You can do it here
+   * (If return != 0 the localsid creation will be cancelled.)
+   */
+  /* As an example Im going to do a +1 to the fib table inserted by the user */
+  srv6_localsid_sample_per_sid_memory_t *ls_mem = localsid->plugin_mem;
+  ls_mem->fib_table += 1;
+  return 0;
+}
+
+static int
+srv6_localsid_removal_fn (ip6_sr_localsid_t *localsid)
+{
+  /* Do you want to do anything fancy upon localsid removal?
+   * You can do it here
+   * (If return != 0 the localsid removal will be cancelled.)
+   */
+  /* 
+   * BTW if you stored something in localsid->plugin_mem you should clean it now
+   */
+
+  //In this example we are only cleaning the memory allocated per localsid
+  clib_mem_free(localsid->plugin_mem);
+  return 0;
+}
+
+/**********************************/
+/* SRv6 LocalSID format functions */
+/*
+ * Prints nicely the parameters of a localsid
+ * Example: print "Table 5"
+ */
+u8 *
+format_srv6_localsid_sample (u8 * s, va_list * args)
+{
+  srv6_localsid_sample_per_sid_memory_t *ls_mem = va_arg (*args, void *);
+  return (format (s, "Table: %u", ls_mem->fib_table));
+}
+
+/*
+ * Process the parameters of a localsid
+ * Example: process from:
+ * sr localsid address cafe::1 behavior new_srv6_localsid 5
+ * everything from behavior on... so in this case 'new_srv6_localsid 5'
+ * Notice that it MUST match the keyword_str and params_str defined above.
+ */
+uword
+unformat_srv6_localsid_sample (unformat_input_t * input, va_list * args)
+{
+  void **plugin_mem = va_arg (*args, void **);
+  srv6_localsid_sample_per_sid_memory_t *ls_mem;
+  u32 table_id;
+  if (unformat (input, "new_srv6_localsid %u", &table_id))
+    {
+      /* Allocate a portion of memory */
+      ls_mem = clib_mem_alloc_aligned_at_offset (
+        sizeof(srv6_localsid_sample_per_sid_memory_t), 0, 0, 1);
+
+      /* Set to zero the memory */
+      memset (ls_mem, 0, sizeof(srv6_localsid_sample_per_sid_memory_t));
+
+      /* Our brand-new car is ready */
+      ls_mem->fib_table = table_id;
+
+      /* Dont forget to add it to the localsid */
+      *plugin_mem = ls_mem;
+      return 1;
+    }
+  return 0;
+}
+
+/*************************/
+/* SRv6 LocalSID FIB DPO */
+static u8 *
+format_srv6_localsid_sample_dpo (u8 * s, va_list * args)
+{
+  index_t index = va_arg (*args, index_t);
+  CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
+
+  return (format (s, "SR: localsid_sample_index:[%u]", index));
+}
+
+void
+srv6_localsid_sample_dpo_lock (dpo_id_t * dpo)
+{
+}
+
+void
+srv6_localsid_sample_dpo_unlock (dpo_id_t * dpo)
+{
+}
+
+const static dpo_vft_t srv6_localsid_sample_vft = {
+  .dv_lock = srv6_localsid_sample_dpo_lock,
+  .dv_unlock = srv6_localsid_sample_dpo_unlock,
+  .dv_format = format_srv6_localsid_sample_dpo,
+};
+
+const static char *const srv6_localsid_sample_ip6_nodes[] = {
+  "srv6-localsid-sample",
+  NULL,
+};
+
+const static char *const *const srv6_localsid_sample_nodes[DPO_PROTO_NUM] = {
+  [DPO_PROTO_IP6] = srv6_localsid_sample_ip6_nodes,
+};
+
+/**********************/
+static clib_error_t * srv6_localsid_sample_init (vlib_main_t * vm)
+{
+  srv6_localsid_sample_main_t * sm = &srv6_localsid_sample_main;
+  int rv = 0;
+  /* Create DPO */
+  sm->srv6_localsid_sample_dpo_type = dpo_register_new_type (
+    &srv6_localsid_sample_vft, srv6_localsid_sample_nodes);
+
+  /* Register SRv6 LocalSID */
+  rv = sr_localsid_register_function (vm, 
+                                  srv6_localsid_name,
+                                  keyword_str,
+                                  def_str,
+                                  params_str,
+                                  &sm->srv6_localsid_sample_dpo_type,
+                                  format_srv6_localsid_sample, 
+                                  unformat_srv6_localsid_sample, 
+                                  srv6_localsid_creation_fn, 
+                                  srv6_localsid_removal_fn);
+  if (rv < 0)
+    clib_error_return (0, "SRv6 LocalSID function could not be registered.");
+  else
+    sm->srv6_localsid_behavior_id = rv;
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (srv6_localsid_sample_init);
+
+VLIB_PLUGIN_REGISTER () = {
+  .version = "1.0",
+};
diff --git a/src/examples/srv6-sample-localsid/srv6_localsid_sample.h b/src/examples/srv6-sample-localsid/srv6_localsid_sample.h
new file mode 100644
index 00000000..474b5de2
--- /dev/null
+++ b/src/examples/srv6-sample-localsid/srv6_localsid_sample.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_srv6_localsid_sample_h__
+#define __included_srv6_localsid_sample_h__
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+typedef struct {
+    /* API message ID base */
+    u16 msg_id_base;
+
+    /* convenience */
+    vlib_main_t * vlib_main;
+    vnet_main_t * vnet_main;
+    
+    /* DPO type */
+    dpo_type_t srv6_localsid_sample_dpo_type;
+
+    /* SRv6 LocalSID behavior number */
+    u32 srv6_localsid_behavior_id;
+
+} srv6_localsid_sample_main_t;
+
+/*
+ * This is the memory that will be stored per each localsid
+ * the user instantiates
+ */
+typedef struct {
+	u32 fib_table;	/* Stupid index used as an example.. */
+} srv6_localsid_sample_per_sid_memory_t ;
+
+srv6_localsid_sample_main_t srv6_localsid_sample_main;
+
+format_function_t format_srv6_localsid_sample;
+unformat_function_t unformat_srv6_localsid_sample;
+
+void srv6_localsid_sample_dpo_lock (dpo_id_t * dpo);
+void srv6_localsid_sample_dpo_unlock (dpo_id_t * dpo);
+
+extern vlib_node_registration_t srv6_localsid_sample_node;
+
+#endif /* __included_sample_h__ */
diff --git a/src/examples/srv6-sample-localsid/srv6_sample_localsid_doc.md b/src/examples/srv6-sample-localsid/srv6_sample_localsid_doc.md
new file mode 100644
index 00000000..78e91ab3
--- /dev/null
+++ b/src/examples/srv6-sample-localsid/srv6_sample_localsid_doc.md
@@ -0,0 +1,30 @@
+# Sample SRv6 LocalSID documentation    {#srv6_plugin_doc}
+
+## Introduction
+
+This plugin is an example of how an user can create a new SRv6 LocalSID behavior by using VPP plugins with the appropiate API calls to the existing SR code.
+
+This **example** plugin registers a new localsid behavior, with cli keyword 'new_srv6_localsid' which only takes one parameter, a fib-table. Upon recival of a packet, this plugin will enforce the next IP6 lookup in the specific fib-table specified by the user. (Indeed it will do the lookup in the fib_table n+1 (since for the shake of the example we increment the fib-table.)
+
+Notice that the plugin only 'defines' a new SRv6 LocalSID behavior, but the existing SR code in VNET is the one actually instantiating new LocalSIDs. Notice that there are callback functions such that when you create or remove a LocalSID you can actually setup specific parameters through the functions in this plugin.
+
+## Variables to watch for
+
+* srv6_localsid_name: This variable is the name (used as a unique key) identifying this SR LocalSID plugin.
+* keyword_str: This is the CLI keyword to be used for the plugin. In this example 'new_srv6_localsid'. (i.e. sr localsid address cafe::1 behavior new_srv6_localsid )
+* def_str: This is a definition of this SR behavior. This is printed when you do 'show sr localsid behaviors'.
+* params_str: This is a definition of the parameters of this localsid. This is printed when you do 'show sr localsid behaviors'.
+
+## Functions to watch for
+
+* srv6_localsid_creation_fn: This function will be called every time a new SR LocalSID is instantiated with the behavior defined in this plugin. 
+* srv6_localsid_removal_fn: This function will be called every time a new SR LocalSID is removed with the behavior defined in this plugin. This function tends to be used for freeing up all the memory created in the previous function.
+* format_srv6_localsid_sample: This function prints nicely the parameters of every SR LocalSID using this behavior.
+* unformat_srv6_localsid_sample: This function parses the CLI command when initialising a new SR LocalSID using this behavior. It parses all the parameters and ensures that the parameters are correct.
+* format_srv6_localsid_sample_dpo: This function formats the 'show ip6 fib' message for the SR LocalSIDs created with this plugin behavior.
+
+## Graph node
+
+The current graph node uses the function 'end_srh_processing' to do the Segment Routing Endpoint behavior. Notice that it does not allow the cleanup of a Segment Routing header (as per the SRv6 behavior specs).
+This function is identical to the one found in /src/vnet/sr/sr_localsid.c
+In case that by some other reason you want to do decapsulation, or SRH clean_up you can use the functions 'end_decaps_srh_processing' or 'end_psp_srh_processing' respectively.
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 6713a9e2..623892e7 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -69,10 +69,6 @@ if ENABLE_SNAT_PLUGIN
 include snat.am
 endif
 
-if ENABLE_SRV6SAMPLE_PLUGIN
-include sample_srv6_localsid.am
-endif
-
 include ../suffix-rules.mk
 
 # Remove *.la files
diff --git a/src/plugins/sample_srv6_localsid.am b/src/plugins/sample_srv6_localsid.am
deleted file mode 100644
index a820ab25..00000000
--- a/src/plugins/sample_srv6_localsid.am
+++ /dev/null
@@ -1,23 +0,0 @@
-
-# Copyright (c)  
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-vppplugins_LTLIBRARIES += sample_srv6_localsid_plugin.la
-
-sample_srv6_localsid_plugin_la_SOURCES =	\
-	srv6-localsid/node.c					\
-	srv6-localsid/srv6_localsid_sample.c
-
-noinst_HEADERS +=	srv6-localsid/srv6_localsid_sample.h
-
-# vi:syntax=automake
diff --git a/src/plugins/srv6-localsid/node.c b/src/plugins/srv6-localsid/node.c
deleted file mode 100644
index 7bae9cd7..00000000
--- a/src/plugins/srv6-localsid/node.c
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-typedef struct {
-  u32 localsid_index;
-} srv6_localsid_sample_trace_t;
-
-/* packet trace format function */
-static u8 * format_srv6_localsid_sample_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  srv6_localsid_sample_trace_t * t = va_arg (*args, srv6_localsid_sample_trace_t *);
-  s = format (s, "SRv6-sample-localsid: localsid_index %d\n",
-              t->localsid_index);
-  return s;
-}
-
-vlib_node_registration_t srv6_localsid_sample_node;
-
-#define foreach_srv6_localsid_counter \
-_(PROCESSED, "srv6-sample-localsid processed packets") \
-_(NO_SRH, "(Error) No SRH.")
-
-typedef enum {
-#define _(sym,str) SRV6_LOCALSID_COUNTER_##sym,
-  foreach_srv6_localsid_counter
-#undef _
-  SRV6_LOCALSID_N_COUNTERS,
-} srv6_localsid_sample_counters;
-
-static char * srv6_localsid_counter_strings[] = {
-#define _(sym,string) string,
-  foreach_srv6_localsid_counter
-#undef _
-};
-
-typedef enum {
-  SRV6_SAMPLE_LOCALSID_NEXT_ERROR,
-  SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP,
-  SRV6_SAMPLE_LOCALSID_N_NEXT,
-} srv6_localsid_sample_next_t;
-
-/**
- * @brief Function doing End processing. 
- */
-//Fixme: support OAM (hop-by-hop header) here!
-static_always_inline void
-end_srh_processing (vlib_node_runtime_t * node,
-                    vlib_buffer_t * b0,
-                    ip6_header_t * ip0,
-                    ip6_sr_header_t * sr0,
-                    u32 * next0)
-{
-  ip6_address_t *new_dst0;
-
-  if(PREDICT_TRUE(ip0->protocol == IP_PROTOCOL_IPV6_ROUTE))
-  {
-    if(PREDICT_TRUE(sr0->type == ROUTING_HEADER_TYPE_SR))
-    {
-      if(PREDICT_TRUE(sr0->segments_left != 0))
-      {
-        sr0->segments_left -= 1;
-        new_dst0 = (ip6_address_t *)(sr0->segments);
-        new_dst0 += sr0->segments_left;
-        ip0->dst_address.as_u64[0] = new_dst0->as_u64[0];
-        ip0->dst_address.as_u64[1] = new_dst0->as_u64[1];
-      }
-      else
-      {
-        *next0 = SRV6_SAMPLE_LOCALSID_NEXT_ERROR;
-        b0->error = node->errors[SRV6_LOCALSID_COUNTER_NO_SRH];
-      }
-    }
-    else
-    {
-      /* Error. Routing header of type != SR */
-      *next0 = SRV6_SAMPLE_LOCALSID_NEXT_ERROR;
-      b0->error = node->errors[SRV6_LOCALSID_COUNTER_NO_SRH];
-    }
-  }
-}
-
-/*
- * @brief SRv6 Sample Localsid graph node
- * WARNING: YOU MUST DO THE DUAL LOOP
- */
-static uword
-srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  u32 next_index;
-  u32 pkts_swapped = 0;
-  
-  ip6_sr_main_t * sm = &sr_main;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-  u32 cpu_index = os_get_cpu_number ();
-
-  while (n_left_from > 0)
-  {
-    u32 n_left_to_next;
-
-    vlib_get_next_frame (vm, node, next_index,
-		   to_next, n_left_to_next);
-
-    while (n_left_from > 0 && n_left_to_next > 0)
-    {
-      u32 bi0;
-      vlib_buffer_t * b0;
-      ip6_header_t * ip0 = 0;
-      ip6_sr_header_t * sr0;
-      u32 next0 = SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP;
-      ip6_sr_localsid_t *ls0;
-      srv6_localsid_sample_per_sid_memory_t *ls0_mem;
-
-      bi0 = from[0];
-      to_next[0] = bi0;
-      from += 1;
-      to_next += 1;
-      n_left_from -= 1;
-      n_left_to_next -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-      ip0 = vlib_buffer_get_current (b0);
-      sr0 = (ip6_sr_header_t *)(ip0+1);
-
-      /* Lookup the SR End behavior based on IP DA (adj) */
-      ls0 = pool_elt_at_index (sm->localsids, vnet_buffer(b0)->ip.adj_index[VLIB_TX]);
-      ls0_mem = ls0->plugin_mem;
-
-      /* SRH processing */
-      end_srh_processing (node, b0, ip0, sr0, &next0);
-
-      /* ==================================================================== */
-      /* INSERT CODE HERE */
-      /* Example starts here */
-      //In this example we are changing the next VRF table by the one in CLI
-      vnet_buffer(b0)->sw_if_index[VLIB_TX] = ls0_mem->fib_table;
-      /* Example finishes here */
-      /* ==================================================================== */
-
-      if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) 
-      {
-        srv6_localsid_sample_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
-        tr->localsid_index = ls0 - sm->localsids;
-      }
-
-      /* This increments the SRv6 per LocalSID counters.*/
-      vlib_increment_combined_counter
-        (((next0 == SRV6_SAMPLE_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : &(sm->sr_ls_valid_counters)),
-        cpu_index,
-        ls0 - sm->localsids,
-        1, vlib_buffer_length_in_chain (vm, b0));
-
-      vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, 
-        n_left_to_next, bi0, next0);
-
-      pkts_swapped ++;
-    }
-    vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-
-  }
-
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (srv6_localsid_sample_node) = {
-  .function = srv6_localsid_sample_fn,
-  .name = "srv6-localsid-sample",
-  .vector_size = sizeof (u32),
-  .format_trace = format_srv6_localsid_sample_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .n_errors = SRV6_LOCALSID_N_COUNTERS,
-  .error_strings = srv6_localsid_counter_strings,
-  .n_next_nodes = SRV6_SAMPLE_LOCALSID_N_NEXT,
-  .next_nodes = {
-        [SRV6_SAMPLE_LOCALSID_NEXT_IP6LOOKUP] = "ip6-lookup",
-        [SRV6_SAMPLE_LOCALSID_NEXT_ERROR] = "error-drop",
-    },
-};
diff --git a/src/plugins/srv6-localsid/srv6_localsid_sample.c b/src/plugins/srv6-localsid/srv6_localsid_sample.c
deleted file mode 100755
index ec16547e..00000000
--- a/src/plugins/srv6-localsid/srv6_localsid_sample.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- *------------------------------------------------------------------
- * srv6_localsid_sample.c - Simple SRv6 LocalSID
- *------------------------------------------------------------------
- */
-
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-unsigned char srv6_localsid_name[32] = "Sample-SRv6-LocalSID-plugin";
-unsigned char keyword_str[32] = "new_srv6_localsid";
-unsigned char def_str[64] = "This is a definition of a sample new_srv6_localsid";
-unsigned char params_str[32] = "";
-
-/*****************************************/
-/* SRv6 LocalSID instantiation and removal functions */
-static int
-srv6_localsid_creation_fn (ip6_sr_localsid_t *localsid)
-{
-  /* 
-   * Do you want to do anything fancy upon localsid instantiation?
-   * You can do it here
-   * (If return != 0 the localsid creation will be cancelled.)
-   */
-  /* As an example Im going to do a +1 to the fib table inserted by the user */
-  srv6_localsid_sample_per_sid_memory_t *ls_mem = localsid->plugin_mem;
-  ls_mem->fib_table += 1;
-  return 0;
-}
-
-static int
-srv6_localsid_removal_fn (ip6_sr_localsid_t *localsid)
-{
-  /* Do you want to do anything fancy upon localsid removal?
-   * You can do it here
-   * (If return != 0 the localsid removal will be cancelled.)
-   */
-  /* 
-   * BTW if you stored something in localsid->plugin_mem you should clean it now
-   */
-
-  //In this example we are only cleaning the memory allocated per localsid
-  clib_mem_free(localsid->plugin_mem);
-  return 0;
-}
-
-/**********************************/
-/* SRv6 LocalSID format functions */
-/*
- * Prints nicely the parameters of a localsid
- * Example: print "Table 5"
- */
-u8 *
-format_srv6_localsid_sample (u8 * s, va_list * args)
-{
-  srv6_localsid_sample_per_sid_memory_t *ls_mem = va_arg (*args, void *);
-  return (format (s, "Table: %u", ls_mem->fib_table));
-}
-
-/*
- * Process the parameters of a localsid
- * Example: process from:
- * sr localsid address cafe::1 behavior new_srv6_localsid 5
- * everything from behavior on... so in this case 'new_srv6_localsid 5'
- * Notice that it MUST match the keyword_str and params_str defined above.
- */
-uword
-unformat_srv6_localsid_sample (unformat_input_t * input, va_list * args)
-{
-  void **plugin_mem = va_arg (*args, void **);
-  srv6_localsid_sample_per_sid_memory_t *ls_mem;
-  u32 table_id;
-  if (unformat (input, "new_srv6_localsid %u", &table_id))
-    {
-      /* Allocate a portion of memory */
-      ls_mem = clib_mem_alloc_aligned_at_offset (
-        sizeof(srv6_localsid_sample_per_sid_memory_t), 0, 0, 1);
-
-      /* Set to zero the memory */
-      memset (ls_mem, 0, sizeof(srv6_localsid_sample_per_sid_memory_t));
-
-      /* Our brand-new car is ready */
-      ls_mem->fib_table = table_id;
-
-      /* Dont forget to add it to the localsid */
-      *plugin_mem = ls_mem;
-      return 1;
-    }
-  return 0;
-}
-
-/*************************/
-/* SRv6 LocalSID FIB DPO */
-static u8 *
-format_srv6_localsid_sample_dpo (u8 * s, va_list * args)
-{
-  index_t index = va_arg (*args, index_t);
-  CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
-
-  return (format (s, "SR: localsid_sample_index:[%u]", index));
-}
-
-void
-srv6_localsid_sample_dpo_lock (dpo_id_t * dpo)
-{
-}
-
-void
-srv6_localsid_sample_dpo_unlock (dpo_id_t * dpo)
-{
-}
-
-const static dpo_vft_t srv6_localsid_sample_vft = {
-  .dv_lock = srv6_localsid_sample_dpo_lock,
-  .dv_unlock = srv6_localsid_sample_dpo_unlock,
-  .dv_format = format_srv6_localsid_sample_dpo,
-};
-
-const static char *const srv6_localsid_sample_ip6_nodes[] = {
-  "srv6-localsid-sample",
-  NULL,
-};
-
-const static char *const *const srv6_localsid_sample_nodes[DPO_PROTO_NUM] = {
-  [DPO_PROTO_IP6] = srv6_localsid_sample_ip6_nodes,
-};
-
-/**********************/
-static clib_error_t * srv6_localsid_sample_init (vlib_main_t * vm)
-{
-  srv6_localsid_sample_main_t * sm = &srv6_localsid_sample_main;
-  int rv = 0;
-  /* Create DPO */
-  sm->srv6_localsid_sample_dpo_type = dpo_register_new_type (
-    &srv6_localsid_sample_vft, srv6_localsid_sample_nodes);
-
-  /* Register SRv6 LocalSID */
-  rv = sr_localsid_register_function (vm, 
-                                  srv6_localsid_name,
-                                  keyword_str,
-                                  def_str,
-                                  params_str,
-                                  &sm->srv6_localsid_sample_dpo_type,
-                                  format_srv6_localsid_sample, 
-                                  unformat_srv6_localsid_sample, 
-                                  srv6_localsid_creation_fn, 
-                                  srv6_localsid_removal_fn);
-  if (rv < 0)
-    clib_error_return (0, "SRv6 LocalSID function could not be registered.");
-  else
-    sm->srv6_localsid_behavior_id = rv;
-
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (srv6_localsid_sample_init);
-
-VLIB_PLUGIN_REGISTER () = {
-  .version = "1.0",
-};
diff --git a/src/plugins/srv6-localsid/srv6_localsid_sample.h b/src/plugins/srv6-localsid/srv6_localsid_sample.h
deleted file mode 100644
index 474b5de2..00000000
--- a/src/plugins/srv6-localsid/srv6_localsid_sample.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_srv6_localsid_sample_h__
-#define __included_srv6_localsid_sample_h__
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-typedef struct {
-    /* API message ID base */
-    u16 msg_id_base;
-
-    /* convenience */
-    vlib_main_t * vlib_main;
-    vnet_main_t * vnet_main;
-    
-    /* DPO type */
-    dpo_type_t srv6_localsid_sample_dpo_type;
-
-    /* SRv6 LocalSID behavior number */
-    u32 srv6_localsid_behavior_id;
-
-} srv6_localsid_sample_main_t;
-
-/*
- * This is the memory that will be stored per each localsid
- * the user instantiates
- */
-typedef struct {
-	u32 fib_table;	/* Stupid index used as an example.. */
-} srv6_localsid_sample_per_sid_memory_t ;
-
-srv6_localsid_sample_main_t srv6_localsid_sample_main;
-
-format_function_t format_srv6_localsid_sample;
-unformat_function_t unformat_srv6_localsid_sample;
-
-void srv6_localsid_sample_dpo_lock (dpo_id_t * dpo);
-void srv6_localsid_sample_dpo_unlock (dpo_id_t * dpo);
-
-extern vlib_node_registration_t srv6_localsid_sample_node;
-
-#endif /* __included_sample_h__ */
diff --git a/src/plugins/srv6-localsid/srv6_sample_localsid_doc.md b/src/plugins/srv6-localsid/srv6_sample_localsid_doc.md
deleted file mode 100644
index b60ab5da..00000000
--- a/src/plugins/srv6-localsid/srv6_sample_localsid_doc.md
+++ /dev/null
@@ -1,38 +0,0 @@
-# SRv6 Sample LocalSID documentation    {#srv6_plugin_doc}
-
-## Disclaimer
-
-This is a memo intended to contain documentation for the sample SRv6 LocalSID behavior plugin
-Everything that is not directly obvious should come here.
-For any feedback on content that should be explained please mailto:pcamaril@cisco.com
- 
-This plugin refers to Segment Routing. Please read the SR documentation first.
-
-## Introduction
-
-This plugin is an example of how an user can create a new SRv6 LocalSID behavior by using VPP plugins with the appropiate API calls to the existing SR code.
-
-This **example** plugin registers a new localsid behavior, with cli keyword 'new_srv6_localsid' which only takes one parameter, a fib-table. Upon recival of a packet, this plugin will enforce the next IP6 lookup in the specific fib-table specified by the user. (Indeed it will do the lookup in the fib_table n+1 (since for the shake of the example we increment the fib-table.)
-
-Notice that the plugin only 'defines' a new SRv6 LocalSID behavior, but the existing SR code in VNET is the one actually instantiating new LocalSIDs. Notice that there are callback functions such that when you create or remove a LocalSID you can actually setup specific parameters through the functions in this plugin.
-
-## Variables to watch for
-
-* srv6_localsid_name: This variable is the name (used as a unique key) identifying this SR LocalSID plugin.
-* keyword_str: This is the CLI keyword to be used for the plugin. In this example 'new_srv6_localsid'. (i.e. sr localsid address cafe::1 behavior new_srv6_localsid )
-* def_str: This is a definition of this SR behavior. This is printed when you do 'show sr localsid behaviors'.
-* params_str: This is a definition of the parameters of this localsid. This is printed when you do 'show sr localsid behaviors'.
-
-## Functions to watch for
-
-* srv6_localsid_creation_fn: This function will be called every time a new SR LocalSID is instantiated with the behavior defined in this plugin. 
-* srv6_localsid_removal_fn: This function will be called every time a new SR LocalSID is removed with the behavior defined in this plugin. This function tends to be used for freeing up all the memory created in the previous function.
-* format_srv6_localsid_sample: This function prints nicely the parameters of every SR LocalSID using this behavior.
-* unformat_srv6_localsid_sample: This function parses the CLI command when initialising a new SR LocalSID using this behavior. It parses all the parameters and ensures that the parameters are correct.
-* format_srv6_localsid_sample_dpo: This function formats the 'show ip6 fib' message for the SR LocalSIDs created with this plugin behavior.
-
-## Graph node
-
-The current graph node uses the function 'end_srh_processing' to do the Segment Routing Endpoint behavior. Notice that it does not allow the cleanup of a Segment Routing header (as per the SRv6 behavior specs).
-This function is identical to the one found in /src/vnet/sr/sr_localsid.c
-In case that by some other reason you want to do decapsulation, or SRH clean_up you can use the functions 'end_decaps_srh_processing' or 'end_psp_srh_processing' respectively.
diff --git a/src/vnet/sr/sr.h b/src/vnet/sr/sr.h
index 5efeb4dd..b832c0fc 100755
--- a/src/vnet/sr/sr.h
+++ b/src/vnet/sr/sr.h
@@ -173,6 +173,7 @@ typedef struct
     } l2;
   };
   u8 traffic_type;					/**< Traffic type (IPv4, IPv6, L2) */
+  u8 padding[3];
 } sr_steering_key_t;
 
 typedef struct
@@ -186,46 +187,29 @@ typedef struct
  */
 typedef struct
 {
-  /* ip6-lookup next index for imposition FIB entries */
-  u32 ip6_lookup_sr_next_index;
-
-  /* ip6-replicate next index for multicast tunnel */
-  u32 ip6_lookup_sr_spray_index;
-
-  /* IP4-lookup -> SR rewrite next index */
-  u32 ip4_lookup_sr_policy_rewrite_encaps_index;
-  u32 ip4_lookup_sr_policy_rewrite_insert_index;
-
-  /* IP6-lookup -> SR rewrite next index */
-  u32 ip6_lookup_sr_policy_rewrite_encaps_index;
-  u32 ip6_lookup_sr_policy_rewrite_insert_index;
-
   /* L2-input -> SR rewrite next index */
   u32 l2_sr_policy_rewrite_index;
 
-  /* IP6-lookup -> SR LocalSID (SR End processing) index */
-  u32 ip6_lookup_sr_localsid_index;
-
   /* SR SID lists */
   ip6_sr_sl_t *sid_lists;
 
   /* SR policies */
   ip6_sr_policy_t *sr_policies;
 
-  /* Find an SR policy by its BindingSID */
-  ip6_address_t *sr_policy_index_by_key;
+  /* Hash table mapping BindingSID to SR policy */
+  mhash_t sr_policies_index_hash;
 
   /* Pool of SR localsid instances */
   ip6_sr_localsid_t *localsids;
 
-  /* Find a SR localsid instance based on its functionID */
-  ip6_address_t *localsids_index_by_key;
+  /* Hash table mapping LOC:FUNC to SR LocalSID instance */
+  mhash_t sr_localsids_index_hash;
 
   /* Pool of SR steer policies instances */
   ip6_sr_steering_policy_t *steer_policies;
 
-  /* Find a steer policy based on its classifier */
-  sr_steering_key_t *steer_policies_index_by_key;
+  /* Hash table mapping steering rules to SR steer instance */
+  mhash_t sr_steer_policies_hash;
 
   /* L2 steering ifaces - sr_policies */
   u32 *sw_iface_sr_policies;
diff --git a/src/vnet/sr/sr_doc.md b/src/vnet/sr/sr_doc.md
index a7220630..fd92bdf2 100644
--- a/src/vnet/sr/sr_doc.md
+++ b/src/vnet/sr/sr_doc.md
@@ -1,6 +1,6 @@
-# SRv6: Segment Routing for IPv6    {#sr_doc}
+# SRv6: Segment Routing for IPv6    {#srv6_doc}
 
-This is a memo intended to contain documentation of the VPP SRv6 implementation
+This is a memo intended to contain documentation of the VPP SRv6 implementation.
 Everything that is not directly obvious should come here.
 For any feedback on content that should be explained please mailto:pcamaril@cisco.com
 
@@ -27,135 +27,29 @@ Segment routing can operate with either an MPLS or an IPv6 data plane. All the c
 * Local SID: is a SID associated with a processing function on the local node, which may go from advancing to the next SID in the SRH, to complex user-defined behaviors. When a FIB lookup, either in the main FIB or in a specific VRF, returns a match on a local SID, the associated function is performed.
 * BindingSID: a BindingSID is a SID (only one) associated one-one with an SR Policy. If a packet arrives with an IPv6 DA corresponding to a BindingSID, then the SR policy will be applied to such packet.
 
-## Creating an SR LocalSID
+## SRv6 Features in VPP
 
-A local SID is associated to a Segment Routing behavior -or function- on the current node.
+The SRv6 Network Programming (*draft-filsfils-spring-srv6-network-programming*) defines the SRv6 architecture.
 
-The most basic behavior is called END. It simply activates the next SID in the current packet, by decrementing the Segments Left value and updating the IPv6 DA.
+VPP supports the following SRv6 LocalSID functions: End, End.X, End.DX6, End.DT6, End.DX4, End.DT4, End.DX2, End.B6, End.B6.Encaps.
 
-A local END SID is instantiated using the following CLI:
+For further information and how to configure each specific function: @subpage srv6_localsid_doc
 
-    sr localsid (del) address XX::YY behavior end
 
-This creates a new entry in the main FIB for IPv6 address XX::YY. All packets whose IPv6 DA matches this FIB entry are redirected to the sr-localsid node, where they are processed as described above.
+The Segment Routing Policy (*draft-filsfils-spring-segment-routing-policy*) defines SR Policies.
 
-Other examples of local SIDs are the following:
+VPP supports SRv6 Policies with T.Insert and T.Encaps behaviors.
 
-    sr localsid (del) address XX::YY behavior end (psp)
-    sr localsid (del) address XX::YY behavior end.x GE0/1/0 2001::a (psp)
-    sr localsid (del) address XX::YY behavior end.dx6 GE0/1/0 2001::a
-    sr localsid (del) address XX::YY behavior end.dx4 GE0/1/0 10.0.0.1
-    sr localsid (del) address XX::YY behavior end.dx2 GigabitE0/11/0
-    sr localsid (del) address XX::YY behavior end.dt6 5
-    sr localsid (del) address XX::YY behavior end.dt6 5
+For further information on how to create SR Policies: @subpage srv6_policy_doc
 
-Note that all of these behaviors match the specifications in **TODO REF NET PGM**. Please refer to this document for a detailed description of each behavior.
+For further information on how to steer traffic into SR Policies: @subpage srv6_steering_doc
 
-Help on the available local SID behaviors and their usage can be obtained with:
-    
-    help sr localsid
+## SRv6 LocalSID development framework
 
-Alternatively they can be obtained using.
-
-    show sr localsids behavior
-
-The difference in between those two commands is that the first one will only display the SR LocalSID behaviors that are built-in VPP, while the latter will display those behaviors plus the ones added with the SR LocalSID Development Framework.
-
-
-VPP keeps a 'My LocalSID Table' where it stores all the SR local SIDs instantiated as well as their parameters. Every time a new local SID is instantiated, a new entry is added to this table. In addition, counters for correctly and incorrectly processed traffic are maintained for each local SID. The counters store both the number of packets and bytes.
-
-The contents of the 'My LocalSID Table' is shown with:
-
-    vpp# show sr localsid
-    SRv6 - My LocalSID Table:
-    =========================
-            Address:        c3::1
-            Behavior:       DX6 (Endpoint with decapsulation and IPv6 cross-connect)
-            Iface:          GigabitEthernet0/5/0
-            Next hop:       b:c3::b
-            Good traffic:   [51277 packets : 5332808 bytes]
-            Bad traffic:    [0 packets : 0 bytes]
-    --------------------
-
-The traffic counters can be reset with:
-
-    vpp# clear sr localsid counters
-
-## Creating a SR Policy
-
-An SR Policy is defined by a Binding SID and a weighted set of Segment Lists.
-
-A new SR policy is created with a first SID list using:
-
-    sr policy add bsid 2001::1 next A1:: next B1:: next C1:: (weight 5) (fib-table 3)
-
-* The weight parameter is only used if more than one SID list is associated with the policy.
-* The fib-table parameter specifies in which table (VRF) the Binding SID is to be installed.
-
-An SR policy is deleted with:
-
-    sr policy del bsid 2001::1
-    sr policy del index 1
-
-The existing SR policies are listed with:
-
-    show sr policies
-
-### Adding/Removing SID Lists from an SR policy
-
-An additional SID list is associated with an existing SR policy with:
-
-    sr policy mod bsid 2001::1 add sl next A2:: next B2:: next C2:: (weight 3)
-    sr policy mod index 3      add sl next A2:: next B2:: next C2:: (weight 3)
-
-Conversely, a SID list can be removed from an SR policy with:
-
-    sr policy mod bsid 2001::1 del sl index 1
-    sr policy mod index 3      del sl index 1
-
-Note that this cannot be used to remove the last SID list of a policy.
-
-The weight of a SID list can also be modified with:
-
-    sr policy mod bsid 2001::1 mod sl index 1 weight 4
-    sr policy mod index 3      mod sl index 1 weight 4
-
-### SR Policies: Spray policies
-
-Spray policies are a specific type of SR policies where the packet is replicated on all the SID lists, rather than load-balanced among them.
-
-SID list weights are ignored with this type of policies.
-
-A Spray policy is instantiated by appending the keyword **spray** to a regular SR policy command, as in:
-
-    sr policy add bsid 2001::1 next A1:: next B1:: next C1:: spray
-
-Spray policies are used for removing multicast state from a network core domain, and instead send a linear unicast copy to every access node. The last SID in each list accesses the multicast tree within the access node.  
-
-### Encapsulation SR policies
-
-In case the user decides to create an SR policy an IPv6 Source Address must be specified for the encapsulated traffic. In order to do so the user might use the following command:
-    
-    set sr encaps source addr XXXX::YYYY
-
-## Steering packets into a SR Policy 
-
-To steer packets in Transit into an SR policy (T.Insert, T.Encaps and T.Encaps.L2 behaviors), the user needs to create an 'sr steering policy'.
-
-    sr steer l3 2001::/64 via sr policy index 1
-    sr steer l3 2001::/64 via sr policy bsid cafe::1
-    sr steer l3 2001::/64 via sr policy bsid cafe::1 fib-table 3
-    sr steer l3 10.0.0.0/16 via sr policy bsid cafe::1
-    sr steer l2 TenGE0/1/0 via sr policy bsid cafe::1
-
-Disclaimer: The T.Encaps.L2 will steer L2 frames into an SR Policy. Notice that creating an SR steering policy for L2 frames will actually automatically *puts the interface into promiscous mode*.
-
-## SR LocalSID development framework
-
-One of the * 'key' * concepts about SRv6 is regarding network programmability. This is why an SRv6 LocalSID is associated with an specific function. 
+One of the *'key'* concepts about SRv6 is network programmability. This is why an SRv6 LocalSID is associated with an specific function. 
 
 However, the trully way to enable network programmability is allowing any developer **easily** create his own SRv6 LocalSID function. That is the reason why we have added some API calls such that any developer can code his own SRv6 LocalSID behaviors as plugins an add them to the running SRv6 code.
 
 The principle is that the developer only codes the behavior -the graph node-. However all the FIB handling, SR LocalSID instantiation and so on are done by the VPP SRv6 code.
 
-For more information please refer to the documentation *SRv6 Sample SR LocalSID plugin*.
+For more information please refer to: @subpage srv6_plugin_doc
diff --git a/src/vnet/sr/sr_localsid.c b/src/vnet/sr/sr_localsid.c
index 610bb7ff..2e3d56de 100755
--- a/src/vnet/sr/sr_localsid.c
+++ b/src/vnet/sr/sr_localsid.c
@@ -72,18 +72,16 @@ sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
   int rv;
 
   ip6_sr_localsid_t *ls = 0;
-  ip6_address_t *key_copy;
 
   dpo_id_t dpo = DPO_INVALID;
 
   /* Search for the item */
-  p = hash_get_mem (sm->localsids_index_by_key, localsid_addr);
+  p = mhash_get (&sm->sr_localsids_index_hash, localsid_addr);
 
   if (p)
     {
       if (is_del)
 	{
-	  hash_pair_t *hp;
 	  /* Retrieve localsid */
 	  ls = pool_elt_at_index (sm->localsids, p[0]);
 	  /* Delete FIB entry */
@@ -116,10 +114,7 @@ sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
 
 	  /* Delete localsid registry */
 	  pool_put (sm->localsids, ls);
-	  hp = hash_get_pair (sm->localsids_index_by_key, localsid_addr);
-	  key_copy = (void *) (hp->key);
-	  hash_unset_mem (sm->localsids_index_by_key, localsid_addr);
-	  vec_free (key_copy);
+	  mhash_unset (&sm->sr_localsids_index_hash, localsid_addr, NULL);
 	  return 1;
 	}
       else			/* create with function already existing; complain */
@@ -232,9 +227,8 @@ sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
     }
 
   /* Set hash key for searching localsid by address */
-  key_copy = vec_new (ip6_address_t, 1);
-  clib_memcpy (key_copy, localsid_addr, sizeof (ip6_address_t));
-  hash_set_mem (sm->localsids_index_by_key, key_copy, ls - sm->localsids);
+  mhash_set (&sm->sr_localsids_index_hash, localsid_addr, ls - sm->localsids,
+	     NULL);
 
   fib_table_entry_special_dpo_add (fib_index, &pfx, FIB_SOURCE_SR,
 				   FIB_ENTRY_FLAG_EXCLUSIVE, &dpo);
@@ -1476,8 +1470,8 @@ sr_localsids_init (vlib_main_t * vm)
 {
   /* Init memory for function keys */
   ip6_sr_main_t *sm = &sr_main;
-  sm->localsids_index_by_key =
-    hash_create_mem (0, sizeof (ip6_address_t), sizeof (uword));
+  mhash_init (&sm->sr_localsids_index_hash, sizeof (uword),
+	      sizeof (ip6_address_t));
   /* Init SR behaviors DPO type */
   sr_localsid_dpo_type = dpo_register_new_type (&sr_loc_vft, sr_loc_nodes);
   /* Init SR behaviors DPO type */
diff --git a/src/vnet/sr/sr_localsid.md b/src/vnet/sr/sr_localsid.md
new file mode 100644
index 00000000..340af4a3
--- /dev/null
+++ b/src/vnet/sr/sr_localsid.md
@@ -0,0 +1,58 @@
+# SR LocalSIDs    {#srv6_localsid_doc}
+
+A local SID is associated to a Segment Routing behavior -or function- on the current node.
+
+The most basic behavior is called END. It simply activates the next SID in the current packet, by decrementing the Segments Left value and updating the IPv6 DA.
+
+A local END SID is instantiated using the following CLI:
+
+    sr localsid (del) address XX::YY behavior end
+
+This creates a new entry in the main FIB for IPv6 address XX::YY. All packets whose IPv6 DA matches this FIB entry are redirected to the sr-localsid node, where they are processed as described above.
+
+Other examples of local SIDs are the following:
+
+    sr localsid (del) address XX::YY behavior end
+    sr localsid (del) address XX::YY behavior end.x GE0/1/0 2001::a
+    sr localsid (del) address XX::YY behavior end.dx6 GE0/1/0 2001::a
+    sr localsid (del) address XX::YY behavior end.dx4 GE0/1/0 10.0.0.1
+    sr localsid (del) address XX::YY behavior end.dx2 GigabitE0/11/0
+    sr localsid (del) address XX::YY behavior end.dt6 5
+    sr localsid (del) address XX::YY behavior end.dt6 5
+
+Note that all of these behaviors match the definitions of the SRv6 architecture (*draft-filsfils-spring-srv6-network-programming*). Please refer to this document for a detailed description of each behavior.
+
+Note also that you can configure the PSP flavor of the End and End.X behaviors by typing:
+    
+    sr localsid (del) address XX::YY behavior end psp
+    sr localsid (del) address XX::YY behavior end.x GE0/1/0 2001::a psp
+
+Help on the available local SID behaviors and their usage can be obtained with:
+    
+    help sr localsid
+
+Alternatively they can be obtained using.
+
+    show sr localsids behavior
+
+The difference in between those two commands is that the first one will only display the SR LocalSID behaviors that are built-in VPP, while the latter will display those behaviors plus the ones added with the SR LocalSID Development Framework.
+
+
+VPP keeps a 'My LocalSID Table' where it stores all the SR local SIDs instantiated as well as their parameters. Every time a new local SID is instantiated, a new entry is added to this table. In addition, counters for correctly and incorrectly processed traffic are maintained for each local SID. The counters store both the number of packets and bytes.
+
+The contents of the 'My LocalSID Table' is shown with:
+
+    vpp# show sr localsid
+    SRv6 - My LocalSID Table:
+    =========================
+            Address:        c3::1
+            Behavior:       DX6 (Endpoint with decapsulation and IPv6 cross-connect)
+            Iface:          GigabitEthernet0/5/0
+            Next hop:       b:c3::b
+            Good traffic:   [51277 packets : 5332808 bytes]
+            Bad traffic:    [0 packets : 0 bytes]
+    --------------------
+
+The traffic counters can be reset with:
+
+    vpp# clear sr localsid counters
diff --git a/src/vnet/sr/sr_policy.md b/src/vnet/sr/sr_policy.md
new file mode 100644
index 00000000..521b8461
--- /dev/null
+++ b/src/vnet/sr/sr_policy.md
@@ -0,0 +1,56 @@
+# Creating a SR Policy    {#srv6_policy_doc}
+
+An SR Policy is defined by a Binding SID and a weighted set of Segment Lists.
+
+A new SR policy is created with a first SID list using:
+
+    sr policy add bsid 2001::1 next A1:: next B1:: next C1:: (weight 5) (fib-table 3)
+
+* The weight parameter is only used if more than one SID list is associated with the policy.
+* The fib-table parameter specifies in which table (VRF) the Binding SID is to be installed.
+
+An SR policy is deleted with:
+
+    sr policy del bsid 2001::1
+    sr policy del index 1
+
+The existing SR policies are listed with:
+
+    show sr policies
+
+## Adding/Removing SID Lists from an SR policy
+
+An additional SID list is associated with an existing SR policy with:
+
+    sr policy mod bsid 2001::1 add sl next A2:: next B2:: next C2:: (weight 3)
+    sr policy mod index 3      add sl next A2:: next B2:: next C2:: (weight 3)
+
+Conversely, a SID list can be removed from an SR policy with:
+
+    sr policy mod bsid 2001::1 del sl index 1
+    sr policy mod index 3      del sl index 1
+
+Note that this cannot be used to remove the last SID list of a policy.
+
+The weight of a SID list can also be modified with:
+
+    sr policy mod bsid 2001::1 mod sl index 1 weight 4
+    sr policy mod index 3      mod sl index 1 weight 4
+
+## SR Policies: Spray policies
+
+Spray policies are a specific type of SR policies where the packet is replicated on all the SID lists, rather than load-balanced among them.
+
+SID list weights are ignored with this type of policies.
+
+A Spray policy is instantiated by appending the keyword **spray** to a regular SR policy command, as in:
+
+    sr policy add bsid 2001::1 next A1:: next B1:: next C1:: spray
+
+Spray policies are used for removing multicast state from a network core domain, and instead send a linear unicast copy to every access node. The last SID in each list accesses the multicast tree within the access node.  
+
+## Encapsulation SR policies
+
+In case the user decides to create an SR policy an IPv6 Source Address must be specified for the encapsulated traffic. In order to do so the user might use the following command:
+    
+    set sr encaps source addr XXXX::YYYY
diff --git a/src/vnet/sr/sr_policy_rewrite.c b/src/vnet/sr/sr_policy_rewrite.c
index dfdd7950..de6c1616 100755
--- a/src/vnet/sr/sr_policy_rewrite.c
+++ b/src/vnet/sr/sr_policy_rewrite.c
@@ -349,6 +349,7 @@ update_lb (ip6_sr_policy_t * sr_policy)
   ip6_sr_sl_t *segment_list;
   ip6_sr_main_t *sm = &sr_main;
   load_balance_path_t path;
+  path.path_index = FIB_NODE_INDEX_INVALID;
   load_balance_path_t *ip4_path_vector = 0;
   load_balance_path_t *ip6_path_vector = 0;
   load_balance_path_t *b_path_vector = 0;
@@ -447,6 +448,7 @@ update_replicate (ip6_sr_policy_t * sr_policy)
   ip6_sr_sl_t *segment_list;
   ip6_sr_main_t *sm = &sr_main;
   load_balance_path_t path;
+  path.path_index = FIB_NODE_INDEX_INVALID;
   load_balance_path_t *b_path_vector = 0;
   load_balance_path_t *ip6_path_vector = 0;
   load_balance_path_t *ip4_path_vector = 0;
@@ -548,11 +550,10 @@ sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments,
 {
   ip6_sr_main_t *sm = &sr_main;
   ip6_sr_policy_t *sr_policy = 0;
-  ip6_address_t *key_copy;
   uword *p;
 
   /* Search for existing keys (BSID) */
-  p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+  p = mhash_get (&sm->sr_policies_index_hash, bsid);
   if (p)
     {
       /* Add SR policy that already exists; complain */
@@ -592,10 +593,8 @@ sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments,
   sr_policy->is_encap = is_encap;
 
   /* Copy the key */
-  key_copy = vec_new (ip6_address_t, 1);
-  clib_memcpy (key_copy, bsid, sizeof (ip6_address_t));
-  hash_set_mem (sm->sr_policy_index_by_key, key_copy,
-		sr_policy - sm->sr_policies);
+  mhash_set (&sm->sr_policies_index_hash, bsid, sr_policy - sm->sr_policies,
+	     NULL);
 
   /* Create a segment list and add the index to the SR policy */
   create_sl (sr_policy, segments, weight, is_encap);
@@ -635,14 +634,12 @@ sr_policy_del (ip6_address_t * bsid, u32 index)
   ip6_sr_main_t *sm = &sr_main;
   ip6_sr_policy_t *sr_policy = 0;
   ip6_sr_sl_t *segment_list;
-  ip6_address_t *key_copy;
   u32 *sl_index;
   uword *p;
 
-  hash_pair_t *hp;
   if (bsid)
     {
-      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+      p = mhash_get (&sm->sr_policies_index_hash, bsid);
       if (p)
 	sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
       else
@@ -692,14 +689,11 @@ sr_policy_del (ip6_address_t * bsid, u32 index)
   }
 
   /* Remove SR policy entry */
-  hp = hash_get_pair (sm->sr_policy_index_by_key, &sr_policy->bsid);
-  key_copy = (void *) (hp->key);
-  hash_unset_mem (sm->sr_policy_index_by_key, &sr_policy->bsid);
-  vec_free (key_copy);
+  mhash_unset (&sm->sr_policies_index_hash, &sr_policy->bsid, NULL);
   pool_put (sm->sr_policies, sr_policy);
 
   /* If FIB empty unlock it */
-  if (!pool_elts (sm->sr_policies))
+  if (!pool_elts (sm->sr_policies) && !pool_elts (sm->steer_policies))
     {
       fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6);
       fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6);
@@ -741,7 +735,7 @@ sr_policy_mod (ip6_address_t * bsid, u32 index, u32 fib_table,
 
   if (bsid)
     {
-      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+      p = mhash_get (&sm->sr_policies_index_hash, bsid);
       if (p)
 	sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
       else
@@ -3208,8 +3202,8 @@ sr_policy_rewrite_init (vlib_main_t * vm)
   ip6_sr_main_t *sm = &sr_main;
 
   /* Init memory for sr policy keys (bsid <-> ip6_address_t) */
-  sm->sr_policy_index_by_key = hash_create_mem (0, sizeof (ip6_address_t),
-						sizeof (uword));
+  mhash_init (&sm->sr_policies_index_hash, sizeof (uword),
+	      sizeof (ip6_address_t));
 
   /* Init SR VPO DPOs type */
   sr_pr_encaps_dpo_type =
diff --git a/src/vnet/sr/sr_steering.c b/src/vnet/sr/sr_steering.c
index 86d6f27c..be1bb7ad 100755
--- a/src/vnet/sr/sr_steering.c
+++ b/src/vnet/sr/sr_steering.c
@@ -62,14 +62,14 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 		    u32 sw_if_index, u8 traffic_type)
 {
   ip6_sr_main_t *sm = &sr_main;
-  sr_steering_key_t key, *key_copy;
+  sr_steering_key_t key;
   ip6_sr_steering_policy_t *steer_pl;
   fib_prefix_t pfx = { 0 };
 
   ip6_sr_policy_t *sr_policy = 0;
   uword *p = 0;
 
-  hash_pair_t *hp;
+  memset (&key, 0, sizeof (sr_steering_key_t));
 
   /* Compute the steer policy key */
   if (prefix)
@@ -78,6 +78,8 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
       key.l3.prefix.as_u64[1] = prefix->as_u64[1];
       key.l3.mask_width = mask_width;
       key.l3.fib_table = (table_id != (u32) ~ 0 ? table_id : 0);
+      if (traffic_type != SR_STEER_IPV4 && traffic_type != SR_STEER_IPV6)
+	return -1;
     }
   else
     {
@@ -92,12 +94,14 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	vnet_get_sw_interface (sm->vnet_main, sw_if_index);
       if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
 	return -3;
+      if (traffic_type != SR_STEER_L2)
+	return -1;
     }
 
   key.traffic_type = traffic_type;
 
   /* Search for the item */
-  p = hash_get_mem (sm->steer_policies_index_by_key, &key);
+  p = mhash_get (&sm->sr_steer_policies_hash, &key);
 
   if (p)
     {
@@ -152,10 +156,17 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 
 	  /* Delete SR steering policy entry */
 	  pool_put (sm->steer_policies, steer_pl);
-	  hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
-	  key_copy = (void *) (hp->key);
-	  hash_unset_mem (sm->steer_policies_index_by_key, &key);
-	  vec_free (key_copy);
+	  mhash_unset (&sm->sr_steer_policies_hash, &key, NULL);
+
+	  /* If no more SR policies or steering policies */
+	  if (!pool_elts (sm->sr_policies) && !pool_elts (sm->steer_policies))
+	    {
+	      fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6);
+	      fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6);
+	      sm->fib_table_ip6 = (u32) ~ 0;
+	      sm->fib_table_ip4 = (u32) ~ 0;
+	    }
+
 	  return 1;
 	}
       else			/* It means user requested to update an existing SR steering policy */
@@ -163,7 +174,7 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	  /* Retrieve SR steering policy */
 	  if (bsid)
 	    {
-	      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+	      p = mhash_get (&sm->sr_policies_index_hash, bsid);
 	      if (p)
 		sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
 	      else
@@ -192,7 +203,6 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 
 	      /* Create a new one */
 	      goto update_fib;
-
 	    }
 	  else if (steer_pl->classify.traffic_type == SR_STEER_IPV4)
 	    {
@@ -224,7 +234,7 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
   /* Retrieve SR policy */
   if (bsid)
     {
-      p = hash_get_mem (sm->sr_policy_index_by_key, bsid);
+      p = mhash_get (&sm->sr_policies_index_hash, bsid);
       if (p)
 	sr_policy = pool_elt_at_index (sm->sr_policies, p[0]);
       else
@@ -255,19 +265,14 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
     {
       /* Incorrect API usage. Should never get here */
       pool_put (sm->steer_policies, steer_pl);
-      hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
-      key_copy = (void *) (hp->key);
-      hash_unset_mem (sm->steer_policies_index_by_key, &key);
-      vec_free (key_copy);
+      mhash_unset (&sm->sr_steer_policies_hash, &key, NULL);
       return -1;
     }
   steer_pl->sr_policy = sr_policy - sm->sr_policies;
 
   /* Create and store key */
-  key_copy = vec_new (sr_steering_key_t, 1);
-  clib_memcpy (key_copy, &key, sizeof (sr_steering_key_t));
-  hash_set_mem (sm->steer_policies_index_by_key,
-		key_copy, steer_pl - sm->steer_policies);
+  mhash_set (&sm->sr_steer_policies_hash, &key, steer_pl - sm->steer_policies,
+	     NULL);
 
   if (traffic_type == SR_STEER_L2)
     {
@@ -345,18 +350,12 @@ update_fib:
 
 cleanup_error_encap:
   pool_put (sm->steer_policies, steer_pl);
-  hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
-  key_copy = (void *) (hp->key);
-  hash_unset_mem (sm->steer_policies_index_by_key, &key);
-  vec_free (key_copy);
+  mhash_unset (&sm->sr_steer_policies_hash, &key, NULL);
   return -5;
 
 cleanup_error_redirection:
   pool_put (sm->steer_policies, steer_pl);
-  hp = hash_get_pair (sm->steer_policies_index_by_key, &key);
-  key_copy = (void *) (hp->key);
-  hash_unset_mem (sm->steer_policies_index_by_key, &key);
-  vec_free (key_copy);
+  mhash_unset (&sm->sr_steer_policies_hash, &key, NULL);
   return -3;
 }
 
@@ -379,6 +378,8 @@ sr_steer_policy_command_fn (vlib_main_t * vm, unformat_input_t * input,
 
   u8 sr_policy_set = 0;
 
+  memset (&prefix, 0, sizeof (ip46_address_t));
+
   int rv;
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
@@ -450,7 +451,7 @@ sr_steer_policy_command_fn (vlib_main_t * vm, unformat_input_t * input,
 				"Unable to do SW redirect. Incorrect interface.");
     case -4:
       return clib_error_return (0,
-				"The requested SR policy could not be deleted. Review the BSID/index.");
+				"The requested SR steering policy could not be deleted.");
     case -5:
       return clib_error_return (0,
 				"The SR policy is not an encapsulation one.");
@@ -541,8 +542,8 @@ sr_steering_init (vlib_main_t * vm)
   ip6_sr_main_t *sm = &sr_main;
 
   /* Init memory for function keys */
-  sm->steer_policies_index_by_key =
-    hash_create_mem (0, sizeof (sr_steering_key_t), sizeof (uword));
+  mhash_init (&sm->sr_steer_policies_hash, sizeof (uword),
+	      sizeof (sr_steering_key_t));
 
   sm->sw_iface_sr_policies = 0;
 
diff --git a/src/vnet/sr/sr_steering.md b/src/vnet/sr/sr_steering.md
new file mode 100644
index 00000000..cf446f81
--- /dev/null
+++ b/src/vnet/sr/sr_steering.md
@@ -0,0 +1,11 @@
+# Steering packets into a SR Policy     {#srv6_steering_doc}
+
+To steer packets in Transit into an SR policy (T.Insert, T.Encaps and T.Encaps.L2 behaviors), the user needs to create an 'sr steering policy'.
+
+    sr steer l3 2001::/64 via sr policy index 1
+    sr steer l3 2001::/64 via sr policy bsid cafe::1
+    sr steer l3 2001::/64 via sr policy bsid cafe::1 fib-table 3
+    sr steer l3 10.0.0.0/16 via sr policy bsid cafe::1
+    sr steer l2 TenGE0/1/0 via sr policy bsid cafe::1
+
+Disclaimer: The T.Encaps.L2 will steer L2 frames into an SR Policy. Notice that creating an SR steering policy for L2 frames will actually automatically *put the interface into promiscous mode*.
-- 
cgit 1.2.3-korg


From ef486b1545d892f9f0e0d35e7e57cb0ca04d7ff7 Mon Sep 17 00:00:00 2001
From: Hongjun Ni 
Date: Wed, 12 Apr 2017 19:21:16 +0800
Subject: Add GTP-U plugin. VPP-694

Basic GTP-U feature

Change-Id: I31226f890a92c5303ac06e112ed7820cae52d9bd
Signed-off-by: Hongjun Ni 
---
 src/configure.ac                       |    1 +
 src/plugins/Makefile.am                |    5 +
 src/plugins/gtpu.am                    |   38 +
 src/plugins/gtpu/gtpu.api              |  120 +++
 src/plugins/gtpu/gtpu.c                | 1128 +++++++++++++++++++++++++++
 src/plugins/gtpu/gtpu.h                |  262 +++++++
 src/plugins/gtpu/gtpu_all_api_h.h      |   18 +
 src/plugins/gtpu/gtpu_api.c            |  256 +++++++
 src/plugins/gtpu/gtpu_decap.c          | 1305 ++++++++++++++++++++++++++++++++
 src/plugins/gtpu/gtpu_encap.c          |  569 ++++++++++++++
 src/plugins/gtpu/gtpu_error.def        |   18 +
 src/plugins/gtpu/gtpu_msg_enum.h       |   31 +
 src/plugins/gtpu/gtpu_test.c           |  498 ++++++++++++
 src/vnet/mfib/mfib_types.h             |    2 +
 src/vnet/udp/udp.h                     |    2 +
 src/vpp-api/java/Makefile.am           |   20 +
 src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.c |  107 +++
 src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.h |   42 +
 test/test_gtpu.py                      |  289 +++++++
 test/vpp_papi_provider.py              |   32 +
 20 files changed, 4743 insertions(+)
 create mode 100644 src/plugins/gtpu.am
 create mode 100644 src/plugins/gtpu/gtpu.api
 create mode 100644 src/plugins/gtpu/gtpu.c
 create mode 100644 src/plugins/gtpu/gtpu.h
 create mode 100644 src/plugins/gtpu/gtpu_all_api_h.h
 create mode 100644 src/plugins/gtpu/gtpu_api.c
 create mode 100644 src/plugins/gtpu/gtpu_decap.c
 create mode 100644 src/plugins/gtpu/gtpu_encap.c
 create mode 100644 src/plugins/gtpu/gtpu_error.def
 create mode 100644 src/plugins/gtpu/gtpu_msg_enum.h
 create mode 100644 src/plugins/gtpu/gtpu_test.c
 create mode 100644 src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.c
 create mode 100644 src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.h
 create mode 100644 test/test_gtpu.py

(limited to 'src/plugins/Makefile.am')

diff --git a/src/configure.ac b/src/configure.ac
index 581b0968..2a907b2f 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -147,6 +147,7 @@ AC_SUBST(AR_FLAGS)
 PLUGIN_ENABLED(acl)
 PLUGIN_ENABLED(dpdk)
 PLUGIN_ENABLED(flowperpkt)
+PLUGIN_ENABLED(gtpu)
 PLUGIN_ENABLED(ila)
 PLUGIN_ENABLED(ioam)
 PLUGIN_ENABLED(ixge)
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 623892e7..388c9ad2 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -41,6 +41,11 @@ if ENABLE_FLOWPERPKT_PLUGIN
 include flowperpkt.am
 endif
 
+
+if ENABLE_GTPU_PLUGIN
+include gtpu.am
+endif
+
 if ENABLE_ILA_PLUGIN
 include ila.am
 endif
diff --git a/src/plugins/gtpu.am b/src/plugins/gtpu.am
new file mode 100644
index 00000000..9e5805b7
--- /dev/null
+++ b/src/plugins/gtpu.am
@@ -0,0 +1,38 @@
+# Copyright (c) 2016 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppapitestplugins_LTLIBRARIES += gtpu_test_plugin.la
+vppplugins_LTLIBRARIES += gtpu_plugin.la
+
+gtpu_plugin_la_SOURCES =		\
+	gtpu/gtpu_decap.c		    \
+    gtpu/gtpu_encap.c		    \
+	gtpu/gtpu.c				    \
+	gtpu/gtpu_api.c
+
+BUILT_SOURCES +=			\
+	gtpu/gtpu.api.h		    \
+	gtpu/gtpu.api.json
+
+API_FILES += gtpu/gtpu.api
+
+noinst_HEADERS +=			\
+  gtpu/gtpu_all_api_h.h		\
+  gtpu/gtpu_msg_enum.h		\
+  gtpu/gtpu.api.h
+
+gtpu_test_plugin_la_SOURCES = \
+  gtpu/gtpu_test.c            \
+  gtpu/gtpu_plugin.api.h
+
+# vi:syntax=automake
diff --git a/src/plugins/gtpu/gtpu.api b/src/plugins/gtpu/gtpu.api
new file mode 100644
index 00000000..b11670b3
--- /dev/null
+++ b/src/plugins/gtpu/gtpu.api
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** \brief /** \brief Set or delete an GTPU tunnel
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - add address if non-zero, else delete
+    @param is_ipv6 - src_address and dst_address is ipv6 or not
+    @param src_address - GTPU tunnel's source address.
+    @param dst_address - GTPU tunnel's destination address.
+    @param mcast_sw_if_index - version, O-bit and C-bit (see nsh_packet.h)
+    @param encap_vrf_id - fib identifier used for outgoing encapsulated packets
+    @param decap_next_index - the index of the next node if success
+    @param teid - Local Tunnel Endpoint Identifier
+*/
+define gtpu_add_del_tunnel
+{
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_ipv6;
+  u8 src_address[16];
+  u8 dst_address[16];
+  u32 mcast_sw_if_index;
+  u32 encap_vrf_id;
+  u32 decap_next_index;
+  u32 teid;
+};
+
+/** \brief reply for set or delete an GTPU tunnel
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param sw_if_index - software index of the interface
+*/
+define gtpu_add_del_tunnel_reply
+{
+  u32 context;
+  i32 retval;
+  u32 sw_if_index;
+};
+
+/** \brief Dump GTPU tunnel
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface
+*/
+define gtpu_tunnel_dump
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+};
+
+/** \brief /** \brief dump details of an GTPU tunnel
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface
+    @param is_ipv6 - src_address and dst_address is ipv6 or not
+    @param src_address - GTPU tunnel's source address.
+    @param dst_address - GTPU tunnel's destination address.
+    @param mcast_sw_if_index - version, O-bit and C-bit (see nsh_packet.h)
+    @param encap_vrf_id - fib identifier used for outgoing encapsulated packets
+    @param decap_next_index - the index of the next node if success
+    @param teid - Local Tunnel Endpoint Identifier
+*/
+define gtpu_tunnel_details
+{
+  u32 context;
+  u32 sw_if_index;
+  u8 is_ipv6;
+  u8 src_address[16];
+  u8 dst_address[16];
+  u32 mcast_sw_if_index;
+  u32 encap_vrf_id;
+  u32 decap_next_index;
+  u32 teid;
+};
+
+/** \brief Interface set gtpu-bypass request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - interface used to reach neighbor
+    @param is_ipv6 - if non-zero, enable ipv6-gtpu-bypass, else ipv4-gtpu-bypass
+    @param enable - if non-zero enable, else disable
+*/
+define sw_interface_set_gtpu_bypass
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+  u8 is_ipv6;
+  u8 enable;
+};
+
+/** \brief Interface set gtpu-bypass response
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+*/
+define sw_interface_set_gtpu_bypass_reply
+{
+  u32 context;
+  i32 retval;
+};
+
+/*
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/gtpu/gtpu.c b/src/plugins/gtpu/gtpu.c
new file mode 100644
index 00000000..6a5c5a34
--- /dev/null
+++ b/src/plugins/gtpu/gtpu.c
@@ -0,0 +1,1128 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+
+gtpu_main_t gtpu_main;
+
+static u8 *
+format_decap_next (u8 * s, va_list * args)
+{
+  u32 next_index = va_arg (*args, u32);
+
+  switch (next_index)
+    {
+    case GTPU_INPUT_NEXT_DROP:
+      return format (s, "drop");
+    case GTPU_INPUT_NEXT_L2_INPUT:
+      return format (s, "l2");
+    default:
+      return format (s, "index %d", next_index);
+    }
+  return s;
+}
+
+u8 *
+format_gtpu_tunnel (u8 * s, va_list * args)
+{
+  gtpu_tunnel_t *t = va_arg (*args, gtpu_tunnel_t *);
+  gtpu_main_t *ngm = >pu_main;
+
+  s = format (s, "[%d] src %U dst %U teid %d sw_if_index %d ",
+	      t - ngm->tunnels,
+	      format_ip46_address, &t->src, IP46_TYPE_ANY,
+	      format_ip46_address, &t->dst, IP46_TYPE_ANY,
+	      t->teid, t->sw_if_index);
+
+  if (ip46_address_is_multicast (&t->dst))
+    s = format (s, "mcast_sw_if_index %d ", t->mcast_sw_if_index);
+
+  s = format (s, "encap_fib_index %d fib_entry_index %d decap_next %U\n",
+	      t->encap_fib_index, t->fib_entry_index,
+	      format_decap_next, t->decap_next_index);
+  return s;
+}
+
+static u8 *
+format_gtpu_name (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  return format (s, "gtpu_tunnel%d", dev_instance);
+}
+
+static uword
+dummy_interface_tx (vlib_main_t * vm,
+		    vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  clib_warning ("you shouldn't be here, leaking buffers...");
+  return frame->n_vectors;
+}
+
+static clib_error_t *
+gtpu_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
+{
+  u32 hw_flags = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ?
+    VNET_HW_INTERFACE_FLAG_LINK_UP : 0;
+  vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags);
+
+  return /* no error */ 0;
+}
+
+/* *INDENT-OFF* */
+VNET_DEVICE_CLASS (gtpu_device_class,static) = {
+  .name = "GTPU",
+  .format_device_name = format_gtpu_name,
+  .format_tx_trace = format_gtpu_encap_trace,
+  .tx_function = dummy_interface_tx,
+  .admin_up_down_function = gtpu_interface_admin_up_down,
+};
+/* *INDENT-ON* */
+
+static u8 *
+format_gtpu_header_with_length (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  s = format (s, "unimplemented dev %u", dev_instance);
+  return s;
+}
+
+/* *INDENT-OFF* */
+VNET_HW_INTERFACE_CLASS (gtpu_hw_class) =
+{
+  .name = "GTPU",
+  .format_header = format_gtpu_header_with_length,
+  .build_rewrite = default_build_rewrite,
+};
+/* *INDENT-ON* */
+
+static void
+gtpu_tunnel_restack_dpo (gtpu_tunnel_t * t)
+{
+  dpo_id_t dpo = DPO_INVALID;
+  u32 encap_index = ip46_address_is_ip4 (&t->dst) ?
+    gtpu4_encap_node.index : gtpu6_encap_node.index;
+  fib_forward_chain_type_t forw_type = ip46_address_is_ip4 (&t->dst) ?
+    FIB_FORW_CHAIN_TYPE_UNICAST_IP4 : FIB_FORW_CHAIN_TYPE_UNICAST_IP6;
+
+  fib_entry_contribute_forwarding (t->fib_entry_index, forw_type, &dpo);
+  dpo_stack_from_node (encap_index, &t->next_dpo, &dpo);
+  dpo_reset (&dpo);
+}
+
+static gtpu_tunnel_t *
+gtpu_tunnel_from_fib_node (fib_node_t * node)
+{
+  return ((gtpu_tunnel_t *) (((char *) node) -
+			     STRUCT_OFFSET_OF (gtpu_tunnel_t, node)));
+}
+
+/**
+ * Function definition to backwalk a FIB node -
+ * Here we will restack the new dpo of GTPU DIP to encap node.
+ */
+static fib_node_back_walk_rc_t
+gtpu_tunnel_back_walk (fib_node_t * node, fib_node_back_walk_ctx_t * ctx)
+{
+  gtpu_tunnel_restack_dpo (gtpu_tunnel_from_fib_node (node));
+  return (FIB_NODE_BACK_WALK_CONTINUE);
+}
+
+/**
+ * Function definition to get a FIB node from its index
+ */
+static fib_node_t *
+gtpu_tunnel_fib_node_get (fib_node_index_t index)
+{
+  gtpu_tunnel_t *t;
+  gtpu_main_t *gtm = >pu_main;
+
+  t = pool_elt_at_index (gtm->tunnels, index);
+
+  return (&t->node);
+}
+
+/**
+ * Function definition to inform the FIB node that its last lock has gone.
+ */
+static void
+gtpu_tunnel_last_lock_gone (fib_node_t * node)
+{
+  /*
+   * The GTPU tunnel is a root of the graph. As such
+   * it never has children and thus is never locked.
+   */
+  ASSERT (0);
+}
+
+/*
+ * Virtual function table registered by GTPU tunnels
+ * for participation in the FIB object graph.
+ */
+const static fib_node_vft_t gtpu_vft = {
+  .fnv_get = gtpu_tunnel_fib_node_get,
+  .fnv_last_lock = gtpu_tunnel_last_lock_gone,
+  .fnv_back_walk = gtpu_tunnel_back_walk,
+};
+
+
+#define foreach_copy_field                      \
+_(teid)                                          \
+_(mcast_sw_if_index)                            \
+_(encap_fib_index)                              \
+_(decap_next_index)                             \
+_(src)                                          \
+_(dst)
+
+static void
+ip_udp_gtpu_rewrite (gtpu_tunnel_t * t, bool is_ip6)
+{
+  union
+  {
+    ip4_gtpu_header_t *h4;
+    ip6_gtpu_header_t *h6;
+    u8 *rw;
+  } r =
+  {
+  .rw = 0};
+  int len = is_ip6 ? sizeof *r.h6 : sizeof *r.h4;
+
+  vec_validate_aligned (r.rw, len - 1, CLIB_CACHE_LINE_BYTES);
+
+  udp_header_t *udp;
+  gtpu_header_t *gtpu;
+  /* Fixed portion of the (outer) ip header */
+  if (!is_ip6)
+    {
+      ip4_header_t *ip = &r.h4->ip4;
+      udp = &r.h4->udp;
+      gtpu = &r.h4->gtpu;
+      ip->ip_version_and_header_length = 0x45;
+      ip->ttl = 254;
+      ip->protocol = IP_PROTOCOL_UDP;
+
+      ip->src_address = t->src.ip4;
+      ip->dst_address = t->dst.ip4;
+
+      /* we fix up the ip4 header length and checksum after-the-fact */
+      ip->checksum = ip4_header_checksum (ip);
+    }
+  else
+    {
+      ip6_header_t *ip = &r.h6->ip6;
+      udp = &r.h6->udp;
+      gtpu = &r.h6->gtpu;
+      ip->ip_version_traffic_class_and_flow_label =
+	clib_host_to_net_u32 (6 << 28);
+      ip->hop_limit = 255;
+      ip->protocol = IP_PROTOCOL_UDP;
+
+      ip->src_address = t->src.ip6;
+      ip->dst_address = t->dst.ip6;
+    }
+
+  /* UDP header, randomize src port on something, maybe? */
+  udp->src_port = clib_host_to_net_u16 (2152);
+  udp->dst_port = clib_host_to_net_u16 (UDP_DST_PORT_GTPU);
+
+  /* GTPU header */
+  gtpu->ver_flags = GTPU_V1_VER | GTPU_PT_GTP;
+  gtpu->type = GTPU_TYPE_GTPU;
+  gtpu->teid = clib_host_to_net_u32 (t->teid);
+
+  t->rewrite = r.rw;
+  /* Now only support 8-byte gtpu header. TBD */
+  _vec_len (t->rewrite) = sizeof (ip4_gtpu_header_t) - 4;
+
+  return;
+}
+
+static bool
+gtpu_decap_next_is_valid (gtpu_main_t * gtm, u32 is_ip6, u32 decap_next_index)
+{
+  vlib_main_t *vm = gtm->vlib_main;
+  u32 input_idx = (!is_ip6) ? gtpu4_input_node.index : gtpu6_input_node.index;
+  vlib_node_runtime_t *r = vlib_node_get_runtime (vm, input_idx);
+
+  return decap_next_index < r->n_next_nodes;
+}
+
+static void
+hash_set_key_copy (uword ** h, void *key, uword v)
+{
+  size_t ksz = hash_header (*h)->user;
+  void *copy = clib_mem_alloc (ksz);
+  clib_memcpy (copy, key, ksz);
+  hash_set_mem (*h, copy, v);
+}
+
+static void
+hash_unset_key_free (uword ** h, void *key)
+{
+  hash_pair_t *hp = hash_get_pair_mem (*h, key);
+  ASSERT (hp);
+  key = uword_to_pointer (hp->key, void *);
+  hash_unset_mem (*h, key);
+  clib_mem_free (key);
+}
+
+static uword
+vtep_addr_ref (ip46_address_t * ip)
+{
+  uword *vtep = ip46_address_is_ip4 (ip) ?
+    hash_get (gtpu_main.vtep4, ip->ip4.as_u32) :
+    hash_get_mem (gtpu_main.vtep6, &ip->ip6);
+  if (vtep)
+    return ++(*vtep);
+  ip46_address_is_ip4 (ip) ?
+    hash_set (gtpu_main.vtep4, ip->ip4.as_u32, 1) :
+    hash_set_key_copy (>pu_main.vtep6, &ip->ip6, 1);
+  return 1;
+}
+
+static uword
+vtep_addr_unref (ip46_address_t * ip)
+{
+  uword *vtep = ip46_address_is_ip4 (ip) ?
+    hash_get (gtpu_main.vtep4, ip->ip4.as_u32) :
+    hash_get_mem (gtpu_main.vtep6, &ip->ip6);
+  ASSERT (vtep);
+  if (--(*vtep) != 0)
+    return *vtep;
+  ip46_address_is_ip4 (ip) ?
+    hash_unset (gtpu_main.vtep4, ip->ip4.as_u32) :
+    hash_unset_key_free (>pu_main.vtep6, &ip->ip6);
+  return 0;
+}
+
+typedef CLIB_PACKED (union
+		     {
+		     struct
+		     {
+		     fib_node_index_t mfib_entry_index;
+		     adj_index_t mcast_adj_index;
+		     }; u64 as_u64;
+		     }) mcast_shared_t;
+
+static inline mcast_shared_t
+mcast_shared_get (ip46_address_t * ip)
+{
+  ASSERT (ip46_address_is_multicast (ip));
+  uword *p = hash_get_mem (gtpu_main.mcast_shared, ip);
+  ASSERT (p);
+  return (mcast_shared_t)
+  {
+  .as_u64 = *p};
+}
+
+static inline void
+mcast_shared_add (ip46_address_t * dst, fib_node_index_t mfei, adj_index_t ai)
+{
+  mcast_shared_t new_ep = {
+    .mcast_adj_index = ai,
+    .mfib_entry_index = mfei,
+  };
+
+  hash_set_key_copy (>pu_main.mcast_shared, dst, new_ep.as_u64);
+}
+
+static inline void
+mcast_shared_remove (ip46_address_t * dst)
+{
+  mcast_shared_t ep = mcast_shared_get (dst);
+
+  adj_unlock (ep.mcast_adj_index);
+  mfib_table_entry_delete_index (ep.mfib_entry_index, MFIB_SOURCE_GTPU);
+
+  hash_unset_key_free (>pu_main.mcast_shared, dst);
+}
+
+static inline fib_protocol_t
+fib_ip_proto (bool is_ip6)
+{
+  return (is_ip6) ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4;
+}
+
+int vnet_gtpu_add_del_tunnel
+  (vnet_gtpu_add_del_tunnel_args_t * a, u32 * sw_if_indexp)
+{
+  gtpu_main_t *gtm = >pu_main;
+  gtpu_tunnel_t *t = 0;
+  vnet_main_t *vnm = gtm->vnet_main;
+  uword *p;
+  u32 hw_if_index = ~0;
+  u32 sw_if_index = ~0;
+  gtpu4_tunnel_key_t key4;
+  gtpu6_tunnel_key_t key6;
+  u32 is_ip6 = a->is_ip6;
+
+  if (!is_ip6)
+    {
+      key4.src = a->dst.ip4.as_u32;	/* decap src in key is encap dst in config */
+      key4.teid = clib_host_to_net_u32 (a->teid);
+      p = hash_get (gtm->gtpu4_tunnel_by_key, key4.as_u64);
+    }
+  else
+    {
+      key6.src = a->dst.ip6;
+      key6.teid = clib_host_to_net_u32 (a->teid);
+      p = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6);
+    }
+
+  if (a->is_add)
+    {
+      l2input_main_t *l2im = &l2input_main;
+
+      /* adding a tunnel: tunnel must not already exist */
+      if (p)
+	return VNET_API_ERROR_TUNNEL_EXIST;
+
+      /*if not set explicitly, default to l2 */
+      if (a->decap_next_index == ~0)
+	a->decap_next_index = GTPU_INPUT_NEXT_L2_INPUT;
+      if (!gtpu_decap_next_is_valid (gtm, is_ip6, a->decap_next_index))
+	return VNET_API_ERROR_INVALID_DECAP_NEXT;
+
+      pool_get_aligned (gtm->tunnels, t, CLIB_CACHE_LINE_BYTES);
+      memset (t, 0, sizeof (*t));
+
+      /* copy from arg structure */
+#define _(x) t->x = a->x;
+      foreach_copy_field;
+#undef _
+
+      ip_udp_gtpu_rewrite (t, is_ip6);
+
+      /* copy the key */
+      if (is_ip6)
+	hash_set_key_copy (>m->gtpu6_tunnel_by_key, &key6,
+			   t - gtm->tunnels);
+      else
+	hash_set (gtm->gtpu4_tunnel_by_key, key4.as_u64, t - gtm->tunnels);
+
+      vnet_hw_interface_t *hi;
+      if (vec_len (gtm->free_gtpu_tunnel_hw_if_indices) > 0)
+	{
+	  vnet_interface_main_t *im = &vnm->interface_main;
+	  hw_if_index = gtm->free_gtpu_tunnel_hw_if_indices
+	    [vec_len (gtm->free_gtpu_tunnel_hw_if_indices) - 1];
+	  _vec_len (gtm->free_gtpu_tunnel_hw_if_indices) -= 1;
+
+	  hi = vnet_get_hw_interface (vnm, hw_if_index);
+	  hi->dev_instance = t - gtm->tunnels;
+	  hi->hw_instance = hi->dev_instance;
+
+	  /* clear old stats of freed tunnel before reuse */
+	  sw_if_index = hi->sw_if_index;
+	  vnet_interface_counter_lock (im);
+	  vlib_zero_combined_counter
+	    (&im->combined_sw_if_counters[VNET_INTERFACE_COUNTER_TX],
+	     sw_if_index);
+	  vlib_zero_combined_counter (&im->combined_sw_if_counters
+				      [VNET_INTERFACE_COUNTER_RX],
+				      sw_if_index);
+	  vlib_zero_simple_counter (&im->sw_if_counters
+				    [VNET_INTERFACE_COUNTER_DROP],
+				    sw_if_index);
+	  vnet_interface_counter_unlock (im);
+	}
+      else
+	{
+	  hw_if_index = vnet_register_interface
+	    (vnm, gtpu_device_class.index, t - gtm->tunnels,
+	     gtpu_hw_class.index, t - gtm->tunnels);
+	  hi = vnet_get_hw_interface (vnm, hw_if_index);
+	}
+
+      t->hw_if_index = hw_if_index;
+      t->sw_if_index = sw_if_index = hi->sw_if_index;
+
+      vec_validate_init_empty (gtm->tunnel_index_by_sw_if_index, sw_if_index,
+			       ~0);
+      gtm->tunnel_index_by_sw_if_index[sw_if_index] = t - gtm->tunnels;
+
+      /* setup l2 input config with l2 feature and bd 0 to drop packet */
+      vec_validate (l2im->configs, sw_if_index);
+      l2im->configs[sw_if_index].feature_bitmap = L2INPUT_FEAT_DROP;
+      l2im->configs[sw_if_index].bd_index = 0;
+
+      vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index);
+      si->flags &= ~VNET_SW_INTERFACE_FLAG_HIDDEN;
+      vnet_sw_interface_set_flags (vnm, sw_if_index,
+				   VNET_SW_INTERFACE_FLAG_ADMIN_UP);
+
+      fib_node_init (&t->node, gtm->fib_node_type);
+      fib_prefix_t tun_dst_pfx;
+      u32 encap_index = !is_ip6 ?
+	gtpu4_encap_node.index : gtpu6_encap_node.index;
+      vnet_flood_class_t flood_class = VNET_FLOOD_CLASS_TUNNEL_NORMAL;
+
+      fib_prefix_from_ip46_addr (&t->dst, &tun_dst_pfx);
+      if (!ip46_address_is_multicast (&t->dst))
+	{
+	  /* Unicast tunnel -
+	   * source the FIB entry for the tunnel's destination
+	   * and become a child thereof. The tunnel will then get poked
+	   * when the forwarding for the entry updates, and the tunnel can
+	   * re-stack accordingly
+	   */
+	  vtep_addr_ref (&t->src);
+	  t->fib_entry_index = fib_table_entry_special_add
+	    (t->encap_fib_index, &tun_dst_pfx, FIB_SOURCE_RR,
+	     FIB_ENTRY_FLAG_NONE);
+	  t->sibling_index = fib_entry_child_add
+	    (t->fib_entry_index, gtm->fib_node_type, t - gtm->tunnels);
+	  gtpu_tunnel_restack_dpo (t);
+	}
+      else
+	{
+	  /* Multicast tunnel -
+	   * as the same mcast group can be used for mutiple mcast tunnels
+	   * with different VNIs, create the output fib adjecency only if
+	   * it does not already exist
+	   */
+	  fib_protocol_t fp = fib_ip_proto (is_ip6);
+
+	  if (vtep_addr_ref (&t->dst) == 1)
+	    {
+	      fib_node_index_t mfei;
+	      adj_index_t ai;
+	      fib_route_path_t path = {
+		.frp_proto = fp,
+		.frp_addr = zero_addr,
+		.frp_sw_if_index = 0xffffffff,
+		.frp_fib_index = ~0,
+		.frp_weight = 0,
+		.frp_flags = FIB_ROUTE_PATH_LOCAL,
+	      };
+	      const mfib_prefix_t mpfx = {
+		.fp_proto = fp,
+		.fp_len = (is_ip6 ? 128 : 32),
+		.fp_grp_addr = tun_dst_pfx.fp_addr,
+	      };
+
+	      /*
+	       * Setup the (*,G) to receive traffic on the mcast group
+	       *  - the forwarding interface is for-us
+	       *  - the accepting interface is that from the API
+	       */
+	      mfib_table_entry_path_update (t->encap_fib_index,
+					    &mpfx,
+					    MFIB_SOURCE_GTPU,
+					    &path, MFIB_ITF_FLAG_FORWARD);
+
+	      path.frp_sw_if_index = a->mcast_sw_if_index;
+	      path.frp_flags = FIB_ROUTE_PATH_FLAG_NONE;
+	      mfei = mfib_table_entry_path_update (t->encap_fib_index,
+						   &mpfx,
+						   MFIB_SOURCE_GTPU,
+						   &path,
+						   MFIB_ITF_FLAG_ACCEPT);
+
+	      /*
+	       * Create the mcast adjacency to send traffic to the group
+	       */
+	      ai = adj_mcast_add_or_lock (fp,
+					  fib_proto_to_link (fp),
+					  a->mcast_sw_if_index);
+
+	      /*
+	       * create a new end-point
+	       */
+	      mcast_shared_add (&t->dst, mfei, ai);
+	    }
+
+	  dpo_id_t dpo = DPO_INVALID;
+	  mcast_shared_t ep = mcast_shared_get (&t->dst);
+
+	  /* Stack shared mcast dst mac addr rewrite on encap */
+	  dpo_set (&dpo, DPO_ADJACENCY_MCAST,
+		   fib_proto_to_dpo (fp), ep.mcast_adj_index);
+
+	  dpo_stack_from_node (encap_index, &t->next_dpo, &dpo);
+
+	  dpo_reset (&dpo);
+	  flood_class = VNET_FLOOD_CLASS_TUNNEL_MASTER;
+	}
+
+      /* Set gtpu tunnel output node */
+      hi->output_node_index = encap_index;
+
+      vnet_get_sw_interface (vnet_get_main (), sw_if_index)->flood_class =
+	flood_class;
+    }
+  else
+    {
+      /* deleting a tunnel: tunnel must exist */
+      if (!p)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      t = pool_elt_at_index (gtm->tunnels, p[0]);
+      sw_if_index = t->sw_if_index;
+
+      vnet_sw_interface_set_flags (vnm, t->sw_if_index, 0 /* down */ );
+      vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, t->sw_if_index);
+      si->flags |= VNET_SW_INTERFACE_FLAG_HIDDEN;
+
+      /* make sure tunnel is removed from l2 bd or xconnect */
+      set_int_l2_mode (gtm->vlib_main, vnm, MODE_L3, t->sw_if_index, 0, 0, 0,
+		       0);
+      vec_add1 (gtm->free_gtpu_tunnel_hw_if_indices, t->hw_if_index);
+
+      gtm->tunnel_index_by_sw_if_index[t->sw_if_index] = ~0;
+
+      if (!is_ip6)
+	hash_unset (gtm->gtpu4_tunnel_by_key, key4.as_u64);
+      else
+	hash_unset_key_free (>m->gtpu6_tunnel_by_key, &key6);
+
+      if (!ip46_address_is_multicast (&t->dst))
+	{
+	  vtep_addr_unref (&t->src);
+	  fib_entry_child_remove (t->fib_entry_index, t->sibling_index);
+	  fib_table_entry_delete_index (t->fib_entry_index, FIB_SOURCE_RR);
+	}
+      else if (vtep_addr_unref (&t->dst) == 0)
+	{
+	  mcast_shared_remove (&t->dst);
+	}
+
+      fib_node_deinit (&t->node);
+      vec_free (t->rewrite);
+      pool_put (gtm->tunnels, t);
+    }
+
+  if (sw_if_indexp)
+    *sw_if_indexp = sw_if_index;
+
+  return 0;
+}
+
+static uword
+get_decap_next_for_node (u32 node_index, u32 ipv4_set)
+{
+  gtpu_main_t *gtm = >pu_main;
+  vlib_main_t *vm = gtm->vlib_main;
+  uword input_node = (ipv4_set) ? gtpu4_input_node.index :
+    gtpu6_input_node.index;
+
+  return vlib_node_add_next (vm, input_node, node_index);
+}
+
+static uword
+unformat_decap_next (unformat_input_t * input, va_list * args)
+{
+  u32 *result = va_arg (*args, u32 *);
+  u32 ipv4_set = va_arg (*args, int);
+  gtpu_main_t *gtm = >pu_main;
+  vlib_main_t *vm = gtm->vlib_main;
+  u32 node_index;
+  u32 tmp;
+
+  if (unformat (input, "l2"))
+    *result = GTPU_INPUT_NEXT_L2_INPUT;
+  else if (unformat (input, "node %U", unformat_vlib_node, vm, &node_index))
+    *result = get_decap_next_for_node (node_index, ipv4_set);
+  else if (unformat (input, "%d", &tmp))
+    *result = tmp;
+  else
+    return 0;
+  return 1;
+}
+
+static clib_error_t *
+gtpu_add_del_tunnel_command_fn (vlib_main_t * vm,
+				unformat_input_t * input,
+				vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip46_address_t src, dst;
+  u8 is_add = 1;
+  u8 src_set = 0;
+  u8 dst_set = 0;
+  u8 grp_set = 0;
+  u8 ipv4_set = 0;
+  u8 ipv6_set = 0;
+  u32 encap_fib_index = 0;
+  u32 mcast_sw_if_index = ~0;
+  u32 decap_next_index = GTPU_INPUT_NEXT_L2_INPUT;
+  u32 teid = 0;
+  u32 tmp;
+  int rv;
+  vnet_gtpu_add_del_tunnel_args_t _a, *a = &_a;
+  u32 tunnel_sw_if_index;
+  clib_error_t *error = NULL;
+
+  /* Cant "universally zero init" (={0}) due to GCC bug 53119 */
+  memset (&src, 0, sizeof src);
+  memset (&dst, 0, sizeof dst);
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "del"))
+	{
+	  is_add = 0;
+	}
+      else if (unformat (line_input, "src %U",
+			 unformat_ip4_address, &src.ip4))
+	{
+	  src_set = 1;
+	  ipv4_set = 1;
+	}
+      else if (unformat (line_input, "dst %U",
+			 unformat_ip4_address, &dst.ip4))
+	{
+	  dst_set = 1;
+	  ipv4_set = 1;
+	}
+      else if (unformat (line_input, "src %U",
+			 unformat_ip6_address, &src.ip6))
+	{
+	  src_set = 1;
+	  ipv6_set = 1;
+	}
+      else if (unformat (line_input, "dst %U",
+			 unformat_ip6_address, &dst.ip6))
+	{
+	  dst_set = 1;
+	  ipv6_set = 1;
+	}
+      else if (unformat (line_input, "group %U %U",
+			 unformat_ip4_address, &dst.ip4,
+			 unformat_vnet_sw_interface,
+			 vnet_get_main (), &mcast_sw_if_index))
+	{
+	  grp_set = dst_set = 1;
+	  ipv4_set = 1;
+	}
+      else if (unformat (line_input, "group %U %U",
+			 unformat_ip6_address, &dst.ip6,
+			 unformat_vnet_sw_interface,
+			 vnet_get_main (), &mcast_sw_if_index))
+	{
+	  grp_set = dst_set = 1;
+	  ipv6_set = 1;
+	}
+      else if (unformat (line_input, "encap-vrf-id %d", &tmp))
+	{
+	  encap_fib_index = fib_table_find (fib_ip_proto (ipv6_set), tmp);
+	  if (encap_fib_index == ~0)
+	    {
+	      error =
+		clib_error_return (0, "nonexistent encap-vrf-id %d", tmp);
+	      goto done;
+	    }
+	}
+      else if (unformat (line_input, "decap-next %U", unformat_decap_next,
+			 &decap_next_index, ipv4_set))
+	;
+      else if (unformat (line_input, "teid %d", &teid))
+	;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (src_set == 0)
+    {
+      error = clib_error_return (0, "tunnel src address not specified");
+      goto done;
+    }
+
+  if (dst_set == 0)
+    {
+      error = clib_error_return (0, "tunnel dst address not specified");
+      goto done;
+    }
+
+  if (grp_set && !ip46_address_is_multicast (&dst))
+    {
+      error = clib_error_return (0, "tunnel group address not multicast");
+      goto done;
+    }
+
+  if (grp_set == 0 && ip46_address_is_multicast (&dst))
+    {
+      error = clib_error_return (0, "dst address must be unicast");
+      goto done;
+    }
+
+  if (grp_set && mcast_sw_if_index == ~0)
+    {
+      error = clib_error_return (0, "tunnel nonexistent multicast device");
+      goto done;
+    }
+
+  if (ipv4_set && ipv6_set)
+    {
+      error = clib_error_return (0, "both IPv4 and IPv6 addresses specified");
+      goto done;
+    }
+
+  if (ip46_address_cmp (&src, &dst) == 0)
+    {
+      error = clib_error_return (0, "src and dst addresses are identical");
+      goto done;
+    }
+
+  if (decap_next_index == ~0)
+    {
+      error = clib_error_return (0, "next node not found");
+      goto done;
+    }
+
+  memset (a, 0, sizeof (*a));
+
+  a->is_add = is_add;
+  a->is_ip6 = ipv6_set;
+
+#define _(x) a->x = x;
+  foreach_copy_field;
+#undef _
+
+  rv = vnet_gtpu_add_del_tunnel (a, &tunnel_sw_if_index);
+
+  switch (rv)
+    {
+    case 0:
+      if (is_add)
+	vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name,
+			 vnet_get_main (), tunnel_sw_if_index);
+      break;
+
+    case VNET_API_ERROR_TUNNEL_EXIST:
+      error = clib_error_return (0, "tunnel already exists...");
+      goto done;
+
+    case VNET_API_ERROR_NO_SUCH_ENTRY:
+      error = clib_error_return (0, "tunnel does not exist...");
+      goto done;
+
+    default:
+      error = clib_error_return
+	(0, "vnet_gtpu_add_del_tunnel returned %d", rv);
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * Add or delete a GTPU Tunnel.
+ *
+ * GTPU provides the features needed to allow L2 bridge domains (BDs)
+ * to span multiple servers. This is done by building an L2 overlay on
+ * top of an L3 network underlay using GTPU tunnels.
+ *
+ * This makes it possible for servers to be co-located in the same data
+ * center or be separated geographically as long as they are reachable
+ * through the underlay L3 network.
+ *
+ * You can refer to this kind of L2 overlay bridge domain as a GTPU
+ * (Virtual eXtensible VLAN) segment.
+ *
+ * @cliexpar
+ * Example of how to create a GTPU Tunnel:
+ * @cliexcmd{create gtpu tunnel src 10.0.3.1 dst 10.0.3.3 teid 13 encap-vrf-id 7}
+ * Example of how to delete a GTPU Tunnel:
+ * @cliexcmd{create gtpu tunnel src 10.0.3.1 dst 10.0.3.3 teid 13 del}
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (create_gtpu_tunnel_command, static) = {
+  .path = "create gtpu tunnel",
+  .short_help =
+  "create gtpu tunnel src "
+  " {dst |group  } teid "
+  " [encap-vrf-id ] [decap-next [l2|node ]] [del]",
+  .function = gtpu_add_del_tunnel_command_fn,
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+show_gtpu_tunnel_command_fn (vlib_main_t * vm,
+			     unformat_input_t * input,
+			     vlib_cli_command_t * cmd)
+{
+  gtpu_main_t *gtm = >pu_main;
+  gtpu_tunnel_t *t;
+
+  if (pool_elts (gtm->tunnels) == 0)
+    vlib_cli_output (vm, "No gtpu tunnels configured...");
+
+  pool_foreach (t, gtm->tunnels, (
+				   {
+				   vlib_cli_output (vm, "%U",
+						    format_gtpu_tunnel, t);
+				   }
+		));
+
+  return 0;
+}
+
+/*?
+ * Display all the GTPU Tunnel entries.
+ *
+ * @cliexpar
+ * Example of how to display the GTPU Tunnel entries:
+ * @cliexstart{show gtpu tunnel}
+ * [0] src 10.0.3.1 dst 10.0.3.3 teid 13 encap_fib_index 0 sw_if_index 5 decap_next l2
+ * @cliexend
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_gtpu_tunnel_command, static) = {
+    .path = "show gtpu tunnel",
+    .short_help = "show gtpu tunnel",
+    .function = show_gtpu_tunnel_command_fn,
+};
+/* *INDENT-ON* */
+
+void
+vnet_int_gtpu_bypass_mode (u32 sw_if_index, u8 is_ip6, u8 is_enable)
+{
+  if (is_ip6)
+    vnet_feature_enable_disable ("ip6-unicast", "ip6-gtpu-bypass",
+				 sw_if_index, is_enable, 0, 0);
+  else
+    vnet_feature_enable_disable ("ip4-unicast", "ip4-gtpu-bypass",
+				 sw_if_index, is_enable, 0, 0);
+}
+
+static clib_error_t *
+set_ip_gtpu_bypass (u32 is_ip6,
+		    unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *error = 0;
+  u32 sw_if_index, is_enable;
+
+  sw_if_index = ~0;
+  is_enable = 1;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat_user
+	  (line_input, unformat_vnet_sw_interface, vnm, &sw_if_index))
+	;
+      else if (unformat (line_input, "del"))
+	is_enable = 0;
+      else
+	{
+	  error = unformat_parse_error (line_input);
+	  goto done;
+	}
+    }
+
+  if (~0 == sw_if_index)
+    {
+      error = clib_error_return (0, "unknown interface `%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
+
+  vnet_int_gtpu_bypass_mode (sw_if_index, is_ip6, is_enable);
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static clib_error_t *
+set_ip4_gtpu_bypass (vlib_main_t * vm,
+		     unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  return set_ip_gtpu_bypass (0, input, cmd);
+}
+
+/*?
+ * This command adds the 'ip4-gtpu-bypass' graph node for a given interface.
+ * By adding the IPv4 gtpu-bypass graph node to an interface, the node checks
+ *  for and validate input gtpu packet and bypass ip4-lookup, ip4-local,
+ * ip4-udp-lookup nodes to speedup gtpu packet forwarding. This node will
+ * cause extra overhead to for non-gtpu packets which is kept at a minimum.
+ *
+ * @cliexpar
+ * @parblock
+ * Example of graph node before ip4-gtpu-bypass is enabled:
+ * @cliexstart{show vlib graph ip4-gtpu-bypass}
+ *            Name                      Next                    Previous
+ * ip4-gtpu-bypass                error-drop [0]
+ *                                gtpu4-input [1]
+ *                                 ip4-lookup [2]
+ * @cliexend
+ *
+ * Example of how to enable ip4-gtpu-bypass on an interface:
+ * @cliexcmd{set interface ip gtpu-bypass GigabitEthernet2/0/0}
+ *
+ * Example of graph node after ip4-gtpu-bypass is enabled:
+ * @cliexstart{show vlib graph ip4-gtpu-bypass}
+ *            Name                      Next                    Previous
+ * ip4-gtpu-bypass                error-drop [0]               ip4-input
+ *                                gtpu4-input [1]        ip4-input-no-checksum
+ *                                 ip4-lookup [2]
+ * @cliexend
+ *
+ * Example of how to display the feature enabed on an interface:
+ * @cliexstart{show ip interface features GigabitEthernet2/0/0}
+ * IP feature paths configured on GigabitEthernet2/0/0...
+ * ...
+ * ipv4 unicast:
+ *   ip4-gtpu-bypass
+ *   ip4-lookup
+ * ...
+ * @cliexend
+ *
+ * Example of how to disable ip4-gtpu-bypass on an interface:
+ * @cliexcmd{set interface ip gtpu-bypass GigabitEthernet2/0/0 del}
+ * @endparblock
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_interface_ip_gtpu_bypass_command, static) = {
+  .path = "set interface ip gtpu-bypass",
+  .function = set_ip4_gtpu_bypass,
+  .short_help = "set interface ip gtpu-bypass  [del]",
+};
+/* *INDENT-ON* */
+
+static clib_error_t *
+set_ip6_gtpu_bypass (vlib_main_t * vm,
+		     unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  return set_ip_gtpu_bypass (1, input, cmd);
+}
+
+/*?
+ * This command adds the 'ip6-gtpu-bypass' graph node for a given interface.
+ * By adding the IPv6 gtpu-bypass graph node to an interface, the node checks
+ *  for and validate input gtpu packet and bypass ip6-lookup, ip6-local,
+ * ip6-udp-lookup nodes to speedup gtpu packet forwarding. This node will
+ * cause extra overhead to for non-gtpu packets which is kept at a minimum.
+ *
+ * @cliexpar
+ * @parblock
+ * Example of graph node before ip6-gtpu-bypass is enabled:
+ * @cliexstart{show vlib graph ip6-gtpu-bypass}
+ *            Name                      Next                    Previous
+ * ip6-gtpu-bypass                error-drop [0]
+ *                                gtpu6-input [1]
+ *                                 ip6-lookup [2]
+ * @cliexend
+ *
+ * Example of how to enable ip6-gtpu-bypass on an interface:
+ * @cliexcmd{set interface ip6 gtpu-bypass GigabitEthernet2/0/0}
+ *
+ * Example of graph node after ip6-gtpu-bypass is enabled:
+ * @cliexstart{show vlib graph ip6-gtpu-bypass}
+ *            Name                      Next                    Previous
+ * ip6-gtpu-bypass                error-drop [0]               ip6-input
+ *                                gtpu6-input [1]        ip4-input-no-checksum
+ *                                 ip6-lookup [2]
+ * @cliexend
+ *
+ * Example of how to display the feature enabed on an interface:
+ * @cliexstart{show ip interface features GigabitEthernet2/0/0}
+ * IP feature paths configured on GigabitEthernet2/0/0...
+ * ...
+ * ipv6 unicast:
+ *   ip6-gtpu-bypass
+ *   ip6-lookup
+ * ...
+ * @cliexend
+ *
+ * Example of how to disable ip6-gtpu-bypass on an interface:
+ * @cliexcmd{set interface ip6 gtpu-bypass GigabitEthernet2/0/0 del}
+ * @endparblock
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_interface_ip6_gtpu_bypass_command, static) = {
+  .path = "set interface ip6 gtpu-bypass",
+  .function = set_ip6_gtpu_bypass,
+  .short_help = "set interface ip gtpu-bypass  [del]",
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+gtpu_init (vlib_main_t * vm)
+{
+  gtpu_main_t *gtm = >pu_main;
+
+  gtm->vnet_main = vnet_get_main ();
+  gtm->vlib_main = vm;
+
+  /* initialize the ip6 hash */
+  gtm->gtpu6_tunnel_by_key = hash_create_mem (0,
+					      sizeof (gtpu6_tunnel_key_t),
+					      sizeof (uword));
+  gtm->vtep6 = hash_create_mem (0, sizeof (ip6_address_t), sizeof (uword));
+  gtm->mcast_shared = hash_create_mem (0,
+				       sizeof (ip46_address_t),
+				       sizeof (mcast_shared_t));
+
+  udp_register_dst_port (vm, UDP_DST_PORT_GTPU,
+			 gtpu4_input_node.index, /* is_ip4 */ 1);
+  udp_register_dst_port (vm, UDP_DST_PORT_GTPU6,
+			 gtpu6_input_node.index, /* is_ip4 */ 0);
+
+  gtm->fib_node_type = fib_node_register_new_type (>pu_vft);
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (gtpu_init);
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .description = "GTPv1-U",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/gtpu/gtpu.h b/src/plugins/gtpu/gtpu.h
new file mode 100644
index 00000000..86861974
--- /dev/null
+++ b/src/plugins/gtpu/gtpu.h
@@ -0,0 +1,262 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#ifndef included_vnet_gtpu_h
+#define included_vnet_gtpu_h
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/**
+ *		Bits
+ * Octets	8	7	6	5	4	3	2	1
+ * 1		          Version	PT	(*)	E	S	PN
+ * 2		Message Type
+ * 3		Length (1st Octet)
+ * 4		Length (2nd Octet)
+ * 5		Tunnel Endpoint Identifier (1st Octet)
+ * 6		Tunnel Endpoint Identifier (2nd Octet)
+ * 7		Tunnel Endpoint Identifier (3rd Octet)
+ * 8		Tunnel Endpoint Identifier (4th Octet)
+ * 9		Sequence Number (1st Octet)1) 4)
+ * 10		Sequence Number (2nd Octet)1) 4)
+ * 11		N-PDU Number2) 4)
+ * 12		Next Extension Header Type3) 4)
+**/
+
+typedef struct
+{
+  u8 ver_flags;
+  u8 type;
+  u16 length;			/* length in octets of the payload */
+  u32 teid;
+  u16 sequence;
+  u8 pdu_number;
+  u8 next_ext_type;
+} gtpu_header_t;
+
+#define GTPU_VER_MASK (7<<5)
+#define GTPU_PT_BIT   (1<<4)
+#define GTPU_E_BIT    (1<<2)
+#define GTPU_S_BIT    (1<<1)
+#define GTPU_PN_BIT   (1<<0)
+#define GTPU_E_S_PN_BIT  (7<<0)
+
+#define GTPU_V1_VER   (1<<5)
+
+#define GTPU_PT_GTP    (1<<4)
+#define GTPU_TYPE_GTPU  255
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED(struct
+{
+  ip4_header_t ip4;            /* 20 bytes */
+  udp_header_t udp;            /* 8 bytes */
+  gtpu_header_t gtpu;	       /* 8 bytes */
+}) ip4_gtpu_header_t;
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED(struct
+{
+  ip6_header_t ip6;            /* 40 bytes */
+  udp_header_t udp;            /* 8 bytes */
+  gtpu_header_t gtpu;     /* 8 bytes */
+}) ip6_gtpu_header_t;
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED
+(struct {
+  /*
+   * Key fields: ip src and gtpu teid on incoming gtpu packet
+   * all fields in NET byte order
+   */
+  union {
+    struct {
+      u32 src;
+      u32 teid;
+    };
+    u64 as_u64;
+  };
+}) gtpu4_tunnel_key_t;
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED
+(struct {
+  /*
+   * Key fields: ip src and gtpu teid on incoming gtpu packet
+   * all fields in NET byte order
+   */
+  ip6_address_t src;
+  u32 teid;
+}) gtpu6_tunnel_key_t;
+/* *INDENT-ON* */
+
+typedef struct
+{
+  /* Rewrite string */
+  u8 *rewrite;
+
+  /* FIB DPO for IP forwarding of gtpu encap packet */
+  dpo_id_t next_dpo;
+
+  /* gtpu teid in HOST byte order */
+  u32 teid;
+
+  /* tunnel src and dst addresses */
+  ip46_address_t src;
+  ip46_address_t dst;
+
+  /* mcast packet output intf index (used only if dst is mcast) */
+  u32 mcast_sw_if_index;
+
+  /* decap next index */
+  u32 decap_next_index;
+
+  /* The FIB index for src/dst addresses */
+  u32 encap_fib_index;
+
+  /* vnet intfc index */
+  u32 sw_if_index;
+  u32 hw_if_index;
+
+  /**
+   * Linkage into the FIB object graph
+   */
+  fib_node_t node;
+
+  /*
+   * The FIB entry for (depending on gtpu tunnel is unicast or mcast)
+   * sending unicast gtpu encap packets or receiving mcast gtpu packets
+   */
+  fib_node_index_t fib_entry_index;
+  adj_index_t mcast_adj_index;
+
+  /**
+   * The tunnel is a child of the FIB entry for its destination. This is
+   * so it receives updates when the forwarding information for that entry
+   * changes.
+   * The tunnels sibling index on the FIB entry's dependency list.
+   */
+  u32 sibling_index;
+} gtpu_tunnel_t;
+
+#define foreach_gtpu_input_next        \
+_(DROP, "error-drop")                  \
+_(L2_INPUT, "l2-input")
+
+typedef enum
+{
+#define _(s,n) GTPU_INPUT_NEXT_##s,
+  foreach_gtpu_input_next
+#undef _
+    GTPU_INPUT_N_NEXT,
+} gtpu_input_next_t;
+
+typedef enum
+{
+#define gtpu_error(n,s) GTPU_ERROR_##n,
+#include 
+#undef gtpu_error
+  GTPU_N_ERROR,
+} gtpu_input_error_t;
+
+typedef struct
+{
+  /* vector of encap tunnel instances */
+  gtpu_tunnel_t *tunnels;
+
+  /* lookup tunnel by key */
+  uword *gtpu4_tunnel_by_key;	/* keyed on ipv4.dst + teid */
+  uword *gtpu6_tunnel_by_key;	/* keyed on ipv6.dst + teid */
+
+  /* local VTEP IPs ref count used by gtpu-bypass node to check if
+     received gtpu packet DIP matches any local VTEP address */
+  uword *vtep4;			/* local ip4 VTEPs keyed on their ip4 addr */
+  uword *vtep6;			/* local ip6 VTEPs keyed on their ip6 addr */
+
+  /* mcast shared info */
+  uword *mcast_shared;		/* keyed on mcast ip46 addr */
+
+  /* Free vlib hw_if_indices */
+  u32 *free_gtpu_tunnel_hw_if_indices;
+
+  /* Mapping from sw_if_index to tunnel index */
+  u32 *tunnel_index_by_sw_if_index;
+
+  /**
+   * Node type for registering to fib changes.
+   */
+  fib_node_type_t fib_node_type;
+
+  /* API message ID base */
+  u16 msg_id_base;
+
+  /* convenience */
+  vlib_main_t *vlib_main;
+  vnet_main_t *vnet_main;
+} gtpu_main_t;
+
+gtpu_main_t gtpu_main;
+
+extern vlib_node_registration_t gtpu4_input_node;
+extern vlib_node_registration_t gtpu6_input_node;
+extern vlib_node_registration_t gtpu4_encap_node;
+extern vlib_node_registration_t gtpu6_encap_node;
+
+u8 *format_gtpu_encap_trace (u8 * s, va_list * args);
+
+typedef struct
+{
+  u8 is_add;
+  u8 is_ip6;
+  ip46_address_t src, dst;
+  u32 mcast_sw_if_index;
+  u32 encap_fib_index;
+  u32 decap_next_index;
+  u32 teid;
+} vnet_gtpu_add_del_tunnel_args_t;
+
+int vnet_gtpu_add_del_tunnel
+  (vnet_gtpu_add_del_tunnel_args_t * a, u32 * sw_if_indexp);
+
+void vnet_int_gtpu_bypass_mode (u32 sw_if_index, u8 is_ip6, u8 is_enable);
+#endif /* included_vnet_gtpu_h */
+
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/gtpu/gtpu_all_api_h.h b/src/plugins/gtpu/gtpu_all_api_h.h
new file mode 100644
index 00000000..a4181ae3
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_all_api_h.h
@@ -0,0 +1,18 @@
+/*
+ * gtpu_all_api_h.h - plug-in api #include file
+ *
+ * Copyright (c) Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include 
diff --git a/src/plugins/gtpu/gtpu_api.c b/src/plugins/gtpu/gtpu_api.c
new file mode 100644
index 00000000..58f48c71
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_api.c
@@ -0,0 +1,256 @@
+/*
+ *------------------------------------------------------------------
+ * gtpu_api.c - gtpu api
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include 
+  /* We'll want to know how many messages IDs we need... */
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#define vl_msg_name_crc_list
+#include 
+#undef vl_msg_name_crc_list
+
+#define REPLY_MSG_ID_BASE gtm->msg_id_base
+#include 
+
+static void
+setup_message_id_table (gtpu_main_t * gtm, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + gtm->msg_id_base);
+  foreach_vl_msg_name_crc_gtpu;
+#undef _
+}
+
+#define foreach_gtpu_plugin_api_msg                             \
+_(SW_INTERFACE_SET_GTPU_BYPASS, sw_interface_set_gtpu_bypass)         \
+_(GTPU_ADD_DEL_TUNNEL, gtpu_add_del_tunnel)                           \
+_(GTPU_TUNNEL_DUMP, gtpu_tunnel_dump)
+
+static void
+  vl_api_sw_interface_set_gtpu_bypass_t_handler
+  (vl_api_sw_interface_set_gtpu_bypass_t * mp)
+{
+  vl_api_sw_interface_set_gtpu_bypass_reply_t *rmp;
+  int rv = 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  gtpu_main_t *gtm = >pu_main;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  vnet_int_gtpu_bypass_mode (sw_if_index, mp->is_ipv6, mp->enable);
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_SET_GTPU_BYPASS_REPLY);
+}
+
+static void vl_api_gtpu_add_del_tunnel_t_handler
+  (vl_api_gtpu_add_del_tunnel_t * mp)
+{
+  vl_api_gtpu_add_del_tunnel_reply_t *rmp;
+  int rv = 0;
+  ip4_main_t *im = &ip4_main;
+  gtpu_main_t *gtm = >pu_main;
+
+  uword *p = hash_get (im->fib_index_by_table_id, ntohl (mp->encap_vrf_id));
+  if (!p)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_FIB;
+      goto out;
+    }
+
+  vnet_gtpu_add_del_tunnel_args_t a = {
+    .is_add = mp->is_add,
+    .is_ip6 = mp->is_ipv6,
+    .mcast_sw_if_index = ntohl (mp->mcast_sw_if_index),
+    .encap_fib_index = p[0],
+    .decap_next_index = ntohl (mp->decap_next_index),
+    .teid = ntohl (mp->teid),
+    .dst = to_ip46 (mp->is_ipv6, mp->dst_address),
+    .src = to_ip46 (mp->is_ipv6, mp->src_address),
+  };
+
+  /* Check src & dst are different */
+  if (ip46_address_cmp (&a.dst, &a.src) == 0)
+    {
+      rv = VNET_API_ERROR_SAME_SRC_DST;
+      goto out;
+    }
+  if (ip46_address_is_multicast (&a.dst) &&
+      !vnet_sw_if_index_is_api_valid (a.mcast_sw_if_index))
+    {
+      rv = VNET_API_ERROR_INVALID_SW_IF_INDEX;
+      goto out;
+    }
+
+  u32 sw_if_index = ~0;
+  rv = vnet_gtpu_add_del_tunnel (&a, &sw_if_index);
+
+out:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2(VL_API_GTPU_ADD_DEL_TUNNEL_REPLY,
+  ({
+    rmp->sw_if_index = ntohl (sw_if_index);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void send_gtpu_tunnel_details
+  (gtpu_tunnel_t * t, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_gtpu_tunnel_details_t *rmp;
+  ip4_main_t *im4 = &ip4_main;
+  ip6_main_t *im6 = &ip6_main;
+  u8 is_ipv6 = !ip46_address_is_ip4 (&t->dst);
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_GTPU_TUNNEL_DETAILS);
+  if (is_ipv6)
+    {
+      memcpy (rmp->src_address, t->src.ip6.as_u8, 16);
+      memcpy (rmp->dst_address, t->dst.ip6.as_u8, 16);
+      rmp->encap_vrf_id = htonl (im6->fibs[t->encap_fib_index].ft_table_id);
+    }
+  else
+    {
+      memcpy (rmp->src_address, t->src.ip4.as_u8, 4);
+      memcpy (rmp->dst_address, t->dst.ip4.as_u8, 4);
+      rmp->encap_vrf_id = htonl (im4->fibs[t->encap_fib_index].ft_table_id);
+    }
+  rmp->mcast_sw_if_index = htonl (t->mcast_sw_if_index);
+  rmp->teid = htonl (t->teid);
+  rmp->decap_next_index = htonl (t->decap_next_index);
+  rmp->sw_if_index = htonl (t->sw_if_index);
+  rmp->is_ipv6 = is_ipv6;
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_gtpu_tunnel_dump_t_handler (vl_api_gtpu_tunnel_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  gtpu_main_t *gtm = >pu_main;
+  gtpu_tunnel_t *t;
+  u32 sw_if_index;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    {
+      return;
+    }
+
+  sw_if_index = ntohl (mp->sw_if_index);
+
+  if (~0 == sw_if_index)
+    {
+      /* *INDENT-OFF* */
+      pool_foreach (t, gtm->tunnels,
+      ({
+        send_gtpu_tunnel_details(t, q, mp->context);
+      }));
+      /* *INDENT-ON* */
+    }
+  else
+    {
+      if ((sw_if_index >= vec_len (gtm->tunnel_index_by_sw_if_index)) ||
+	  (~0 == gtm->tunnel_index_by_sw_if_index[sw_if_index]))
+	{
+	  return;
+	}
+      t = >m->tunnels[gtm->tunnel_index_by_sw_if_index[sw_if_index]];
+      send_gtpu_tunnel_details (t, q, mp->context);
+    }
+}
+
+
+static clib_error_t *
+gtpu_api_hookup (vlib_main_t * vm)
+{
+  gtpu_main_t *gtm = >pu_main;
+
+  u8 *name = format (0, "gtpu_%08x%c", api_version, 0);
+  gtm->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + gtm->msg_id_base),     \
+                           #n,                  \
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_gtpu_plugin_api_msg;
+#undef _
+
+  /* Add our API messages to the global name_crc hash table */
+  setup_message_id_table (gtm, &api_main);
+
+  return 0;
+}
+
+VLIB_API_INIT_FUNCTION (gtpu_api_hookup);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c
new file mode 100644
index 00000000..b0be32ec
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_decap.c
@@ -0,0 +1,1305 @@
+/*
+ * decap.c: gtpu tunnel decap packet processing
+ *
+ * Copyright (c) 2013 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+
+vlib_node_registration_t gtpu4_input_node;
+vlib_node_registration_t gtpu6_input_node;
+
+typedef struct {
+  u32 next_index;
+  u32 tunnel_index;
+  u32 error;
+  u32 teid;
+} gtpu_rx_trace_t;
+
+static u8 * format_gtpu_rx_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  gtpu_rx_trace_t * t = va_arg (*args, gtpu_rx_trace_t *);
+
+  if (t->tunnel_index != ~0)
+    {
+      s = format (s, "GTPU decap from gtpu_tunnel%d teid %d next %d error %d",
+                  t->tunnel_index, t->teid, t->next_index, t->error);
+    }
+  else
+    {
+      s = format (s, "GTPU decap error - tunnel for teid %d does not exist",
+		  t->teid);
+    }
+  return s;
+}
+
+always_inline u32
+validate_gtpu_fib (vlib_buffer_t *b, gtpu_tunnel_t *t, u32 is_ip4)
+{
+  u32 fib_index, sw_if_index;
+
+  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
+
+  if (is_ip4)
+    fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
+	vec_elt (ip4_main.fib_index_by_sw_if_index, sw_if_index) :
+	vnet_buffer (b)->sw_if_index[VLIB_TX];
+  else
+    fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
+	vec_elt (ip6_main.fib_index_by_sw_if_index, sw_if_index) :
+	vnet_buffer (b)->sw_if_index[VLIB_TX];
+
+  return (fib_index == t->encap_fib_index);
+}
+
+always_inline uword
+gtpu_input (vlib_main_t * vm,
+             vlib_node_runtime_t * node,
+             vlib_frame_t * from_frame,
+             u32 is_ip4)
+{
+  u32 n_left_from, next_index, * from, * to_next;
+  gtpu_main_t * gtm = >pu_main;
+  vnet_main_t * vnm = gtm->vnet_main;
+  vnet_interface_main_t * im = &vnm->interface_main;
+  u32 last_tunnel_index = ~0;
+  gtpu4_tunnel_key_t last_key4;
+  gtpu6_tunnel_key_t last_key6;
+  u32 pkts_decapsulated = 0;
+  u32 thread_index = vlib_get_thread_index();
+  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
+
+  if (is_ip4)
+    last_key4.as_u64 = ~0;
+  else
+    memset (&last_key6, 0xff, sizeof (last_key6));
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+  stats_sw_if_index = node->runtime_data[0];
+  stats_n_packets = stats_n_bytes = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+	  u32 next0, next1;
+          ip4_header_t * ip4_0, * ip4_1;
+          ip6_header_t * ip6_0, * ip6_1;
+          gtpu_header_t * gtpu0, * gtpu1;
+          u32 gtpu_hdr_len0 = 0, gtpu_hdr_len1 =0 ;
+	  uword * p0, * p1;
+          u32 tunnel_index0, tunnel_index1;
+          gtpu_tunnel_t * t0, * t1, * mt0 = NULL, * mt1 = NULL;
+          gtpu4_tunnel_key_t key4_0, key4_1;
+          gtpu6_tunnel_key_t key6_0, key6_1;
+          u32 error0, error1;
+	  u32 sw_if_index0, sw_if_index1, len0, len1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	    CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	  }
+
+	  bi0 = from[0];
+	  bi1 = from[1];
+	  to_next[0] = bi0;
+	  to_next[1] = bi1;
+	  from += 2;
+	  to_next += 2;
+	  n_left_to_next -= 2;
+	  n_left_from -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          /* udp leaves current_data pointing at the gtpu header */
+          gtpu0 = vlib_buffer_get_current (b0);
+          gtpu1 = vlib_buffer_get_current (b1);
+          if (is_ip4) {
+	    vlib_buffer_advance
+	      (b0, -(word)(sizeof(udp_header_t)+sizeof(ip4_header_t)));
+	    vlib_buffer_advance
+	      (b1, -(word)(sizeof(udp_header_t)+sizeof(ip4_header_t)));
+            ip4_0 = vlib_buffer_get_current (b0);
+            ip4_1 = vlib_buffer_get_current (b1);
+	  } else {
+            vlib_buffer_advance
+              (b0, -(word)(sizeof(udp_header_t)+sizeof(ip6_header_t)));
+            vlib_buffer_advance
+              (b1, -(word)(sizeof(udp_header_t)+sizeof(ip6_header_t)));
+            ip6_0 = vlib_buffer_get_current (b0);
+            ip6_1 = vlib_buffer_get_current (b1);
+	  }
+
+          /* pop (ip, udp, gtpu) */
+          if (is_ip4) {
+            vlib_buffer_advance
+              (b0, sizeof(*ip4_0)+sizeof(udp_header_t));
+	    vlib_buffer_advance
+              (b1, sizeof(*ip4_1)+sizeof(udp_header_t));
+          } else {
+	    vlib_buffer_advance
+              (b0, sizeof(*ip6_0)+sizeof(udp_header_t));
+            vlib_buffer_advance
+              (b1, sizeof(*ip6_1)+sizeof(udp_header_t));
+          }
+
+          tunnel_index0 = ~0;
+          error0 = 0;
+
+          tunnel_index1 = ~0;
+          error1 = 0;
+
+	  if (PREDICT_FALSE ((gtpu0->ver_flags & GTPU_VER_MASK) != GTPU_V1_VER))
+	    {
+	      error0 = GTPU_ERROR_BAD_VER;
+	      next0 = GTPU_INPUT_NEXT_DROP;
+	      goto trace0;
+	    }
+
+	  /* Manipulate packet 0 */
+          if (is_ip4) {
+            key4_0.src = ip4_0->src_address.as_u32;
+            key4_0.teid = gtpu0->teid;
+
+ 	    /* Make sure GTPU tunnel exist according to packet SIP and teid
+ 	     * SIP identify a GTPU path, and teid identify a tunnel in a given GTPU path */
+           if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
+              {
+                p0 = hash_get (gtm->gtpu4_tunnel_by_key, key4_0.as_u64);
+                if (PREDICT_FALSE (p0 == NULL))
+                  {
+                    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+                    next0 = GTPU_INPUT_NEXT_DROP;
+                    goto trace0;
+                  }
+                last_key4.as_u64 = key4_0.as_u64;
+                tunnel_index0 = last_tunnel_index = p0[0];
+              }
+            else
+              tunnel_index0 = last_tunnel_index;
+	    t0 = pool_elt_at_index (gtm->tunnels, tunnel_index0);
+
+	    /* Validate GTPU tunnel encap-fib index agaist packet */
+	    if (PREDICT_FALSE (validate_gtpu_fib (b0, t0, is_ip4) == 0))
+	      {
+		error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+		next0 = GTPU_INPUT_NEXT_DROP;
+		goto trace0;
+	      }
+
+	    /* Validate GTPU tunnel SIP against packet DIP */
+	    if (PREDICT_TRUE (ip4_0->dst_address.as_u32 == t0->src.ip4.as_u32))
+	      goto next0; /* valid packet */
+	    if (PREDICT_FALSE (ip4_address_is_multicast (&ip4_0->dst_address)))
+	      {
+		key4_0.src = ip4_0->dst_address.as_u32;
+		key4_0.teid = gtpu0->teid;
+		/* Make sure mcast GTPU tunnel exist by packet DIP and teid */
+		p0 = hash_get (gtm->gtpu4_tunnel_by_key, key4_0.as_u64);
+		if (PREDICT_TRUE (p0 != NULL))
+		  {
+		    mt0 = pool_elt_at_index (gtm->tunnels, p0[0]);
+		    goto next0; /* valid packet */
+		  }
+	      }
+	    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+	    next0 = GTPU_INPUT_NEXT_DROP;
+	    goto trace0;
+
+         } else /* !is_ip4 */ {
+            key6_0.src.as_u64[0] = ip6_0->src_address.as_u64[0];
+            key6_0.src.as_u64[1] = ip6_0->src_address.as_u64[1];
+            key6_0.teid = gtpu0->teid;
+
+ 	    /* Make sure GTPU tunnel exist according to packet SIP and teid
+ 	     * SIP identify a GTPU path, and teid identify a tunnel in a given GTPU path */
+            if (PREDICT_FALSE (memcmp(&key6_0, &last_key6, sizeof(last_key6)) != 0))
+              {
+                p0 = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6_0);
+                if (PREDICT_FALSE (p0 == NULL))
+                  {
+                    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+                    next0 = GTPU_INPUT_NEXT_DROP;
+                    goto trace0;
+                  }
+                clib_memcpy (&last_key6, &key6_0, sizeof(key6_0));
+                tunnel_index0 = last_tunnel_index = p0[0];
+              }
+            else
+              tunnel_index0 = last_tunnel_index;
+	    t0 = pool_elt_at_index (gtm->tunnels, tunnel_index0);
+
+	    /* Validate GTPU tunnel encap-fib index agaist packet */
+	    if (PREDICT_FALSE (validate_gtpu_fib (b0, t0, is_ip4) == 0))
+	      {
+		error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+		next0 = GTPU_INPUT_NEXT_DROP;
+		goto trace0;
+	      }
+
+	    /* Validate GTPU tunnel SIP against packet DIP */
+	    if (PREDICT_TRUE (ip6_address_is_equal (&ip6_0->dst_address,
+						    &t0->src.ip6)))
+		goto next0; /* valid packet */
+	    if (PREDICT_FALSE (ip6_address_is_multicast (&ip6_0->dst_address)))
+	      {
+		key6_0.src.as_u64[0] = ip6_0->dst_address.as_u64[0];
+		key6_0.src.as_u64[1] = ip6_0->dst_address.as_u64[1];
+		key6_0.teid = gtpu0->teid;
+		p0 = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6_0);
+		if (PREDICT_TRUE (p0 != NULL))
+		  {
+		    mt0 = pool_elt_at_index (gtm->tunnels, p0[0]);
+		    goto next0; /* valid packet */
+		  }
+	      }
+	    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+	    next0 = GTPU_INPUT_NEXT_DROP;
+	    goto trace0;
+          }
+
+	next0:
+	  /* Manipulate gtpu header */
+	  if (PREDICT_FALSE((gtpu0->ver_flags & GTPU_E_S_PN_BIT) != 0))
+	    {
+	      gtpu_hdr_len0 = sizeof(gtpu_header_t);
+
+	      /* Manipulate Sequence Number and N-PDU Number */
+	      /* TBD */
+
+	      /* Manipulate Next Extension Header */
+	      /* TBD */
+	    }
+	  else
+	    {
+	      gtpu_hdr_len0 = sizeof(gtpu_header_t) - 4;
+	    }
+
+	  /* Pop gtpu header */
+	  vlib_buffer_advance (b0, gtpu_hdr_len0);
+
+          next0 = t0->decap_next_index;
+          sw_if_index0 = t0->sw_if_index;
+          len0 = vlib_buffer_length_in_chain (vm, b0);
+
+          /* Required to make the l2 tag push / pop code work on l2 subifs */
+          if (PREDICT_TRUE(next0 == GTPU_INPUT_NEXT_L2_INPUT))
+            vnet_update_l2_len (b0);
+
+          /* Set packet input sw_if_index to unicast GTPU tunnel for learning */
+          vnet_buffer(b0)->sw_if_index[VLIB_RX] = sw_if_index0;
+	  sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same gtpu tunnel so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = sw_if_index0;
+	    }
+
+        trace0:
+          b0->error = error0 ? node->errors[error0] : 0;
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              gtpu_rx_trace_t *tr
+                = vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->next_index = next0;
+              tr->error = error0;
+              tr->tunnel_index = tunnel_index0;
+              tr->teid = clib_net_to_host_u32(gtpu0->teid);
+            }
+
+          if (PREDICT_FALSE ((gtpu1->ver_flags & GTPU_VER_MASK) != GTPU_V1_VER))
+	    {
+	      error1 = GTPU_ERROR_BAD_VER;
+	      next1 = GTPU_INPUT_NEXT_DROP;
+	      goto trace1;
+	    }
+
+          /* Manipulate packet 1 */
+          if (is_ip4) {
+            key4_1.src = ip4_1->src_address.as_u32;
+            key4_1.teid = gtpu1->teid;
+
+ 	    /* Make sure GTPU tunnel exist according to packet SIP and teid
+ 	     * SIP identify a GTPU path, and teid identify a tunnel in a given GTPU path */
+	    if (PREDICT_FALSE (key4_1.as_u64 != last_key4.as_u64))
+              {
+                p1 = hash_get (gtm->gtpu4_tunnel_by_key, key4_1.as_u64);
+                if (PREDICT_FALSE (p1 == NULL))
+                  {
+                    error1 = GTPU_ERROR_NO_SUCH_TUNNEL;
+                    next1 = GTPU_INPUT_NEXT_DROP;
+                    goto trace1;
+                  }
+                last_key4.as_u64 = key4_1.as_u64;
+                tunnel_index1 = last_tunnel_index = p1[0];
+              }
+            else
+              tunnel_index1 = last_tunnel_index;
+ 	    t1 = pool_elt_at_index (gtm->tunnels, tunnel_index1);
+
+	    /* Validate GTPU tunnel encap-fib index agaist packet */
+	    if (PREDICT_FALSE (validate_gtpu_fib (b1, t1, is_ip4) == 0))
+	      {
+		error1 = GTPU_ERROR_NO_SUCH_TUNNEL;
+		next1 = GTPU_INPUT_NEXT_DROP;
+		goto trace1;
+	      }
+
+	    /* Validate GTPU tunnel SIP against packet DIP */
+	    if (PREDICT_TRUE (ip4_1->dst_address.as_u32 == t1->src.ip4.as_u32))
+	      goto next1; /* valid packet */
+	    if (PREDICT_FALSE (ip4_address_is_multicast (&ip4_1->dst_address)))
+	      {
+		key4_1.src = ip4_1->dst_address.as_u32;
+		key4_1.teid = gtpu1->teid;
+		/* Make sure mcast GTPU tunnel exist by packet DIP and teid */
+		p1 = hash_get (gtm->gtpu4_tunnel_by_key, key4_1.as_u64);
+		if (PREDICT_TRUE (p1 != NULL))
+		  {
+		    mt1 = pool_elt_at_index (gtm->tunnels, p1[0]);
+		    goto next1; /* valid packet */
+		  }
+	      }
+	    error1 = GTPU_ERROR_NO_SUCH_TUNNEL;
+	    next1 = GTPU_INPUT_NEXT_DROP;
+	    goto trace1;
+
+         } else /* !is_ip4 */ {
+            key6_1.src.as_u64[0] = ip6_1->src_address.as_u64[0];
+            key6_1.src.as_u64[1] = ip6_1->src_address.as_u64[1];
+            key6_1.teid = gtpu1->teid;
+
+ 	    /* Make sure GTPU tunnel exist according to packet SIP and teid
+ 	     * SIP identify a GTPU path, and teid identify a tunnel in a given GTPU path */
+            if (PREDICT_FALSE (memcmp(&key6_1, &last_key6, sizeof(last_key6)) != 0))
+              {
+                p1 = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6_1);
+
+                if (PREDICT_FALSE (p1 == NULL))
+                  {
+                    error1 = GTPU_ERROR_NO_SUCH_TUNNEL;
+                    next1 = GTPU_INPUT_NEXT_DROP;
+                    goto trace1;
+                  }
+
+                clib_memcpy (&last_key6, &key6_1, sizeof(key6_1));
+                tunnel_index1 = last_tunnel_index = p1[0];
+              }
+            else
+              tunnel_index1 = last_tunnel_index;
+ 	    t1 = pool_elt_at_index (gtm->tunnels, tunnel_index1);
+
+	    /* Validate GTPU tunnel encap-fib index agaist packet */
+	    if (PREDICT_FALSE (validate_gtpu_fib (b1, t1, is_ip4) == 0))
+	      {
+		error1 = GTPU_ERROR_NO_SUCH_TUNNEL;
+		next1 = GTPU_INPUT_NEXT_DROP;
+		goto trace1;
+	      }
+
+	    /* Validate GTPU tunnel SIP against packet DIP */
+	    if (PREDICT_TRUE (ip6_address_is_equal (&ip6_1->dst_address,
+						    &t1->src.ip6)))
+		goto next1; /* valid packet */
+	    if (PREDICT_FALSE (ip6_address_is_multicast (&ip6_1->dst_address)))
+	      {
+		key6_1.src.as_u64[0] = ip6_1->dst_address.as_u64[0];
+		key6_1.src.as_u64[1] = ip6_1->dst_address.as_u64[1];
+		key6_1.teid = gtpu1->teid;
+		p1 = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6_1);
+		if (PREDICT_TRUE (p1 != NULL))
+		  {
+		    mt1 = pool_elt_at_index (gtm->tunnels, p1[0]);
+		    goto next1; /* valid packet */
+		  }
+	      }
+	    error1 = GTPU_ERROR_NO_SUCH_TUNNEL;
+	    next1 = GTPU_INPUT_NEXT_DROP;
+	    goto trace1;
+	  }
+
+	next1:
+	  /* Manipulate gtpu header */
+	  if (PREDICT_FALSE((gtpu1->ver_flags & GTPU_E_S_PN_BIT) != 0))
+	    {
+	      gtpu_hdr_len1 = sizeof(gtpu_header_t);
+
+	      /* Manipulate Sequence Number and N-PDU Number */
+	      /* TBD */
+
+	      /* Manipulate Next Extension Header */
+	      /* TBD */
+	    }
+	  else
+	    {
+	      gtpu_hdr_len1 = sizeof(gtpu_header_t) - 4;
+	    }
+
+	  /* Pop gtpu header */
+	  vlib_buffer_advance (b1, gtpu_hdr_len1);
+
+          next1 = t1->decap_next_index;
+          sw_if_index1 = t1->sw_if_index;
+          len1 = vlib_buffer_length_in_chain (vm, b1);
+
+          /* Required to make the l2 tag push / pop code work on l2 subifs */
+          if (PREDICT_TRUE(next1 == GTPU_INPUT_NEXT_L2_INPUT))
+            vnet_update_l2_len (b1);
+
+          /* Set packet input sw_if_index to unicast GTPU tunnel for learning */
+          vnet_buffer(b1)->sw_if_index[VLIB_RX] = sw_if_index1;
+	  sw_if_index1 = (mt1) ? mt1->sw_if_index : sw_if_index1;
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len1;
+
+	  /* Batch stats increment on the same gtpu tunnel so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (sw_if_index1 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len1;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len1;
+	      stats_sw_if_index = sw_if_index1;
+	    }
+
+        trace1:
+          b1->error = error1 ? node->errors[error1] : 0;
+
+          if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              gtpu_rx_trace_t *tr
+                = vlib_add_trace (vm, node, b1, sizeof (*tr));
+              tr->next_index = next1;
+              tr->error = error1;
+              tr->tunnel_index = tunnel_index1;
+              tr->teid = clib_net_to_host_u32(gtpu1->teid);
+            }
+
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t * b0;
+	  u32 next0;
+          ip4_header_t * ip4_0;
+          ip6_header_t * ip6_0;
+          gtpu_header_t * gtpu0;
+          u32 gtpu_hdr_len0 = 0;
+	  uword * p0;
+          u32 tunnel_index0;
+          gtpu_tunnel_t * t0, * mt0 = NULL;
+          gtpu4_tunnel_key_t key4_0;
+          gtpu6_tunnel_key_t key6_0;
+          u32 error0;
+	  u32 sw_if_index0, len0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+          /* udp leaves current_data pointing at the gtpu header */
+          gtpu0 = vlib_buffer_get_current (b0);
+          if (is_ip4) {
+	    vlib_buffer_advance
+	      (b0, -(word)(sizeof(udp_header_t)+sizeof(ip4_header_t)));
+            ip4_0 = vlib_buffer_get_current (b0);
+          } else {
+            vlib_buffer_advance
+              (b0, -(word)(sizeof(udp_header_t)+sizeof(ip6_header_t)));
+            ip6_0 = vlib_buffer_get_current (b0);
+          }
+
+          /* pop (ip, udp) */
+          if (is_ip4) {
+            vlib_buffer_advance
+              (b0, sizeof(*ip4_0)+sizeof(udp_header_t));
+          } else {
+	    vlib_buffer_advance
+              (b0, sizeof(*ip6_0)+sizeof(udp_header_t));
+          }
+
+          tunnel_index0 = ~0;
+          error0 = 0;
+          if (PREDICT_FALSE ((gtpu0->ver_flags & GTPU_VER_MASK) != GTPU_V1_VER))
+	    {
+	      error0 = GTPU_ERROR_BAD_VER;
+	      next0 = GTPU_INPUT_NEXT_DROP;
+	      goto trace00;
+	    }
+
+          if (is_ip4) {
+            key4_0.src = ip4_0->src_address.as_u32;
+            key4_0.teid = gtpu0->teid;
+
+ 	    /* Make sure GTPU tunnel exist according to packet SIP and teid
+ 	     * SIP identify a GTPU path, and teid identify a tunnel in a given GTPU path */
+            if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
+              {
+                p0 = hash_get (gtm->gtpu4_tunnel_by_key, key4_0.as_u64);
+                if (PREDICT_FALSE (p0 == NULL))
+                  {
+                    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+                    next0 = GTPU_INPUT_NEXT_DROP;
+                    goto trace00;
+                  }
+                last_key4.as_u64 = key4_0.as_u64;
+                tunnel_index0 = last_tunnel_index = p0[0];
+              }
+            else
+              tunnel_index0 = last_tunnel_index;
+	    t0 = pool_elt_at_index (gtm->tunnels, tunnel_index0);
+
+	    /* Validate GTPU tunnel encap-fib index agaist packet */
+	    if (PREDICT_FALSE (validate_gtpu_fib (b0, t0, is_ip4) == 0))
+	      {
+		error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+		next0 = GTPU_INPUT_NEXT_DROP;
+		goto trace00;
+	      }
+
+	    /* Validate GTPU tunnel SIP against packet DIP */
+	    if (PREDICT_TRUE (ip4_0->dst_address.as_u32 == t0->src.ip4.as_u32))
+	      goto next00; /* valid packet */
+	    if (PREDICT_FALSE (ip4_address_is_multicast (&ip4_0->dst_address)))
+	      {
+		key4_0.src = ip4_0->dst_address.as_u32;
+		key4_0.teid = gtpu0->teid;
+		/* Make sure mcast GTPU tunnel exist by packet DIP and teid */
+		p0 = hash_get (gtm->gtpu4_tunnel_by_key, key4_0.as_u64);
+		if (PREDICT_TRUE (p0 != NULL))
+		  {
+		    mt0 = pool_elt_at_index (gtm->tunnels, p0[0]);
+		    goto next00; /* valid packet */
+		  }
+	      }
+	    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+	    next0 = GTPU_INPUT_NEXT_DROP;
+	    goto trace00;
+
+          } else /* !is_ip4 */ {
+            key6_0.src.as_u64[0] = ip6_0->src_address.as_u64[0];
+            key6_0.src.as_u64[1] = ip6_0->src_address.as_u64[1];
+            key6_0.teid = gtpu0->teid;
+
+ 	    /* Make sure GTPU tunnel exist according to packet SIP and teid
+ 	     * SIP identify a GTPU path, and teid identify a tunnel in a given GTPU path */
+            if (PREDICT_FALSE (memcmp(&key6_0, &last_key6, sizeof(last_key6)) != 0))
+              {
+                p0 = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6_0);
+                if (PREDICT_FALSE (p0 == NULL))
+                  {
+                    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+                    next0 = GTPU_INPUT_NEXT_DROP;
+                    goto trace00;
+                  }
+                clib_memcpy (&last_key6, &key6_0, sizeof(key6_0));
+                tunnel_index0 = last_tunnel_index = p0[0];
+              }
+            else
+              tunnel_index0 = last_tunnel_index;
+	    t0 = pool_elt_at_index (gtm->tunnels, tunnel_index0);
+
+	    /* Validate GTPU tunnel encap-fib index agaist packet */
+	    if (PREDICT_FALSE (validate_gtpu_fib (b0, t0, is_ip4) == 0))
+	      {
+		error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+		next0 = GTPU_INPUT_NEXT_DROP;
+		goto trace00;
+	      }
+
+	    /* Validate GTPU tunnel SIP against packet DIP */
+	    if (PREDICT_TRUE (ip6_address_is_equal (&ip6_0->dst_address,
+						    &t0->src.ip6)))
+		goto next00; /* valid packet */
+	    if (PREDICT_FALSE (ip6_address_is_multicast (&ip6_0->dst_address)))
+	      {
+		key6_0.src.as_u64[0] = ip6_0->dst_address.as_u64[0];
+		key6_0.src.as_u64[1] = ip6_0->dst_address.as_u64[1];
+		key6_0.teid = gtpu0->teid;
+		p0 = hash_get_mem (gtm->gtpu6_tunnel_by_key, &key6_0);
+		if (PREDICT_TRUE (p0 != NULL))
+		  {
+		    mt0 = pool_elt_at_index (gtm->tunnels, p0[0]);
+		    goto next00; /* valid packet */
+		  }
+	      }
+	    error0 = GTPU_ERROR_NO_SUCH_TUNNEL;
+	    next0 = GTPU_INPUT_NEXT_DROP;
+	    goto trace00;
+          }
+
+	next00:
+	  /* Manipulate gtpu header */
+	  if (PREDICT_FALSE((gtpu0->ver_flags & GTPU_E_S_PN_BIT) != 0))
+	    {
+	      gtpu_hdr_len0 = sizeof(gtpu_header_t);
+
+	      /* Manipulate Sequence Number and N-PDU Number */
+	      /* TBD */
+
+	      /* Manipulate Next Extension Header */
+	      /* TBD */
+	    }
+	  else
+	    {
+	      gtpu_hdr_len0 = sizeof(gtpu_header_t) - 4;
+	    }
+
+	  /* Pop gtpu header */
+	  vlib_buffer_advance (b0, gtpu_hdr_len0);
+
+	  next0 = t0->decap_next_index;
+	  sw_if_index0 = t0->sw_if_index;
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+
+          /* Required to make the l2 tag push / pop code work on l2 subifs */
+          if (PREDICT_TRUE(next0 == GTPU_INPUT_NEXT_L2_INPUT))
+            vnet_update_l2_len (b0);
+
+          /* Set packet input sw_if_index to unicast GTPU tunnel for learning */
+          vnet_buffer(b0)->sw_if_index[VLIB_RX] = sw_if_index0;
+	  sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same gtpu tunnel so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = sw_if_index0;
+	    }
+
+        trace00:
+          b0->error = error0 ? node->errors[error0] : 0;
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              gtpu_rx_trace_t *tr
+                = vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->next_index = next0;
+              tr->error = error0;
+              tr->tunnel_index = tunnel_index0;
+              tr->teid = clib_net_to_host_u32(gtpu0->teid);
+            }
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  /* Do we still need this now that tunnel tx stats is kept? */
+  vlib_node_increment_counter (vm, is_ip4?
+			       gtpu4_input_node.index:gtpu6_input_node.index,
+                               GTPU_ERROR_DECAPSULATED,
+                               pkts_decapsulated);
+
+  /* Increment any remaining batch stats */
+  if (stats_n_packets)
+    {
+      vlib_increment_combined_counter
+	(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+	 thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
+      node->runtime_data[0] = stats_sw_if_index;
+    }
+
+  return from_frame->n_vectors;
+}
+
+static uword
+gtpu4_input (vlib_main_t * vm,
+             vlib_node_runtime_t * node,
+             vlib_frame_t * from_frame)
+{
+	return gtpu_input(vm, node, from_frame, /* is_ip4 */ 1);
+}
+
+static uword
+gtpu6_input (vlib_main_t * vm,
+             vlib_node_runtime_t * node,
+             vlib_frame_t * from_frame)
+{
+	return gtpu_input(vm, node, from_frame, /* is_ip4 */ 0);
+}
+
+static char * gtpu_error_strings[] = {
+#define gtpu_error(n,s) s,
+#include 
+#undef gtpu_error
+#undef _
+};
+
+VLIB_REGISTER_NODE (gtpu4_input_node) = {
+  .function = gtpu4_input,
+  .name = "gtpu4-input",
+  /* Takes a vector of packets. */
+  .vector_size = sizeof (u32),
+
+  .n_errors = GTPU_N_ERROR,
+  .error_strings = gtpu_error_strings,
+
+  .n_next_nodes = GTPU_INPUT_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [GTPU_INPUT_NEXT_##s] = n,
+    foreach_gtpu_input_next
+#undef _
+  },
+
+//temp  .format_buffer = format_gtpu_header,
+  .format_trace = format_gtpu_rx_trace,
+  // $$$$ .unformat_buffer = unformat_gtpu_header,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (gtpu4_input_node, gtpu4_input)
+
+VLIB_REGISTER_NODE (gtpu6_input_node) = {
+  .function = gtpu6_input,
+  .name = "gtpu6-input",
+  /* Takes a vector of packets. */
+  .vector_size = sizeof (u32),
+
+  .n_errors = GTPU_N_ERROR,
+  .error_strings = gtpu_error_strings,
+
+  .n_next_nodes = GTPU_INPUT_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [GTPU_INPUT_NEXT_##s] = n,
+    foreach_gtpu_input_next
+#undef _
+  },
+
+//temp  .format_buffer = format_gtpu_header,
+  .format_trace = format_gtpu_rx_trace,
+  // $$$$ .unformat_buffer = unformat_gtpu_header,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (gtpu6_input_node, gtpu6_input)
+
+
+typedef enum {
+  IP_GTPU_BYPASS_NEXT_DROP,
+  IP_GTPU_BYPASS_NEXT_GTPU,
+  IP_GTPU_BYPASS_N_NEXT,
+} ip_vxan_bypass_next_t;
+
+always_inline uword
+ip_gtpu_bypass_inline (vlib_main_t * vm,
+			vlib_node_runtime_t * node,
+			vlib_frame_t * frame,
+			u32 is_ip4)
+{
+  gtpu_main_t * gtm = >pu_main;
+  u32 * from, * to_next, n_left_from, n_left_to_next, next_index;
+  vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
+  ip4_address_t addr4; /* last IPv4 address matching a local VTEP address */
+  ip6_address_t addr6; /* last IPv6 address matching a local VTEP address */
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  if (node->flags & VLIB_NODE_FLAG_TRACE)
+    ip4_forward_next_trace (vm, node, frame, VLIB_TX);
+
+  if (is_ip4) addr4.data_u32 = ~0;
+  else ip6_address_set_zero (&addr6);
+
+  while (n_left_from > 0)
+    {
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+      	{
+      	  vlib_buffer_t * b0, * b1;
+      	  ip4_header_t * ip40, * ip41;
+      	  ip6_header_t * ip60, * ip61;
+      	  udp_header_t * udp0, * udp1;
+      	  u32 bi0, ip_len0, udp_len0, flags0, next0;
+      	  u32 bi1, ip_len1, udp_len1, flags1, next1;
+      	  i32 len_diff0, len_diff1;
+      	  u8 error0, good_udp0, proto0;
+      	  u8 error1, good_udp1, proto1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	    CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	  }
+
+      	  bi0 = to_next[0] = from[0];
+      	  bi1 = to_next[1] = from[1];
+      	  from += 2;
+      	  n_left_from -= 2;
+      	  to_next += 2;
+      	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+	  if (is_ip4)
+	    {
+	      ip40 = vlib_buffer_get_current (b0);
+	      ip41 = vlib_buffer_get_current (b1);
+	    }
+	  else
+	    {
+	      ip60 = vlib_buffer_get_current (b0);
+	      ip61 = vlib_buffer_get_current (b1);
+	    }
+
+	  /* Setup packet for next IP feature */
+	  vnet_feature_next(vnet_buffer(b0)->sw_if_index[VLIB_RX], &next0, b0);
+	  vnet_feature_next(vnet_buffer(b1)->sw_if_index[VLIB_RX], &next1, b1);
+
+	  if (is_ip4)
+	    {
+	      /* Treat IP frag packets as "experimental" protocol for now
+		 until support of IP frag reassembly is implemented */
+	      proto0 = ip4_is_fragment(ip40) ? 0xfe : ip40->protocol;
+	      proto1 = ip4_is_fragment(ip41) ? 0xfe : ip41->protocol;
+	    }
+	  else
+	    {
+	      proto0 = ip60->protocol;
+	      proto1 = ip61->protocol;
+	    }
+
+	  /* Process packet 0 */
+	  if (proto0 != IP_PROTOCOL_UDP)
+	    goto exit0; /* not UDP packet */
+
+	  if (is_ip4)
+	    udp0 = ip4_next_header (ip40);
+	  else
+	    udp0 = ip6_next_header (ip60);
+
+	  if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_GTPU))
+	    goto exit0; /* not GTPU packet */
+
+	  /* Validate DIP against VTEPs*/
+	  if (is_ip4)
+	    {
+	      if (addr4.as_u32 != ip40->dst_address.as_u32)
+	        {
+		  if (!hash_get (gtm->vtep4, ip40->dst_address.as_u32))
+		      goto exit0; /* no local VTEP for GTPU packet */
+		  addr4 = ip40->dst_address;
+	        }
+	    }
+	  else
+	    {
+	      if (!ip6_address_is_equal (&addr6, &ip60->dst_address))
+	        {
+		  if (!hash_get_mem (gtm->vtep6, &ip60->dst_address))
+		      goto exit0; /* no local VTEP for GTPU packet */
+		  addr6 = ip60->dst_address;
+	        }
+	    }
+
+	  flags0 = b0->flags;
+	  good_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
+
+	  /* Don't verify UDP checksum for packets with explicit zero checksum. */
+	  good_udp0 |= udp0->checksum == 0;
+
+	  /* Verify UDP length */
+	  if (is_ip4)
+	    ip_len0 = clib_net_to_host_u16 (ip40->length);
+	  else
+	    ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
+	  udp_len0 = clib_net_to_host_u16 (udp0->length);
+	  len_diff0 = ip_len0 - udp_len0;
+
+	  /* Verify UDP checksum */
+	  if (PREDICT_FALSE (!good_udp0))
+	    {
+	      if ((flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED) == 0)
+	        {
+		  if (is_ip4)
+		    flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
+		  else
+		    flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
+		  good_udp0 =
+		    (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
+	        }
+	    }
+
+	  if (is_ip4)
+	    {
+	      error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
+	      error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
+	    }
+	  else
+	    {
+	      error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
+	      error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
+	    }
+
+	  next0 = error0 ?
+	    IP_GTPU_BYPASS_NEXT_DROP : IP_GTPU_BYPASS_NEXT_GTPU;
+	  b0->error = error0 ? error_node->errors[error0] : 0;
+
+	  /* gtpu-input node expect current at GTPU header */
+	  if (is_ip4)
+	    vlib_buffer_advance (b0, sizeof(ip4_header_t)+sizeof(udp_header_t));
+	  else
+	    vlib_buffer_advance (b0, sizeof(ip6_header_t)+sizeof(udp_header_t));
+
+	exit0:
+	  /* Process packet 1 */
+	  if (proto1 != IP_PROTOCOL_UDP)
+	    goto exit1; /* not UDP packet */
+
+	  if (is_ip4)
+	    udp1 = ip4_next_header (ip41);
+	  else
+	    udp1 = ip6_next_header (ip61);
+
+	  if (udp1->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_GTPU))
+	    goto exit1; /* not GTPU packet */
+
+	  /* Validate DIP against VTEPs*/
+	  if (is_ip4)
+	    {
+	      if (addr4.as_u32 != ip41->dst_address.as_u32)
+	        {
+		  if (!hash_get (gtm->vtep4, ip41->dst_address.as_u32))
+		      goto exit1; /* no local VTEP for GTPU packet */
+		  addr4 = ip41->dst_address;
+		}
+	    }
+	  else
+	    {
+	      if (!ip6_address_is_equal (&addr6, &ip61->dst_address))
+	        {
+		  if (!hash_get_mem (gtm->vtep6, &ip61->dst_address))
+		      goto exit1; /* no local VTEP for GTPU packet */
+		  addr6 = ip61->dst_address;
+		}
+	    }
+
+	  flags1 = b1->flags;
+	  good_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
+
+	  /* Don't verify UDP checksum for packets with explicit zero checksum. */
+	  good_udp1 |= udp1->checksum == 0;
+
+	  /* Verify UDP length */
+	  if (is_ip4)
+	    ip_len1 = clib_net_to_host_u16 (ip41->length);
+	  else
+	    ip_len1 = clib_net_to_host_u16 (ip61->payload_length);
+	  udp_len1 = clib_net_to_host_u16 (udp1->length);
+	  len_diff1 = ip_len1 - udp_len1;
+
+	  /* Verify UDP checksum */
+	  if (PREDICT_FALSE (!good_udp1))
+	    {
+	      if ((flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED) == 0)
+	        {
+		  if (is_ip4)
+		    flags1 = ip4_tcp_udp_validate_checksum (vm, b1);
+		  else
+		    flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, b1);
+		  good_udp1 =
+		    (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
+	        }
+	    }
+
+	  if (is_ip4)
+	    {
+	      error1 = good_udp1 ? 0 : IP4_ERROR_UDP_CHECKSUM;
+	      error1 = (len_diff1 >= 0) ? error1 : IP4_ERROR_UDP_LENGTH;
+	    }
+	  else
+	    {
+	      error1 = good_udp1 ? 0 : IP6_ERROR_UDP_CHECKSUM;
+	      error1 = (len_diff1 >= 0) ? error1 : IP6_ERROR_UDP_LENGTH;
+	    }
+
+	  next1 = error1 ?
+	    IP_GTPU_BYPASS_NEXT_DROP : IP_GTPU_BYPASS_NEXT_GTPU;
+	  b1->error = error1 ? error_node->errors[error1] : 0;
+
+	  /* gtpu-input node expect current at GTPU header */
+	  if (is_ip4)
+	    vlib_buffer_advance (b1, sizeof(ip4_header_t)+sizeof(udp_header_t));
+	  else
+	    vlib_buffer_advance (b1, sizeof(ip6_header_t)+sizeof(udp_header_t));
+
+	exit1:
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  vlib_buffer_t * b0;
+	  ip4_header_t * ip40;
+	  ip6_header_t * ip60;
+	  udp_header_t * udp0;
+      	  u32 bi0, ip_len0, udp_len0, flags0, next0;
+	  i32 len_diff0;
+	  u8 error0, good_udp0, proto0;
+
+	  bi0 = to_next[0] = from[0];
+	  from += 1;
+	  n_left_from -= 1;
+	  to_next += 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  if (is_ip4)
+	    ip40 = vlib_buffer_get_current (b0);
+	  else
+	    ip60 = vlib_buffer_get_current (b0);
+
+	  /* Setup packet for next IP feature */
+	  vnet_feature_next(vnet_buffer(b0)->sw_if_index[VLIB_RX], &next0, b0);
+
+	  if (is_ip4)
+	    /* Treat IP4 frag packets as "experimental" protocol for now
+	       until support of IP frag reassembly is implemented */
+	    proto0 = ip4_is_fragment(ip40) ? 0xfe : ip40->protocol;
+	  else
+	    proto0 = ip60->protocol;
+
+	  if (proto0 != IP_PROTOCOL_UDP)
+	    goto exit; /* not UDP packet */
+
+	  if (is_ip4)
+	    udp0 = ip4_next_header (ip40);
+	  else
+	    udp0 = ip6_next_header (ip60);
+
+	  if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_GTPU))
+	    goto exit; /* not GTPU packet */
+
+	  /* Validate DIP against VTEPs*/
+	  if (is_ip4)
+	    {
+	      if (addr4.as_u32 != ip40->dst_address.as_u32)
+	        {
+		  if (!hash_get (gtm->vtep4, ip40->dst_address.as_u32))
+		      goto exit; /* no local VTEP for GTPU packet */
+		  addr4 = ip40->dst_address;
+		}
+	    }
+	  else
+	    {
+	      if (!ip6_address_is_equal (&addr6, &ip60->dst_address))
+	        {
+		  if (!hash_get_mem (gtm->vtep6, &ip60->dst_address))
+		      goto exit; /* no local VTEP for GTPU packet */
+		  addr6 = ip60->dst_address;
+		}
+	    }
+
+	  flags0 = b0->flags;
+	  good_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
+
+	  /* Don't verify UDP checksum for packets with explicit zero checksum. */
+	  good_udp0 |= udp0->checksum == 0;
+
+	  /* Verify UDP length */
+	  if (is_ip4)
+	    ip_len0 = clib_net_to_host_u16 (ip40->length);
+	  else
+	    ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
+	  udp_len0 = clib_net_to_host_u16 (udp0->length);
+	  len_diff0 = ip_len0 - udp_len0;
+
+	  /* Verify UDP checksum */
+	  if (PREDICT_FALSE (!good_udp0))
+	    {
+	      if ((flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED) == 0)
+	        {
+		  if (is_ip4)
+		    flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
+		  else
+		    flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
+		  good_udp0 =
+		    (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
+	        }
+	    }
+
+	  if (is_ip4)
+	    {
+	      error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
+	      error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
+	    }
+	  else
+	    {
+	      error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
+	      error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
+	    }
+
+	  next0 = error0 ?
+	    IP_GTPU_BYPASS_NEXT_DROP : IP_GTPU_BYPASS_NEXT_GTPU;
+	  b0->error = error0 ? error_node->errors[error0] : 0;
+
+	  /* gtpu-input node expect current at GTPU header */
+	  if (is_ip4)
+	    vlib_buffer_advance (b0, sizeof(ip4_header_t)+sizeof(udp_header_t));
+	  else
+	    vlib_buffer_advance (b0, sizeof(ip6_header_t)+sizeof(udp_header_t));
+
+	exit:
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  return frame->n_vectors;
+}
+
+static uword
+ip4_gtpu_bypass (vlib_main_t * vm,
+		  vlib_node_runtime_t * node,
+		  vlib_frame_t * frame)
+{
+  return ip_gtpu_bypass_inline (vm, node, frame, /* is_ip4 */ 1);
+}
+
+VLIB_REGISTER_NODE (ip4_gtpu_bypass_node) = {
+  .function = ip4_gtpu_bypass,
+  .name = "ip4-gtpu-bypass",
+  .vector_size = sizeof (u32),
+
+  .n_next_nodes = IP_GTPU_BYPASS_N_NEXT,
+  .next_nodes = {
+    [IP_GTPU_BYPASS_NEXT_DROP] = "error-drop",
+    [IP_GTPU_BYPASS_NEXT_GTPU] = "gtpu4-input",
+  },
+
+  .format_buffer = format_ip4_header,
+  .format_trace = format_ip4_forward_next_trace,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_gtpu_bypass_node,ip4_gtpu_bypass)
+
+/* Dummy init function to get us linked in. */
+clib_error_t * ip4_gtpu_bypass_init (vlib_main_t * vm)
+{ return 0; }
+
+VLIB_INIT_FUNCTION (ip4_gtpu_bypass_init);
+
+static uword
+ip6_gtpu_bypass (vlib_main_t * vm,
+		  vlib_node_runtime_t * node,
+		  vlib_frame_t * frame)
+{
+  return ip_gtpu_bypass_inline (vm, node, frame, /* is_ip4 */ 0);
+}
+
+VLIB_REGISTER_NODE (ip6_gtpu_bypass_node) = {
+  .function = ip6_gtpu_bypass,
+  .name = "ip6-gtpu-bypass",
+  .vector_size = sizeof (u32),
+
+  .n_next_nodes = IP_GTPU_BYPASS_N_NEXT,
+  .next_nodes = {
+    [IP_GTPU_BYPASS_NEXT_DROP] = "error-drop",
+    [IP_GTPU_BYPASS_NEXT_GTPU] = "gtpu6-input",
+  },
+
+  .format_buffer = format_ip6_header,
+  .format_trace = format_ip6_forward_next_trace,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_gtpu_bypass_node,ip6_gtpu_bypass)
+
+/* Dummy init function to get us linked in. */
+clib_error_t * ip6_gtpu_bypass_init (vlib_main_t * vm)
+{ return 0; }
+
+VLIB_INIT_FUNCTION (ip6_gtpu_bypass_init);
diff --git a/src/plugins/gtpu/gtpu_encap.c b/src/plugins/gtpu/gtpu_encap.c
new file mode 100644
index 00000000..a2cfc646
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_encap.c
@@ -0,0 +1,569 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Statistics (not all errors) */
+#define foreach_gtpu_encap_error    \
+_(ENCAPSULATED, "good packets encapsulated")
+
+static char * gtpu_encap_error_strings[] = {
+#define _(sym,string) string,
+  foreach_gtpu_encap_error
+#undef _
+};
+
+typedef enum {
+#define _(sym,str) GTPU_ENCAP_ERROR_##sym,
+    foreach_gtpu_encap_error
+#undef _
+    GTPU_ENCAP_N_ERROR,
+} gtpu_encap_error_t;
+
+typedef enum {
+    GTPU_ENCAP_NEXT_DROP,
+    GTPU_ENCAP_N_NEXT,
+} gtpu_encap_next_t;
+
+typedef struct {
+  u32 tunnel_index;
+  u32 teid;
+} gtpu_encap_trace_t;
+
+u8 * format_gtpu_encap_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  gtpu_encap_trace_t * t
+      = va_arg (*args, gtpu_encap_trace_t *);
+
+  s = format (s, "GTPU encap to gtpu_tunnel%d teid %d",
+	      t->tunnel_index, t->teid);
+  return s;
+}
+
+
+#define foreach_fixed_header4_offset            \
+    _(0) _(1) _(2) _(3)
+
+#define foreach_fixed_header6_offset            \
+    _(0) _(1) _(2) _(3) _(4) _(5) _(6)
+
+always_inline uword
+gtpu_encap_inline (vlib_main_t * vm,
+		    vlib_node_runtime_t * node,
+		    vlib_frame_t * from_frame,
+		    u32 is_ip4)
+{
+  u32 n_left_from, next_index, * from, * to_next;
+  gtpu_main_t * gtm = >pu_main;
+  vnet_main_t * vnm = gtm->vnet_main;
+  vnet_interface_main_t * im = &vnm->interface_main;
+  u32 pkts_encapsulated = 0;
+  u16 old_l0 = 0, old_l1 = 0;
+  u32 thread_index = vlib_get_thread_index();
+  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
+  u32 sw_if_index0 = 0, sw_if_index1 = 0;
+  u32 next0 = 0, next1 = 0;
+  vnet_hw_interface_t * hi0, * hi1;
+  gtpu_tunnel_t * t0 = NULL, * t1 = NULL;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+  stats_sw_if_index = node->runtime_data[0];
+  stats_n_packets = stats_n_bytes = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+          u32 flow_hash0, flow_hash1;
+	  u32 len0, len1;
+          ip4_header_t * ip4_0, * ip4_1;
+          ip6_header_t * ip6_0, * ip6_1;
+          udp_header_t * udp0, * udp1;
+          gtpu_header_t * gtpu0, * gtpu1;
+          u64 * copy_src0, * copy_dst0;
+          u64 * copy_src1, * copy_dst1;
+          u32 * copy_src_last0, * copy_dst_last0;
+          u32 * copy_src_last1, * copy_dst_last1;
+          u16 new_l0, new_l1;
+          ip_csum_t sum0, sum1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	    CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	  }
+
+	  bi0 = from[0];
+	  bi1 = from[1];
+	  to_next[0] = bi0;
+	  to_next[1] = bi1;
+	  from += 2;
+	  to_next += 2;
+	  n_left_to_next -= 2;
+	  n_left_from -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          flow_hash0 = vnet_l2_compute_flow_hash (b0);
+          flow_hash1 = vnet_l2_compute_flow_hash (b1);
+
+	  /* Get next node index and adj index from tunnel next_dpo */
+	  if (sw_if_index0 != vnet_buffer(b0)->sw_if_index[VLIB_TX])
+	    {
+	      sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX];
+	      hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
+	      t0 = >m->tunnels[hi0->dev_instance];
+	      /* Note: change to always set next0 if it may be set to drop */
+	      next0 = t0->next_dpo.dpoi_next_node;
+	    }
+          vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index;
+
+	  /* Get next node index and adj index from tunnel next_dpo */
+	  if (sw_if_index1 != vnet_buffer(b1)->sw_if_index[VLIB_TX])
+	    {
+	      sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_TX];
+	      hi1 = vnet_get_sup_hw_interface (vnm, sw_if_index1);
+	      t1 = >m->tunnels[hi1->dev_instance];
+	      /* Note: change to always set next1 if it may be set to drop */
+	      next1 = t1->next_dpo.dpoi_next_node;
+	    }
+          vnet_buffer(b1)->ip.adj_index[VLIB_TX] = t1->next_dpo.dpoi_index;
+
+          /* Apply the rewrite string. $$$$ vnet_rewrite? */
+          vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite));
+          vlib_buffer_advance (b1, -(word)_vec_len(t1->rewrite));
+
+	  if (is_ip4)
+	    {
+	      ip4_0 = vlib_buffer_get_current(b0);
+	      ip4_1 = vlib_buffer_get_current(b1);
+
+	      /* Copy the fixed header */
+	      copy_dst0 = (u64 *) ip4_0;
+	      copy_src0 = (u64 *) t0->rewrite;
+	      copy_dst1 = (u64 *) ip4_1;
+	      copy_src1 = (u64 *) t1->rewrite;
+	      /* Copy first 32 octets 8-bytes at a time */
+#define _(offs) copy_dst0[offs] = copy_src0[offs];
+	      foreach_fixed_header4_offset;
+#undef _
+#define _(offs) copy_dst1[offs] = copy_src1[offs];
+	      foreach_fixed_header4_offset;
+#undef _
+	      /* Last 4 octets. Hopefully gcc will be our friend */
+              copy_dst_last0 = (u32 *)(©_dst0[4]);
+              copy_src_last0 = (u32 *)(©_src0[4]);
+              copy_dst_last0[0] = copy_src_last0[0];
+              copy_dst_last1 = (u32 *)(©_dst1[4]);
+              copy_src_last1 = (u32 *)(©_src1[4]);
+              copy_dst_last1[0] = copy_src_last1[0];
+
+	      /* Fix the IP4 checksum and length */
+	      sum0 = ip4_0->checksum;
+	      new_l0 = /* old_l0 always 0, see the rewrite setup */
+                clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
+              sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t,
+				     length /* changed member */);
+	      ip4_0->checksum = ip_csum_fold (sum0);
+	      ip4_0->length = new_l0;
+	      sum1 = ip4_1->checksum;
+	      new_l1 = /* old_l1 always 0, see the rewrite setup */
+                clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1));
+              sum1 = ip_csum_update (sum1, old_l1, new_l1, ip4_header_t,
+				     length /* changed member */);
+	      ip4_1->checksum = ip_csum_fold (sum1);
+	      ip4_1->length = new_l1;
+
+	      /* Fix UDP length and set source port */
+	      udp0 = (udp_header_t *)(ip4_0+1);
+	      new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					     - sizeof (*ip4_0));
+	      udp0->length = new_l0;
+	      udp0->src_port = flow_hash0;
+	      udp1 = (udp_header_t *)(ip4_1+1);
+	      new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
+					     - sizeof (*ip4_1));
+	      udp1->length = new_l1;
+	      udp1->src_port = flow_hash1;
+
+	      /* Fix GTPU length */
+	      gtpu0 = (gtpu_header_t *)(udp0+1);
+	      new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					     - sizeof (*ip4_0) - sizeof(*udp0));
+	      gtpu0->length = new_l0;
+	      gtpu1 = (gtpu_header_t *)(udp1+1);
+	      new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
+					     - sizeof (*ip4_1) - sizeof(*udp1));
+	      gtpu1->length = new_l1;
+	    }
+	  else /* ipv6 */
+	    {
+              int bogus = 0;
+
+	      ip6_0 = vlib_buffer_get_current(b0);
+	      ip6_1 = vlib_buffer_get_current(b1);
+
+	      /* Copy the fixed header */
+	      copy_dst0 = (u64 *) ip6_0;
+	      copy_src0 = (u64 *) t0->rewrite;
+	      copy_dst1 = (u64 *) ip6_1;
+	      copy_src1 = (u64 *) t1->rewrite;
+	      /* Copy first 56 (ip6) octets 8-bytes at a time */
+#define _(offs) copy_dst0[offs] = copy_src0[offs];
+	      foreach_fixed_header6_offset;
+#undef _
+#define _(offs) copy_dst1[offs] = copy_src1[offs];
+	      foreach_fixed_header6_offset;
+#undef _
+	      /* Fix IP6 payload length */
+	      new_l0 =
+                clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)
+				      - sizeof(*ip6_0));
+	      ip6_0->payload_length = new_l0;
+	      new_l1 =
+                clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1)
+				      - sizeof(*ip6_1));
+	      ip6_1->payload_length = new_l1;
+
+	      /* Fix UDP length  and set source port */
+	      udp0 = (udp_header_t *)(ip6_0+1);
+	      udp0->length = new_l0;
+	      udp0->src_port = flow_hash0;
+	      udp1 = (udp_header_t *)(ip6_1+1);
+	      udp1->length = new_l1;
+	      udp1->src_port = flow_hash1;
+
+	      /* IPv6 UDP checksum is mandatory */
+	      udp0->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b0,
+								 ip6_0, &bogus);
+	      if (udp0->checksum == 0)
+		udp0->checksum = 0xffff;
+	      udp1->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b1,
+                                                        ip6_1, &bogus);
+	      if (udp1->checksum == 0)
+		udp1->checksum = 0xffff;
+
+	      /* Fix GTPU length */
+	      gtpu0 = (gtpu_header_t *)(udp0+1);
+	      new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					     - sizeof (*ip4_0) - sizeof(*udp0));
+	      gtpu0->length = new_l0;
+	      gtpu1 = (gtpu_header_t *)(udp1+1);
+	      new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
+					     - sizeof (*ip4_1) - sizeof(*udp1));
+	      gtpu1->length = new_l1;
+	    }
+
+          pkts_encapsulated += 2;
+ 	  len0 = vlib_buffer_length_in_chain (vm, b0);
+ 	  len1 = vlib_buffer_length_in_chain (vm, b1);
+	  stats_n_packets += 2;
+	  stats_n_bytes += len0 + len1;
+
+	  /* Batch stats increment on the same gtpu tunnel so counter is not
+	     incremented per packet. Note stats are still incremented for deleted
+	     and admin-down tunnel where packets are dropped. It is not worthwhile
+	     to check for this rare case and affect normal path performance. */
+	  if (PREDICT_FALSE ((sw_if_index0 != stats_sw_if_index) ||
+			     (sw_if_index1 != stats_sw_if_index)))
+	    {
+	      stats_n_packets -= 2;
+	      stats_n_bytes -= len0 + len1;
+	      if (sw_if_index0 == sw_if_index1)
+	        {
+		  if (stats_n_packets)
+		    vlib_increment_combined_counter
+		      (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		       thread_index, stats_sw_if_index,
+		       stats_n_packets, stats_n_bytes);
+		  stats_sw_if_index = sw_if_index0;
+		  stats_n_packets = 2;
+		  stats_n_bytes = len0 + len1;
+	        }
+	      else
+	        {
+		  vlib_increment_combined_counter
+		      (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		       thread_index, sw_if_index0, 1, len0);
+		  vlib_increment_combined_counter
+		      (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		       thread_index, sw_if_index1, 1, len1);
+		}
+	    }
+
+	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              gtpu_encap_trace_t *tr =
+                vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->tunnel_index = t0 - gtm->tunnels;
+              tr->teid = t0->teid;
+           }
+
+          if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              gtpu_encap_trace_t *tr =
+                vlib_add_trace (vm, node, b1, sizeof (*tr));
+              tr->tunnel_index = t1 - gtm->tunnels;
+              tr->teid = t1->teid;
+            }
+
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 flow_hash0;
+	  u32 len0;
+          ip4_header_t * ip4_0;
+          ip6_header_t * ip6_0;
+          udp_header_t * udp0;
+          gtpu_header_t * gtpu0;
+          u64 * copy_src0, * copy_dst0;
+          u32 * copy_src_last0, * copy_dst_last0;
+          u16 new_l0;
+          ip_csum_t sum0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+          flow_hash0 = vnet_l2_compute_flow_hash(b0);
+
+	  /* Get next node index and adj index from tunnel next_dpo */
+	  if (sw_if_index0 != vnet_buffer(b0)->sw_if_index[VLIB_TX])
+	    {
+	      sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX];
+	      hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
+	      t0 = >m->tunnels[hi0->dev_instance];
+	      /* Note: change to always set next0 if it may be set to drop */
+	      next0 = t0->next_dpo.dpoi_next_node;
+	    }
+	  vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index;
+
+          /* Apply the rewrite string. $$$$ vnet_rewrite? */
+          vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite));
+
+	  if (is_ip4)
+	    {
+	      ip4_0 = vlib_buffer_get_current(b0);
+
+	      /* Copy the fixed header */
+	      copy_dst0 = (u64 *) ip4_0;
+	      copy_src0 = (u64 *) t0->rewrite;
+	      /* Copy first 32 octets 8-bytes at a time */
+#define _(offs) copy_dst0[offs] = copy_src0[offs];
+	      foreach_fixed_header4_offset;
+#undef _
+	      /* Last 4 octets. Hopefully gcc will be our friend */
+              copy_dst_last0 = (u32 *)(©_dst0[4]);
+              copy_src_last0 = (u32 *)(©_src0[4]);
+              copy_dst_last0[0] = copy_src_last0[0];
+
+	      /* Fix the IP4 checksum and length */
+	      sum0 = ip4_0->checksum;
+	      new_l0 = /* old_l0 always 0, see the rewrite setup */
+                clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
+              sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t,
+				     length /* changed member */);
+	      ip4_0->checksum = ip_csum_fold (sum0);
+	      ip4_0->length = new_l0;
+
+	      /* Fix UDP length and set source port */
+	      udp0 = (udp_header_t *)(ip4_0+1);
+	      new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					     - sizeof (*ip4_0));
+	      udp0->length = new_l0;
+	      udp0->src_port = flow_hash0;
+
+	      /* Fix GTPU length */
+	      gtpu0 = (gtpu_header_t *)(udp0+1);
+	      new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					     - sizeof (*ip4_0) - sizeof(*udp0));
+	      gtpu0->length = new_l0;
+	    }
+
+	  else /* ip6 path */
+	    {
+              int bogus = 0;
+
+	      ip6_0 = vlib_buffer_get_current(b0);
+	      /* Copy the fixed header */
+	      copy_dst0 = (u64 *) ip6_0;
+	      copy_src0 = (u64 *) t0->rewrite;
+	      /* Copy first 56 (ip6) octets 8-bytes at a time */
+#define _(offs) copy_dst0[offs] = copy_src0[offs];
+	      foreach_fixed_header6_offset;
+#undef _
+	      /* Fix IP6 payload length */
+	      new_l0 =
+                clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)
+				      - sizeof(*ip6_0));
+	      ip6_0->payload_length = new_l0;
+
+	      /* Fix UDP length  and set source port */
+	      udp0 = (udp_header_t *)(ip6_0+1);
+	      udp0->length = new_l0;
+	      udp0->src_port = flow_hash0;
+
+	      /* IPv6 UDP checksum is mandatory */
+	      udp0->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b0,
+								 ip6_0, &bogus);
+	      if (udp0->checksum == 0)
+		udp0->checksum = 0xffff;
+
+	      /* Fix GTPU length */
+	      gtpu0 = (gtpu_header_t *)(udp0+1);
+	      new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					     - sizeof (*ip4_0) - sizeof(*udp0));
+	      gtpu0->length = new_l0;
+	    }
+
+          pkts_encapsulated ++;
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+	  stats_n_packets += 1;
+	  stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same gtpu tunnel so counter is not
+	     incremented per packet. Note stats are still incremented for deleted
+	     and admin-down tunnel where packets are dropped. It is not worthwhile
+	     to check for this rare case and affect normal path performance. */
+	  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = sw_if_index0;
+	    }
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              gtpu_encap_trace_t *tr =
+                vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->tunnel_index = t0 - gtm->tunnels;
+              tr->teid = t0->teid;
+            }
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Do we still need this now that tunnel tx stats is kept? */
+  vlib_node_increment_counter (vm, node->node_index,
+                               GTPU_ENCAP_ERROR_ENCAPSULATED,
+                               pkts_encapsulated);
+
+  /* Increment any remaining batch stats */
+  if (stats_n_packets)
+    {
+      vlib_increment_combined_counter
+	(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+	 thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
+      node->runtime_data[0] = stats_sw_if_index;
+    }
+
+  return from_frame->n_vectors;
+}
+
+static uword
+gtpu4_encap (vlib_main_t * vm,
+	      vlib_node_runtime_t * node,
+	      vlib_frame_t * from_frame)
+{
+  return gtpu_encap_inline (vm, node, from_frame, /* is_ip4 */ 1);
+}
+
+static uword
+gtpu6_encap (vlib_main_t * vm,
+	      vlib_node_runtime_t * node,
+	      vlib_frame_t * from_frame)
+{
+  return gtpu_encap_inline (vm, node, from_frame, /* is_ip4 */ 0);
+}
+
+VLIB_REGISTER_NODE (gtpu4_encap_node) = {
+  .function = gtpu4_encap,
+  .name = "gtpu4-encap",
+  .vector_size = sizeof (u32),
+  .format_trace = format_gtpu_encap_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(gtpu_encap_error_strings),
+  .error_strings = gtpu_encap_error_strings,
+  .n_next_nodes = GTPU_ENCAP_N_NEXT,
+  .next_nodes = {
+        [GTPU_ENCAP_NEXT_DROP] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (gtpu4_encap_node, gtpu4_encap)
+
+VLIB_REGISTER_NODE (gtpu6_encap_node) = {
+  .function = gtpu6_encap,
+  .name = "gtpu6-encap",
+  .vector_size = sizeof (u32),
+  .format_trace = format_gtpu_encap_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(gtpu_encap_error_strings),
+  .error_strings = gtpu_encap_error_strings,
+  .n_next_nodes = GTPU_ENCAP_N_NEXT,
+  .next_nodes = {
+        [GTPU_ENCAP_NEXT_DROP] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (gtpu6_encap_node, gtpu6_encap)
+
diff --git a/src/plugins/gtpu/gtpu_error.def b/src/plugins/gtpu/gtpu_error.def
new file mode 100644
index 00000000..a55b2026
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_error.def
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+gtpu_error (DECAPSULATED, "good packets decapsulated")
+gtpu_error (NO_SUCH_TUNNEL, "no such tunnel packets")
+gtpu_error (BAD_VER, "packets with bad version in gtpu header")
+gtpu_error (BAD_FLAGS, "packets with bad flags field in gtpu header")
diff --git a/src/plugins/gtpu/gtpu_msg_enum.h b/src/plugins/gtpu/gtpu_msg_enum.h
new file mode 100644
index 00000000..6a2ea448
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_msg_enum.h
@@ -0,0 +1,31 @@
+/*
+ * gtpu_msg_enum.h - vpp engine plug-in message enumeration
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_gtpu_msg_enum_h
+#define included_gtpu_msg_enum_h
+
+#include 
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include 
+  /* We'll want to know how many messages IDs we need... */
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_gtpu_msg_enum_h */
diff --git a/src/plugins/gtpu/gtpu_test.c b/src/plugins/gtpu/gtpu_test.c
new file mode 100644
index 00000000..91c5c616
--- /dev/null
+++ b/src/plugins/gtpu/gtpu_test.c
@@ -0,0 +1,498 @@
+/*
+ * Copyright (c) 2016 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define __plugin_msg_base gtpu_test_main.msg_id_base
+#include 
+
+
+uword unformat_ip46_address (unformat_input_t * input, va_list * args)
+{
+  ip46_address_t *ip46 = va_arg (*args, ip46_address_t *);
+  ip46_type_t type = va_arg (*args, ip46_type_t);
+  if ((type != IP46_TYPE_IP6) &&
+      unformat(input, "%U", unformat_ip4_address, &ip46->ip4)) {
+    ip46_address_mask_ip4(ip46);
+    return 1;
+  } else if ((type != IP46_TYPE_IP4) &&
+      unformat(input, "%U", unformat_ip6_address, &ip46->ip6)) {
+    return 1;
+  }
+  return 0;
+}
+uword unformat_ip46_prefix (unformat_input_t * input, va_list * args)
+{
+  ip46_address_t *ip46 = va_arg (*args, ip46_address_t *);
+  u8 *len = va_arg (*args, u8 *);
+  ip46_type_t type = va_arg (*args, ip46_type_t);
+
+  u32 l;
+  if ((type != IP46_TYPE_IP6) && unformat(input, "%U/%u", unformat_ip4_address, &ip46->ip4, &l)) {
+    if (l > 32)
+      return 0;
+    *len = l + 96;
+    ip46->pad[0] = ip46->pad[1] = ip46->pad[2] = 0;
+  } else if ((type != IP46_TYPE_IP4) && unformat(input, "%U/%u", unformat_ip6_address, &ip46->ip6, &l)) {
+    if (l > 128)
+      return 0;
+    *len = l;
+  } else {
+    return 0;
+  }
+  return 1;
+}
+/////////////////////////
+
+#define vl_msg_id(n,h) n,
+typedef enum {
+#include 
+    /* We'll want to know how many messages IDs we need... */
+    VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+
+#define vl_endianfun             /* define message structures */
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+typedef struct {
+    /* API message ID base */
+    u16 msg_id_base;
+    vat_main_t *vat_main;
+} gtpu_test_main_t;
+
+gtpu_test_main_t gtpu_test_main;
+
+static void vl_api_gtpu_add_del_tunnel_reply_t_handler
+  (vl_api_gtpu_add_del_tunnel_reply_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+  i32 retval = ntohl (mp->retval);
+  if (vam->async_mode)
+    {
+      vam->async_errors += (retval < 0);
+    }
+  else
+    {
+      vam->retval = retval;
+      vam->sw_if_index = ntohl (mp->sw_if_index);
+      vam->result_ready = 1;
+    }
+}
+
+
+#define foreach_standard_reply_retval_handler   \
+    _(sw_interface_set_gtpu_bypass_reply)
+
+#define _(n)                                            \
+    static void vl_api_##n##_t_handler                  \
+    (vl_api_##n##_t * mp)                               \
+    {                                                   \
+        vat_main_t * vam = gtpu_test_main.vat_main;   \
+        i32 retval = ntohl(mp->retval);                 \
+        if (vam->async_mode) {                          \
+            vam->async_errors += (retval < 0);          \
+        } else {                                        \
+            vam->retval = retval;                       \
+            vam->result_ready = 1;                      \
+        }                                               \
+    }
+  foreach_standard_reply_retval_handler;
+#undef _
+
+/*
+ * Table of message reply handlers, must include boilerplate handlers
+ * we just generated
+ */
+#define foreach_vpe_api_reply_msg                               \
+  _(SW_INTERFACE_SET_GTPU_BYPASS_REPLY, sw_interface_set_gtpu_bypass_reply) \
+  _(GTPU_ADD_DEL_TUNNEL_REPLY, gtpu_add_del_tunnel_reply)               \
+  _(GTPU_TUNNEL_DETAILS, gtpu_tunnel_details)
+
+
+static uword
+api_unformat_sw_if_index (unformat_input_t * input, va_list * args)
+{
+  vat_main_t *vam = va_arg (*args, vat_main_t *);
+  u32 *result = va_arg (*args, u32 *);
+  u8 *if_name;
+  uword *p;
+
+  if (!unformat (input, "%s", &if_name))
+    return 0;
+
+  p = hash_get_mem (vam->sw_if_index_by_interface_name, if_name);
+  if (p == 0)
+    return 0;
+  *result = p[0];
+  return 1;
+}
+
+static int
+api_sw_interface_set_gtpu_bypass (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_sw_interface_set_gtpu_bypass_t *mp;
+  u32 sw_if_index = 0;
+  u8 sw_if_index_set = 0;
+  u8 is_enable = 1;
+  u8 is_ipv6 = 0;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", api_unformat_sw_if_index, vam, &sw_if_index))
+      sw_if_index_set = 1;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+      sw_if_index_set = 1;
+      else if (unformat (i, "enable"))
+      is_enable = 1;
+      else if (unformat (i, "disable"))
+      is_enable = 0;
+      else if (unformat (i, "ip4"))
+      is_ipv6 = 0;
+      else if (unformat (i, "ip6"))
+      is_ipv6 = 1;
+      else
+      break;
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      errmsg ("missing interface name or sw_if_index");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (SW_INTERFACE_SET_GTPU_BYPASS, mp);
+
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->enable = is_enable;
+  mp->is_ipv6 = is_ipv6;
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply... */
+  W (ret);
+  return ret;
+}
+
+static uword unformat_gtpu_decap_next
+  (unformat_input_t * input, va_list * args)
+{
+  u32 *result = va_arg (*args, u32 *);
+  u32 tmp;
+
+  if (unformat (input, "l2"))
+    *result = GTPU_INPUT_NEXT_L2_INPUT;
+  else if (unformat (input, "%d", &tmp))
+    *result = tmp;
+  else
+    return 0;
+  return 1;
+}
+
+static int
+api_gtpu_add_del_tunnel (vat_main_t * vam)
+{
+  unformat_input_t *line_input = vam->input;
+  vl_api_gtpu_add_del_tunnel_t *mp;
+  ip46_address_t src, dst;
+  u8 is_add = 1;
+  u8 ipv4_set = 0, ipv6_set = 0;
+  u8 src_set = 0;
+  u8 dst_set = 0;
+  u8 grp_set = 0;
+  u32 mcast_sw_if_index = ~0;
+  u32 encap_vrf_id = 0;
+  u32 decap_next_index = ~0;
+  u32 teid = 0;
+  int ret;
+
+  /* Can't "universally zero init" (={0}) due to GCC bug 53119 */
+  memset (&src, 0, sizeof src);
+  memset (&dst, 0, sizeof dst);
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "del"))
+      is_add = 0;
+      else
+      if (unformat (line_input, "src %U", unformat_ip4_address, &src.ip4))
+      {
+	ipv4_set = 1;
+	src_set = 1;
+      }
+      else
+      if (unformat (line_input, "dst %U", unformat_ip4_address, &dst.ip4))
+      {
+	ipv4_set = 1;
+	dst_set = 1;
+      }
+      else
+      if (unformat (line_input, "src %U", unformat_ip6_address, &src.ip6))
+      {
+	ipv6_set = 1;
+	src_set = 1;
+      }
+      else
+      if (unformat (line_input, "dst %U", unformat_ip6_address, &dst.ip6))
+      {
+	ipv6_set = 1;
+	dst_set = 1;
+      }
+      else if (unformat (line_input, "group %U %U",
+		       unformat_ip4_address, &dst.ip4,
+		       api_unformat_sw_if_index, vam, &mcast_sw_if_index))
+      {
+	grp_set = dst_set = 1;
+	ipv4_set = 1;
+      }
+      else if (unformat (line_input, "group %U",
+		       unformat_ip4_address, &dst.ip4))
+      {
+	grp_set = dst_set = 1;
+	ipv4_set = 1;
+      }
+      else if (unformat (line_input, "group %U %U",
+		       unformat_ip6_address, &dst.ip6,
+		       api_unformat_sw_if_index, vam, &mcast_sw_if_index))
+      {
+	grp_set = dst_set = 1;
+	ipv6_set = 1;
+      }
+      else if (unformat (line_input, "group %U",
+		       unformat_ip6_address, &dst.ip6))
+      {
+	grp_set = dst_set = 1;
+	ipv6_set = 1;
+      }
+      else
+      if (unformat (line_input, "mcast_sw_if_index %u", &mcast_sw_if_index))
+      ;
+      else if (unformat (line_input, "encap-vrf-id %d", &encap_vrf_id))
+      ;
+      else if (unformat (line_input, "decap-next %U",
+		       unformat_gtpu_decap_next, &decap_next_index))
+      ;
+      else if (unformat (line_input, "teid %d", &teid))
+      ;
+      else
+      {
+	errmsg ("parse error '%U'", format_unformat_error, line_input);
+	return -99;
+      }
+    }
+
+  if (src_set == 0)
+    {
+      errmsg ("tunnel src address not specified");
+      return -99;
+    }
+  if (dst_set == 0)
+    {
+      errmsg ("tunnel dst address not specified");
+      return -99;
+    }
+
+  if (grp_set && !ip46_address_is_multicast (&dst))
+    {
+      errmsg ("tunnel group address not multicast");
+      return -99;
+    }
+  if (grp_set && mcast_sw_if_index == ~0)
+    {
+      errmsg ("tunnel nonexistent multicast device");
+      return -99;
+    }
+  if (grp_set == 0 && ip46_address_is_multicast (&dst))
+    {
+      errmsg ("tunnel dst address must be unicast");
+      return -99;
+    }
+
+
+  if (ipv4_set && ipv6_set)
+    {
+      errmsg ("both IPv4 and IPv6 addresses specified");
+      return -99;
+    }
+
+  M (GTPU_ADD_DEL_TUNNEL, mp);
+
+  if (ipv6_set)
+    {
+      clib_memcpy (mp->src_address, &src.ip6, sizeof (src.ip6));
+      clib_memcpy (mp->dst_address, &dst.ip6, sizeof (dst.ip6));
+    }
+  else
+    {
+      clib_memcpy (mp->src_address, &src.ip4, sizeof (src.ip4));
+      clib_memcpy (mp->dst_address, &dst.ip4, sizeof (dst.ip4));
+    }
+  mp->encap_vrf_id = ntohl (encap_vrf_id);
+  mp->decap_next_index = ntohl (decap_next_index);
+  mp->mcast_sw_if_index = ntohl (mcast_sw_if_index);
+  mp->teid = ntohl (teid);
+  mp->is_add = is_add;
+  mp->is_ipv6 = ipv6_set;
+
+  S (mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_gtpu_tunnel_details_t_handler
+  (vl_api_gtpu_tunnel_details_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+  ip46_address_t src = to_ip46 (mp->is_ipv6, mp->dst_address);
+  ip46_address_t dst = to_ip46 (mp->is_ipv6, mp->src_address);
+
+  print (vam->ofp, "%11d%24U%24U%14d%18d%13d%19d",
+       ntohl (mp->sw_if_index),
+       format_ip46_address, &src, IP46_TYPE_ANY,
+       format_ip46_address, &dst, IP46_TYPE_ANY,
+       ntohl (mp->encap_vrf_id),
+       ntohl (mp->decap_next_index), ntohl (mp->teid),
+       ntohl (mp->mcast_sw_if_index));
+}
+
+static int
+api_gtpu_tunnel_dump (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_gtpu_tunnel_dump_t *mp;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "sw_if_index %d", &sw_if_index))
+      sw_if_index_set = 1;
+      else
+      break;
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      sw_if_index = ~0;
+    }
+
+  if (!vam->json_output)
+    {
+      print (vam->ofp, "%11s%24s%24s%14s%18s%13s%19s",
+	   "sw_if_index", "src_address", "dst_address",
+	   "encap_vrf_id", "decap_next_index", "teid", "mcast_sw_if_index");
+    }
+
+  /* Get list of gtpu-tunnel interfaces */
+  M (GTPU_TUNNEL_DUMP, mp);
+
+  mp->sw_if_index = htonl (sw_if_index);
+
+  S (mp);
+
+  W (ret);
+  return ret;
+}
+
+/*
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg                                            \
+_(sw_interface_set_gtpu_bypass,                                        \
+      " | sw_if_index  [ip4 | ip6] [enable | disable]")     \
+_(gtpu_add_del_tunnel,                                                 \
+        "src  { dst  | group \n"      \
+        "{  | mcast_sw_if_index  } }\n"                     \
+        "teid  [encap-vrf-id ] [decap-next ] [del]")  \
+_(gtpu_tunnel_dump, "[ | sw_if_index ]")                    \
+
+static void
+gtpu_vat_api_hookup (vat_main_t *vam)
+{
+  gtpu_test_main_t * gtm = >pu_test_main;
+  /* Hook up handlers for replies from the data plane plug-in */
+#define _(N,n)                                                  \
+  vl_msg_api_set_handlers((VL_API_##N + gtm->msg_id_base),       \
+                          #n,                                   \
+                          vl_api_##n##_t_handler,               \
+                          vl_noop_handler,                      \
+                          vl_api_##n##_t_endian,                \
+                          vl_api_##n##_t_print,                 \
+                          sizeof(vl_api_##n##_t), 1);
+  foreach_vpe_api_reply_msg;
+#undef _
+
+  /* API messages we can send */
+#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _
+
+  /* Help strings */
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t * vat_plugin_register (vat_main_t *vam)
+{
+  gtpu_test_main_t * gtm = >pu_test_main;
+
+  u8 * name;
+
+  gtm->vat_main = vam;
+
+  /* Ask the vpp engine for the first assigned message-id */
+  name = format (0, "gtpu_%08x%c", api_version, 0);
+  gtm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (gtm->msg_id_base != (u16) ~0)
+    gtpu_vat_api_hookup (vam);
+
+  vec_free(name);
+
+  return 0;
+}
diff --git a/src/vnet/mfib/mfib_types.h b/src/vnet/mfib/mfib_types.h
index fe53aa68..e278af7f 100644
--- a/src/vnet/mfib/mfib_types.h
+++ b/src/vnet/mfib/mfib_types.h
@@ -167,6 +167,7 @@ typedef enum mfib_source_t_
     MFIB_SOURCE_DHCP,
     MFIB_SOURCE_SRv6,
     MFIB_SOURCE_DEFAULT_ROUTE,
+    MFIB_SOURCE_GTPU,
 } mfib_source_t;
 
 #define MFIB_SOURCE_NAMES {                        \
@@ -177,6 +178,7 @@ typedef enum mfib_source_t_
     [MFIB_SOURCE_VXLAN] = "VXLAN",                 \
     [MFIB_SOURCE_SRv6] = "SRv6",                   \
     [MFIB_SOURCE_DEFAULT_ROUTE] = "Default Route", \
+    [MFIB_SOURCE_GTPU] = "GTPU",                   \
 }
 
 /**
diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h
index bd163e27..5b63f44d 100644
--- a/src/vnet/udp/udp.h
+++ b/src/vnet/udp/udp.h
@@ -81,6 +81,7 @@ typedef enum
 _ (67, dhcp_to_server)                          \
 _ (68, dhcp_to_client)                          \
 _ (500, ikev2)                                  \
+_ (2152, GTPU)					\
 _ (3784, bfd4)                                  \
 _ (3785, bfd_echo4)                             \
 _ (4341, lisp_gpe)                              \
@@ -95,6 +96,7 @@ _ (6633, vpath_3)
 #define foreach_udp6_dst_port                   \
 _ (547, dhcpv6_to_server)                       \
 _ (546, dhcpv6_to_client)			\
+_ (2152, GTPU6)					\
 _ (3784, bfd6)                                  \
 _ (3785, bfd_echo6)                             \
 _ (4341, lisp_gpe6)                             \
diff --git a/src/vpp-api/java/Makefile.am b/src/vpp-api/java/Makefile.am
index f18e0c24..4f1f7ed2 100644
--- a/src/vpp-api/java/Makefile.am
+++ b/src/vpp-api/java/Makefile.am
@@ -128,6 +128,26 @@ jvpp-acl/io_fd_vpp_jvpp_acl_JVppAclImpl.h: $(jvpp_registry_ok) $(jvpp_acl_json_f
 	$(call japigen,acl,JVppAclImpl)
 endif
 
+#
+# GTPU Plugin
+#
+if ENABLE_GTPU_PLUGIN
+noinst_LTLIBRARIES += libjvpp_gtpu.la
+libjvpp_gtpu_la_SOURCES = jvpp-gtpu/jvpp_gtpu.c
+libjvpp_gtpu_la_CPPFLAGS = -Ijvpp-gtpu
+libjvpp_gtpu_la_LIBADD = $(JVPP_LIBS)
+libjvpp_gtpu_la_DEPENDENCIES = libjvpp_common.la
+
+BUILT_SOURCES += jvpp-gtpu/io_fd_vpp_jvpp_gtpu_JVppGtpuImpl.h
+JAR_FILES += jvpp-gtpu-$(PACKAGE_VERSION).jar
+CLEANDIRS += jvpp-gtpu/target
+
+jvpp_gtpu_json_files = @top_builddir@/plugins/gtpu/gtpu.api.json
+
+jvpp-gtpu/io_fd_vpp_jvpp_gtpu_JVppGtpuImpl.h: $(jvpp_registry_ok) $(jvpp_gtpu_json_files)
+	$(call japigen,gtpu,JVppGtpuImpl)
+endif
+
 #
 # SNAT Plugin
 #
diff --git a/src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.c b/src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.c
new file mode 100644
index 00000000..12b3090e
--- /dev/null
+++ b/src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-gtpu/io_fd_vpp_jvpp_gtpu_JVppGtpuImpl.h"
+#include "jvpp_gtpu.h"
+#include "jvpp-gtpu/jvpp_gtpu_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_gtpu_JVppgtpuImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_gtpu_JVppGtpuImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  gtpu_main_t * plugin_main = >pu_main;
+  clib_warning ("Java_io_fd_vpp_jvpp_gtpu_JVppGtpuImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+  plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+  plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+  // verify API has not changed since jar generation
+  #define _(N)             \
+      get_message_id(env, #N);  \
+      foreach_supported_api_message;
+  #undef _
+
+  #define _(N,n)                                  \
+      vl_msg_api_set_handlers(get_message_id(env, #N), #n,     \
+              vl_api_##n##_t_handler,             \
+              vl_noop_handler,                    \
+              vl_noop_handler,                    \
+              vl_noop_handler,                    \
+              sizeof(vl_api_##n##_t), 1);
+      foreach_api_reply_handler;
+  #undef _
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_gtpu_JVppGtpuImpl_close0
+(JNIEnv *env, jclass clazz) {
+  gtpu_main_t * plugin_main = >pu_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP ACL */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP ACL */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.h b/src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.h
new file mode 100644
index 00000000..447776ce
--- /dev/null
+++ b/src/vpp-api/java/jvpp-gtpu/jvpp_gtpu.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_gtpu_h__
+#define __included_jvpp_gtpu_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-gtpu */
+typedef struct {
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} gtpu_main_t;
+
+gtpu_main_t gtpu_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_gtpu_h__ */
diff --git a/test/test_gtpu.py b/test/test_gtpu.py
new file mode 100644
index 00000000..9411fffc
--- /dev/null
+++ b/test/test_gtpu.py
@@ -0,0 +1,289 @@
+#!/usr/bin/env python
+
+import socket
+from util import ip4n_range
+import unittest
+from framework import VppTestCase, VppTestRunner
+from template_bd import BridgeDomain
+
+from scapy.layers.l2 import Ether, Raw
+from scapy.layers.inet import IP, UDP
+from scapy.contrib.gtp import GTP_U_Header
+from scapy.utils import atol
+
+
+class TestGtpu(BridgeDomain, VppTestCase):
+    """ GTPU Test Case """
+
+    def __init__(self, *args):
+        BridgeDomain.__init__(self)
+        VppTestCase.__init__(self, *args)
+
+    def encapsulate(self, pkt, vni):
+        """
+        Encapsulate the original payload frame by adding GTPU header with its
+        UDP, IP and Ethernet fields
+        """
+        return (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                IP(src=self.pg0.remote_ip4, dst=self.pg0.local_ip4) /
+                UDP(sport=self.dport, dport=self.dport, chksum=0) /
+                GTP_U_Header(TEID=vni, gtp_type=self.gtp_type, length=150) /
+                pkt)
+
+    def encap_mcast(self, pkt, src_ip, src_mac, vni):
+        """
+        Encapsulate the original payload frame by adding GTPU header with its
+        UDP, IP and Ethernet fields
+        """
+        return (Ether(src=src_mac, dst=self.mcast_mac) /
+                IP(src=src_ip, dst=self.mcast_ip4) /
+                UDP(sport=self.dport, dport=self.dport, chksum=0) /
+                GTP_U_Header(TEID=vni, gtp_type=self.gtp_type, length=150) /
+                pkt)
+
+    def decapsulate(self, pkt):
+        """
+        Decapsulate the original payload frame by removing GTPU header
+        """
+        return pkt[GTP_U_Header].payload
+
+    # Method for checking GTPU encapsulation.
+    #
+    def check_encapsulation(self, pkt, vni, local_only=False, mcast_pkt=False):
+        # Verify source MAC is VPP_MAC and destination MAC is MY_MAC resolved
+        #  by VPP using ARP.
+        self.assertEqual(pkt[Ether].src, self.pg0.local_mac)
+        if not local_only:
+            if not mcast_pkt:
+                self.assertEqual(pkt[Ether].dst, self.pg0.remote_mac)
+            else:
+                self.assertEqual(pkt[Ether].dst, type(self).mcast_mac)
+        # Verify GTPU tunnel source IP is VPP_IP and destination IP is MY_IP.
+        self.assertEqual(pkt[IP].src, self.pg0.local_ip4)
+        if not local_only:
+            if not mcast_pkt:
+                self.assertEqual(pkt[IP].dst, self.pg0.remote_ip4)
+            else:
+                self.assertEqual(pkt[IP].dst, type(self).mcast_ip4)
+        # Verify UDP destination port is GTPU 2152, source UDP port could be
+        #  arbitrary.
+        self.assertEqual(pkt[UDP].dport, type(self).dport)
+        # Verify TEID
+        self.assertEqual(pkt[GTP_U_Header].TEID, vni)
+
+    def test_encap(self):
+        """ Encapsulation test
+        Send frames from pg1
+        Verify receipt of encapsulated frames on pg0
+        """
+        self.pg1.add_stream([self.frame_reply])
+
+        self.pg0.enable_capture()
+
+        self.pg_start()
+
+        # Pick first received frame and check if it's corectly encapsulated.
+        out = self.pg0.get_capture(1)
+        pkt = out[0]
+        self.check_encapsulation(pkt, self.single_tunnel_bd)
+
+        # payload = self.decapsulate(pkt)
+        # self.assert_eq_pkts(payload, self.frame_reply)
+
+    def test_ucast_flood(self):
+        """ Unicast flood test
+        Send frames from pg3
+        Verify receipt of encapsulated frames on pg0
+        """
+        self.pg3.add_stream([self.frame_reply])
+
+        self.pg0.enable_capture()
+
+        self.pg_start()
+
+        # Get packet from each tunnel and assert it's corectly encapsulated.
+        out = self.pg0.get_capture(self.n_ucast_tunnels)
+        for pkt in out:
+            self.check_encapsulation(pkt, self.ucast_flood_bd, True)
+            # payload = self.decapsulate(pkt)
+            # self.assert_eq_pkts(payload, self.frame_reply)
+
+    def test_mcast_flood(self):
+        """ Multicast flood test
+        Send frames from pg2
+        Verify receipt of encapsulated frames on pg0
+        """
+        self.pg2.add_stream([self.frame_reply])
+
+        self.pg0.enable_capture()
+
+        self.pg_start()
+
+        # Pick first received frame and check if it's corectly encapsulated.
+        out = self.pg0.get_capture(1)
+        pkt = out[0]
+        self.check_encapsulation(pkt, self.mcast_flood_bd,
+                                 local_only=False, mcast_pkt=True)
+
+        # payload = self.decapsulate(pkt)
+        # self.assert_eq_pkts(payload, self.frame_reply)
+
+    @classmethod
+    def create_gtpu_flood_test_bd(cls, teid, n_ucast_tunnels):
+        # Create 10 ucast gtpu tunnels under bd
+        ip_range_start = 10
+        ip_range_end = ip_range_start + n_ucast_tunnels
+        next_hop_address = cls.pg0.remote_ip4n
+        for dest_ip4n in ip4n_range(next_hop_address, ip_range_start,
+                                    ip_range_end):
+            # add host route so dest_ip4n will not be resolved
+            cls.vapi.ip_add_del_route(dest_ip4n, 32, next_hop_address)
+            r = cls.vapi.gtpu_add_del_tunnel(
+                src_addr=cls.pg0.local_ip4n,
+                dst_addr=dest_ip4n,
+                teid=teid)
+            cls.vapi.sw_interface_set_l2_bridge(r.sw_if_index, bd_id=teid)
+
+    @classmethod
+    def add_del_shared_mcast_dst_load(cls, is_add):
+        """
+        add or del tunnels sharing the same mcast dst
+        to test gtpu ref_count mechanism
+        """
+        n_shared_dst_tunnels = 20
+        teid_start = 1000
+        teid_end = teid_start + n_shared_dst_tunnels
+        for teid in range(teid_start, teid_end):
+            r = cls.vapi.gtpu_add_del_tunnel(
+                src_addr=cls.pg0.local_ip4n,
+                dst_addr=cls.mcast_ip4n,
+                mcast_sw_if_index=1,
+                teid=teid,
+                is_add=is_add)
+            if r.sw_if_index == 0xffffffff:
+                raise "bad sw_if_index"
+
+    @classmethod
+    def add_shared_mcast_dst_load(cls):
+        cls.add_del_shared_mcast_dst_load(is_add=1)
+
+    @classmethod
+    def del_shared_mcast_dst_load(cls):
+        cls.add_del_shared_mcast_dst_load(is_add=0)
+
+    @classmethod
+    def add_del_mcast_tunnels_load(cls, is_add):
+        """
+        add or del tunnels to test gtpu stability
+        """
+        n_distinct_dst_tunnels = 20
+        ip_range_start = 10
+        ip_range_end = ip_range_start + n_distinct_dst_tunnels
+        for dest_ip4n in ip4n_range(cls.mcast_ip4n, ip_range_start,
+                                    ip_range_end):
+            teid = bytearray(dest_ip4n)[3]
+            cls.vapi.gtpu_add_del_tunnel(
+                src_addr=cls.pg0.local_ip4n,
+                dst_addr=dest_ip4n,
+                mcast_sw_if_index=1,
+                teid=teid,
+                is_add=is_add)
+
+    @classmethod
+    def add_mcast_tunnels_load(cls):
+        cls.add_del_mcast_tunnels_load(is_add=1)
+
+    @classmethod
+    def del_mcast_tunnels_load(cls):
+        cls.add_del_mcast_tunnels_load(is_add=0)
+
+    # Class method to start the GTPU test case.
+    #  Overrides setUpClass method in VppTestCase class.
+    #  Python try..except statement is used to ensure that the tear down of
+    #  the class will be executed even if exception is raised.
+    #  @param cls The class pointer.
+    @classmethod
+    def setUpClass(cls):
+        super(TestGtpu, cls).setUpClass()
+
+        try:
+            cls.dport = 2152
+            cls.gtp_type = 0xff
+
+            # Create 2 pg interfaces.
+            cls.create_pg_interfaces(range(4))
+            for pg in cls.pg_interfaces:
+                pg.admin_up()
+
+            # Configure IPv4 addresses on VPP pg0.
+            cls.pg0.config_ip4()
+
+            # Resolve MAC address for VPP's IP address on pg0.
+            cls.pg0.resolve_arp()
+
+            # Our Multicast address
+            cls.mcast_ip4 = '239.1.1.1'
+            cls.mcast_ip4n = socket.inet_pton(socket.AF_INET, cls.mcast_ip4)
+            iplong = atol(cls.mcast_ip4)
+            cls.mcast_mac = "01:00:5e:%02x:%02x:%02x" % (
+                (iplong >> 16) & 0x7F, (iplong >> 8) & 0xFF, iplong & 0xFF)
+
+            # Create GTPU VTEP on VPP pg0, and put gtpu_tunnel0 and pg1
+            #  into BD.
+            cls.single_tunnel_bd = 11
+            r = cls.vapi.gtpu_add_del_tunnel(
+                src_addr=cls.pg0.local_ip4n,
+                dst_addr=cls.pg0.remote_ip4n,
+                teid=cls.single_tunnel_bd)
+            cls.vapi.sw_interface_set_l2_bridge(r.sw_if_index,
+                                                bd_id=cls.single_tunnel_bd)
+            cls.vapi.sw_interface_set_l2_bridge(cls.pg1.sw_if_index,
+                                                bd_id=cls.single_tunnel_bd)
+
+            # Setup teid 2 to test multicast flooding
+            cls.n_ucast_tunnels = 10
+            cls.mcast_flood_bd = 12
+            cls.create_gtpu_flood_test_bd(cls.mcast_flood_bd,
+                                          cls.n_ucast_tunnels)
+            r = cls.vapi.gtpu_add_del_tunnel(
+                src_addr=cls.pg0.local_ip4n,
+                dst_addr=cls.mcast_ip4n,
+                mcast_sw_if_index=1,
+                teid=cls.mcast_flood_bd)
+            cls.vapi.sw_interface_set_l2_bridge(r.sw_if_index,
+                                                bd_id=cls.mcast_flood_bd)
+            cls.vapi.sw_interface_set_l2_bridge(cls.pg2.sw_if_index,
+                                                bd_id=cls.mcast_flood_bd)
+
+            # Add and delete mcast tunnels to check stability
+            cls.add_shared_mcast_dst_load()
+            cls.add_mcast_tunnels_load()
+            cls.del_shared_mcast_dst_load()
+            cls.del_mcast_tunnels_load()
+
+            # Setup teid 3 to test unicast flooding
+            cls.ucast_flood_bd = 13
+            cls.create_gtpu_flood_test_bd(cls.ucast_flood_bd,
+                                          cls.n_ucast_tunnels)
+            cls.vapi.sw_interface_set_l2_bridge(cls.pg3.sw_if_index,
+                                                bd_id=cls.ucast_flood_bd)
+        except Exception:
+            super(TestGtpu, cls).tearDownClass()
+            raise
+
+    # Method to define VPP actions before tear down of the test case.
+    #  Overrides tearDown method in VppTestCase class.
+    #  @param self The object pointer.
+    def tearDown(self):
+        super(TestGtpu, self).tearDown()
+        if not self.vpp_dead:
+            self.logger.info(self.vapi.cli("show bridge-domain 11 detail"))
+            self.logger.info(self.vapi.cli("show bridge-domain 12 detail"))
+            self.logger.info(self.vapi.cli("show bridge-domain 13 detail"))
+            self.logger.info(self.vapi.cli("show int"))
+            self.logger.info(self.vapi.cli("show gtpu tunnel"))
+            self.logger.info(self.vapi.cli("show trace"))
+
+
+if __name__ == '__main__':
+    unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 83c4a83b..ff8e2cfd 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -1775,3 +1775,35 @@ class VppPapiProvider(object):
                 'is_translation': is_translation,
                 'mtu': mtu
             })
+
+    def gtpu_add_del_tunnel(
+            self,
+            src_addr,
+            dst_addr,
+            is_add=1,
+            is_ipv6=0,
+            mcast_sw_if_index=0xFFFFFFFF,
+            encap_vrf_id=0,
+            decap_next_index=0xFFFFFFFF,
+            teid=0):
+        """
+
+        :param is_add:  (Default value = 1)
+        :param is_ipv6:  (Default value = 0)
+        :param src_addr:
+        :param dst_addr:
+        :param mcast_sw_if_index:  (Default value = 0xFFFFFFFF)
+        :param encap_vrf_id:  (Default value = 0)
+        :param decap_next_index:  (Default value = 0xFFFFFFFF)
+        :param teid:  (Default value = 0)
+
+        """
+        return self.api(self.papi.gtpu_add_del_tunnel,
+                        {'is_add': is_add,
+                         'is_ipv6': is_ipv6,
+                         'src_address': src_addr,
+                         'dst_address': dst_addr,
+                         'mcast_sw_if_index': mcast_sw_if_index,
+                         'encap_vrf_id': encap_vrf_id,
+                         'decap_next_index': decap_next_index,
+                         'teid': teid})
-- 
cgit 1.2.3-korg


From 5c749734b14c2d3be8689b0c5b72ae8d1ddec099 Mon Sep 17 00:00:00 2001
From: Ole Troan 
Date: Mon, 13 Mar 2017 13:39:52 +0100
Subject: Flowprobe: Stateful flows and IPv6, L4 recording

Change-Id: I67839281623721bf42f0a918a53356143d9dc78a
Signed-off-by: Ole Troan 
Signed-off-by: Pavel Kotucek 
Signed-off-by: Ole Troan 
---
 MAINTAINERS                                     |    8 +-
 doxygen/user_doc.md                             |    2 +-
 src/configure.ac                                |    2 +-
 src/plugins/Makefile.am                         |    4 +-
 src/plugins/flowperpkt.am                       |   38 -
 src/plugins/flowperpkt/flowperpkt.api           |   40 -
 src/plugins/flowperpkt/flowperpkt.c             |  621 -------------
 src/plugins/flowperpkt/flowperpkt.h             |   90 --
 src/plugins/flowperpkt/flowperpkt_all_api_h.h   |   18 -
 src/plugins/flowperpkt/flowperpkt_msg_enum.h    |   31 -
 src/plugins/flowperpkt/flowperpkt_plugin_doc.md |   13 -
 src/plugins/flowperpkt/flowperpkt_test.c        |  200 ----
 src/plugins/flowperpkt/l2_node.c                |  561 ------------
 src/plugins/flowperpkt/node.c                   |  574 ------------
 src/plugins/flowprobe.am                        |   37 +
 src/plugins/flowprobe/flowprobe.api             |   40 +
 src/plugins/flowprobe/flowprobe.c               | 1122 +++++++++++++++++++++++
 src/plugins/flowprobe/flowprobe.h               |  167 ++++
 src/plugins/flowprobe/flowprobe_all_api_h.h     |   18 +
 src/plugins/flowprobe/flowprobe_msg_enum.h      |   31 +
 src/plugins/flowprobe/flowprobe_plugin_doc.md   |   13 +
 src/plugins/flowprobe/flowprobe_test.c          |  263 ++++++
 src/plugins/flowprobe/node.c                    |  998 ++++++++++++++++++++
 src/plugins/ioam/analyse/ioam_summary_export.c  |    3 +-
 src/plugins/ioam/udp-ping/udp_ping_export.c     |    3 +-
 src/plugins/snat/snat_ipfix_logging.c           |    6 +-
 src/vnet/flow/flow_api.c                        |    2 +-
 src/vnet/flow/flow_report.c                     |   21 +-
 src/vnet/flow/flow_report.h                     |    3 +-
 src/vnet/flow/flow_report_classify.c            |    2 +-
 src/vnet/ip/ip6_packet.h                        |    6 +
 src/vpp-api/java/jvpp/gen/jvpp_gen.py           |    2 +-
 test/test_flowperpkt.py                         |  173 ----
 test/test_flowprobe.py                          |  967 +++++++++++++++++++
 34 files changed, 3695 insertions(+), 2384 deletions(-)
 delete mode 100644 src/plugins/flowperpkt.am
 delete mode 100644 src/plugins/flowperpkt/flowperpkt.api
 delete mode 100644 src/plugins/flowperpkt/flowperpkt.c
 delete mode 100644 src/plugins/flowperpkt/flowperpkt.h
 delete mode 100644 src/plugins/flowperpkt/flowperpkt_all_api_h.h
 delete mode 100644 src/plugins/flowperpkt/flowperpkt_msg_enum.h
 delete mode 100644 src/plugins/flowperpkt/flowperpkt_plugin_doc.md
 delete mode 100644 src/plugins/flowperpkt/flowperpkt_test.c
 delete mode 100644 src/plugins/flowperpkt/l2_node.c
 delete mode 100644 src/plugins/flowperpkt/node.c
 create mode 100644 src/plugins/flowprobe.am
 create mode 100644 src/plugins/flowprobe/flowprobe.api
 create mode 100644 src/plugins/flowprobe/flowprobe.c
 create mode 100644 src/plugins/flowprobe/flowprobe.h
 create mode 100644 src/plugins/flowprobe/flowprobe_all_api_h.h
 create mode 100644 src/plugins/flowprobe/flowprobe_msg_enum.h
 create mode 100644 src/plugins/flowprobe/flowprobe_plugin_doc.md
 create mode 100644 src/plugins/flowprobe/flowprobe_test.c
 create mode 100644 src/plugins/flowprobe/node.c
 delete mode 100644 test/test_flowperpkt.py
 create mode 100644 test/test_flowprobe.py

(limited to 'src/plugins/Makefile.am')

diff --git a/MAINTAINERS b/MAINTAINERS
index 2f198319..0f5dfc04 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -112,10 +112,10 @@ VNET VXLAN
 M:	John Lo 
 F:	src/vnet/vxlan/
 
-Plugin - flowperpkt
-M:	Dave Barach 
-F:	src/plugins/flowperpkt/
-F:	src/plugins/flowperpkt.am
+Plugin - flowprobe
+M:	Ole Troan 
+F:	src/plugins/flowprobe/
+F:	src/plugins/flowprobe.am
 
 Plugin - SIXRD
 M:	Ole Troan 
diff --git a/doxygen/user_doc.md b/doxygen/user_doc.md
index d052c53b..becc2e0a 100644
--- a/doxygen/user_doc.md
+++ b/doxygen/user_doc.md
@@ -11,7 +11,7 @@ Several modules provide operational, dataplane-user focused documentation.
 - @subpage lldp_doc
 - @subpage map_doc
 - @subpage dpdk_crypto_ipsec_doc
-- @subpage flowperpkt_plugin_doc
+- @subpage flowprobe_plugin_doc
 - @subpage qos_doc
 - @subpage span_doc
 - @subpage srv6_doc
diff --git a/src/configure.ac b/src/configure.ac
index 2a907b2f..173b3153 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -146,7 +146,7 @@ AC_SUBST(AR_FLAGS)
 # Please keep alphabetical order
 PLUGIN_ENABLED(acl)
 PLUGIN_ENABLED(dpdk)
-PLUGIN_ENABLED(flowperpkt)
+PLUGIN_ENABLED(flowprobe)
 PLUGIN_ENABLED(gtpu)
 PLUGIN_ENABLED(ila)
 PLUGIN_ENABLED(ioam)
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 388c9ad2..f26d0fd2 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -37,8 +37,8 @@ if ENABLE_DPDK_PLUGIN
 include dpdk.am
 endif
 
-if ENABLE_FLOWPERPKT_PLUGIN
-include flowperpkt.am
+if ENABLE_FLOWPROBE_PLUGIN
+include flowprobe.am
 endif
 
 
diff --git a/src/plugins/flowperpkt.am b/src/plugins/flowperpkt.am
deleted file mode 100644
index a400603a..00000000
--- a/src/plugins/flowperpkt.am
+++ /dev/null
@@ -1,38 +0,0 @@
-
-# Copyright (c)  
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-vppplugins_LTLIBRARIES += flowperpkt_plugin.la
-vppapitestplugins_LTLIBRARIES += flowperpkt_test_plugin.la
-
-flowperpkt_plugin_la_SOURCES = flowperpkt/flowperpkt.c  \
-        flowperpkt/l2_node.c \
-        flowperpkt/node.c \
-	flowperpkt/flowperpkt_plugin.api.h
-
-BUILT_SOURCES +=				\
-	flowperpkt/flowperpkt.api.h		\
-	flowperpkt/flowperpkt.api.json
-
-noinst_HEADERS +=				\
-  flowperpkt/flowperpkt_all_api_h.h		\
-  flowperpkt/flowperpkt_msg_enum.h		\
-  flowperpkt/flowperpkt.api.h
-
-flowperpkt_test_plugin_la_SOURCES = \
-  flowperpkt/flowperpkt_test.c			\
-  flowperpkt/flowperpkt_plugin.api.h
-
-API_FILES += flowperpkt/flowperpkt.api
-
-# vi:syntax=automake
diff --git a/src/plugins/flowperpkt/flowperpkt.api b/src/plugins/flowperpkt/flowperpkt.api
deleted file mode 100644
index 3ff92dca..00000000
--- a/src/plugins/flowperpkt/flowperpkt.api
+++ /dev/null
@@ -1,40 +0,0 @@
-/* Define a simple enable-disable binary API to control the feature */
-
-/** \file
-    This file defines the vpp control-plane API messages
-    used to control the flowperpkt plugin
-*/
-
-/** \brief Enable / disable per-packet IPFIX recording on an interface
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_add - add address if non-zero, else delete
-    @param is_ipv6 - if non-zero the address is ipv6, else ipv4
-    @param sw_if_index - index of the interface
-*/
-autoreply manual_print define flowperpkt_tx_interface_add_del
-{
-  /* Client identifier, set from api_main.my_client_index */
-  u32 client_index;
-
-  /* Arbitrary context, so client can match reply to request */
-  u32 context;
-
-  /* Enable / disable the feature */
-  u8 is_add;
-  u8 which;  /* 0 = ipv4, 1 = l2, 2 = ipv6 */
-
-  /* Interface handle */
-  u32 sw_if_index;
-};
-
-autoreply define flowperpkt_params
-{
-  u32 client_index;
-  u32 context;
-  u8 record_l2;
-  u8 record_l3;
-  u8 record_l4;
-  u32 active_timer;  /* ~0 is off, 0 is default */
-  u32 passive_timer; /* ~0 is off, 0 is default */
-};
diff --git a/src/plugins/flowperpkt/flowperpkt.c b/src/plugins/flowperpkt/flowperpkt.c
deleted file mode 100644
index 3e5fc8b0..00000000
--- a/src/plugins/flowperpkt/flowperpkt.c
+++ /dev/null
@@ -1,621 +0,0 @@
-/*
- * flowperpkt.c - per-packet data capture flow report plugin
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * @file
- * @brief Per-packet IPFIX flow record generator plugin
- *
- * This file implements vpp plugin registration mechanics,
- * debug CLI, and binary API handling.
- */
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-/* define message IDs */
-#include 
-
-/* define message structures */
-#define vl_typedefs
-#include 
-#undef vl_typedefs
-
-/* define generated endian-swappers */
-#define vl_endianfun
-#include 
-#undef vl_endianfun
-
-/* instantiate all the print functions we know about */
-#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
-#define vl_printfun
-#include 
-#undef vl_printfun
-
-flowperpkt_main_t flowperpkt_main;
-
-/* Get the API version number */
-#define vl_api_version(n,v) static u32 api_version=(v);
-#include 
-#undef vl_api_version
-
-#define REPLY_MSG_ID_BASE fm->msg_id_base
-#include 
-
-/* Define the per-interface configurable features */
-/* *INDENT-OFF* */
-VNET_FEATURE_INIT (flow_perpacket_ipv4, static) =
-{
-  .arc_name = "ip4-output",
-  .node_name = "flowperpkt-ipv4",
-  .runs_before = VNET_FEATURES ("interface-output"),
-};
-
-VNET_FEATURE_INIT (flow_perpacket_l2, static) =
-{
-  .arc_name = "interface-output",
-  .node_name = "flowperpkt-l2",
-  .runs_before = VNET_FEATURES ("interface-tx"),
-};
-/* *INDENT-ON* */
-
-/* Macro to finish up custom dump fns */
-#define FINISH                                  \
-    vec_add1 (s, 0);                            \
-    vl_print (handle, (char *)s);               \
-    vec_free (s);                               \
-    return handle;
-
-/**
- * @brief Create an IPFIX template packet rewrite string
- * @param frm flow_report_main_t *
- * @param fr flow_report_t *
- * @param collector_address ip4_address_t * the IPFIX collector address
- * @param src_address ip4_address_t * the source address we should use
- * @param collector_port u16 the collector port we should use, host byte order
- * @returns u8 * vector containing the indicated IPFIX template packet
- */
-static inline u8 *
-flowperpkt_template_rewrite_inline (flow_report_main_t * frm,
-				    flow_report_t * fr,
-				    ip4_address_t * collector_address,
-				    ip4_address_t * src_address,
-				    u16 collector_port, int variant)
-{
-  ip4_header_t *ip;
-  udp_header_t *udp;
-  ipfix_message_header_t *h;
-  ipfix_set_header_t *s;
-  ipfix_template_header_t *t;
-  ipfix_field_specifier_t *f;
-  ipfix_field_specifier_t *first_field;
-  u8 *rewrite = 0;
-  ip4_ipfix_template_packet_t *tp;
-  u32 field_count = 0;
-  flow_report_stream_t *stream;
-  flowperpkt_main_t *fm = &flowperpkt_main;
-
-  stream = &frm->streams[fr->stream_index];
-
-  if (variant == FLOW_VARIANT_IPV4)
-    {
-      /*
-       * ip4 Supported Fields:
-       *
-       * ingressInterface, TLV type 10, u32
-       * egressInterface, TLV type 14, u32
-       * sourceIpv4Address, TLV type 8, u32
-       * destinationIPv4Address, TLV type 12, u32
-       * ipClassOfService, TLV type 5, u8
-       * flowStartNanoseconds, TLV type 156, dateTimeNanoseconds (f64)
-       *   Implementation: f64 nanoseconds since VPP started
-       *   warning: wireshark doesn't really understand this TLV
-       * dataLinkFrameSize, TLV type 312, u16
-       *   warning: wireshark doesn't understand this TLV at all
-       */
-
-      /* Currently 7 fields */
-      field_count += 7;
-
-      /* allocate rewrite space */
-      vec_validate_aligned
-	(rewrite,
-	 sizeof (ip4_ipfix_template_packet_t)
-	 + field_count * sizeof (ipfix_field_specifier_t) - 1,
-	 CLIB_CACHE_LINE_BYTES);
-    }
-  else if (variant == FLOW_VARIANT_L2)
-    {
-      /*
-       * L2 Supported Fields:
-       *
-       * ingressInterface, TLV type 10, u32
-       * egressInterface, TLV type 14, u32
-       * sourceMacAddress, TLV type 56, u8[6] we hope
-       * destinationMacAddress, TLV type 57, u8[6] we hope
-       * ethernetType, TLV type 256, u16
-       * flowStartNanoseconds, TLV type 156, dateTimeNanoseconds (f64)
-       *   Implementation: f64 nanoseconds since VPP started
-       *   warning: wireshark doesn't really understand this TLV
-       * dataLinkFrameSize, TLV type 312, u16
-       *   warning: wireshark doesn't understand this TLV at all
-       */
-
-      /* Currently 7 fields */
-      field_count += 7;
-
-      /* allocate rewrite space */
-      vec_validate_aligned
-	(rewrite,
-	 sizeof (ip4_ipfix_template_packet_t)
-	 + field_count * sizeof (ipfix_field_specifier_t) - 1,
-	 CLIB_CACHE_LINE_BYTES);
-    }
-
-  tp = (ip4_ipfix_template_packet_t *) rewrite;
-  ip = (ip4_header_t *) & tp->ip4;
-  udp = (udp_header_t *) (ip + 1);
-  h = (ipfix_message_header_t *) (udp + 1);
-  s = (ipfix_set_header_t *) (h + 1);
-  t = (ipfix_template_header_t *) (s + 1);
-  first_field = f = (ipfix_field_specifier_t *) (t + 1);
-
-  ip->ip_version_and_header_length = 0x45;
-  ip->ttl = 254;
-  ip->protocol = IP_PROTOCOL_UDP;
-  ip->src_address.as_u32 = src_address->as_u32;
-  ip->dst_address.as_u32 = collector_address->as_u32;
-  udp->src_port = clib_host_to_net_u16 (stream->src_port);
-  udp->dst_port = clib_host_to_net_u16 (collector_port);
-  udp->length = clib_host_to_net_u16 (vec_len (rewrite) - sizeof (*ip));
-
-  /* FIXUP: message header export_time */
-  /* FIXUP: message header sequence_number */
-  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
-
-  /* Add TLVs to the template */
-  if (variant == FLOW_VARIANT_IPV4)
-    {
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , ingressInterface,
-			   4);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , egressInterface,
-			   4);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , sourceIPv4Address,
-			   4);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , destinationIPv4Address, 4);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , ipClassOfService,
-			   1);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , flowStartNanoseconds,
-			   8);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , dataLinkFrameSize,
-			   2);
-      f++;
-    }
-  else if (variant == FLOW_VARIANT_L2)
-    {
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , ingressInterface,
-			   4);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , egressInterface,
-			   4);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , sourceMacAddress,
-			   6);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , destinationMacAddress, 6);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0 /* enterprise */ , ethernetType,
-					  2);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , flowStartNanoseconds,
-			   8);
-      f++;
-      f->e_id_length =
-	ipfix_e_id_length (0 /* enterprise */ , dataLinkFrameSize,
-			   2);
-      f++;
-    }
-
-  /* Extend in the obvious way, right here... */
-
-  /* Back to the template packet... */
-  ip = (ip4_header_t *) & tp->ip4;
-  udp = (udp_header_t *) (ip + 1);
-
-  ASSERT (f - first_field);
-  /* Field count in this template */
-  t->id_count = ipfix_id_count (fr->template_id, f - first_field);
-
-  if (variant == FLOW_VARIANT_IPV4)
-    fm->ipv4_report_id = fr->template_id;
-  else if (variant == FLOW_VARIANT_L2)
-    fm->l2_report_id = fr->template_id;
-
-  /* set length in octets */
-  s->set_id_length =
-    ipfix_set_id_length (2 /* set_id */ , (u8 *) f - (u8 *) s);
-
-  /* message length in octets */
-  h->version_length = version_length ((u8 *) f - (u8 *) h);
-
-  ip->length = clib_host_to_net_u16 ((u8 *) f - (u8 *) ip);
-  ip->checksum = ip4_header_checksum (ip);
-
-  return rewrite;
-}
-
-u8 *
-flowperpkt_template_rewrite_ipv4 (flow_report_main_t * frm,
-				  flow_report_t * fr,
-				  ip4_address_t * collector_address,
-				  ip4_address_t * src_address,
-				  u16 collector_port)
-{
-  return flowperpkt_template_rewrite_inline
-    (frm, fr, collector_address, src_address, collector_port,
-     FLOW_VARIANT_IPV4);
-}
-
-u8 *
-flowperpkt_template_rewrite_l2 (flow_report_main_t * frm,
-				flow_report_t * fr,
-				ip4_address_t * collector_address,
-				ip4_address_t * src_address,
-				u16 collector_port)
-{
-  return flowperpkt_template_rewrite_inline
-    (frm, fr, collector_address, src_address, collector_port,
-     FLOW_VARIANT_L2);
-}
-
-
-/**
- * @brief Flush accumulated data
- * @param frm flow_report_main_t *
- * @param fr flow_report_t *
- * @param f vlib_frame_t *
- *
- * Notes:
- * This function must simply return the incoming frame, or no template packets
- * will be sent.
- */
-vlib_frame_t *
-flowperpkt_data_callback_ipv4 (flow_report_main_t * frm,
-			       flow_report_t * fr,
-			       vlib_frame_t * f, u32 * to_next,
-			       u32 node_index)
-{
-  flowperpkt_flush_callback_ipv4 ();
-  return f;
-}
-
-vlib_frame_t *
-flowperpkt_data_callback_l2 (flow_report_main_t * frm,
-			     flow_report_t * fr,
-			     vlib_frame_t * f, u32 * to_next, u32 node_index)
-{
-  flowperpkt_flush_callback_l2 ();
-  return f;
-}
-
-/**
- * @brief configure / deconfigure the IPFIX flow-per-packet
- * @param fm flowperpkt_main_t * fm
- * @param sw_if_index u32 the desired interface
- * @param is_add int 1 to enable the feature, 0 to disable it
- * @returns 0 if successful, non-zero otherwise
- */
-
-static int flowperpkt_tx_interface_add_del_feature
-  (flowperpkt_main_t * fm, u32 sw_if_index, int which, int is_add)
-{
-  flow_report_main_t *frm = &flow_report_main;
-  vnet_flow_report_add_del_args_t _a, *a = &_a;
-  int rv;
-
-  if (which == FLOW_VARIANT_IPV4 && !fm->ipv4_report_created)
-    {
-      memset (a, 0, sizeof (*a));
-      a->rewrite_callback = flowperpkt_template_rewrite_ipv4;
-      a->flow_data_callback = flowperpkt_data_callback_ipv4;
-      a->is_add = 1;
-      a->domain_id = 1;		/*$$$$ config parameter */
-      a->src_port = 4739;	/*$$$$ config parameter */
-      fm->ipv4_report_created = 1;
-
-      rv = vnet_flow_report_add_del (frm, a);
-      if (rv)
-	{
-	  clib_warning ("vnet_flow_report_add_del returned %d", rv);
-	  return -1;
-	}
-    }
-  else if (which == FLOW_VARIANT_L2 && !fm->l2_report_created)
-    {
-      memset (a, 0, sizeof (*a));
-      a->rewrite_callback = flowperpkt_template_rewrite_l2;
-      a->flow_data_callback = flowperpkt_data_callback_l2;
-      a->is_add = 1;
-      a->domain_id = 1;		/*$$$$ config parameter */
-      a->src_port = 4739;	/*$$$$ config parameter */
-      fm->l2_report_created = 1;
-
-      rv = vnet_flow_report_add_del (frm, a);
-      if (rv)
-	{
-	  clib_warning ("vnet_flow_report_add_del returned %d", rv);
-	  return -1;
-	}
-    }
-
-  if (which == FLOW_VARIANT_IPV4)
-    vnet_feature_enable_disable ("ip4-output", "flowperpkt-ipv4",
-				 sw_if_index, is_add, 0, 0);
-  else if (which == FLOW_VARIANT_L2)
-    vnet_feature_enable_disable ("interface-output", "flowperpkt-l2",
-				 sw_if_index, is_add, 0, 0);
-
-  return 0;
-}
-
-/**
- * @brief API message handler
- * @param mp vl_api_flowperpkt_tx_interface_add_del_t * mp the api message
- */
-void vl_api_flowperpkt_tx_interface_add_del_t_handler
-  (vl_api_flowperpkt_tx_interface_add_del_t * mp)
-{
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  vl_api_flowperpkt_tx_interface_add_del_reply_t *rmp;
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  int rv = 0;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  if (mp->which != FLOW_VARIANT_IPV4 && mp->which != FLOW_VARIANT_L2)
-    {
-      rv = VNET_API_ERROR_UNIMPLEMENTED;
-      goto out;
-    }
-
-  rv = flowperpkt_tx_interface_add_del_feature (fm, sw_if_index, mp->which,
-						mp->is_add);
-out:
-  BAD_SW_IF_INDEX_LABEL;
-
-  REPLY_MACRO (VL_API_FLOWPERPKT_TX_INTERFACE_ADD_DEL_REPLY);
-}
-
-/**
- * @brief API message custom-dump function
- * @param mp vl_api_flowperpkt_tx_interface_add_del_t * mp the api message
- * @param handle void * print function handle
- * @returns u8 * output string
- */
-static void *vl_api_flowperpkt_tx_interface_add_del_t_print
-  (vl_api_flowperpkt_tx_interface_add_del_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: flowperpkt_tx_interface_add_del ");
-  s = format (s, "sw_if_index %d is_add %d which %d ",
-	      clib_host_to_net_u32 (mp->sw_if_index),
-	      (int) mp->is_add, (int) mp->which);
-  FINISH;
-}
-
-/* List of message types that this plugin understands */
-#define foreach_flowperpkt_plugin_api_msg                           \
-_(FLOWPERPKT_TX_INTERFACE_ADD_DEL, flowperpkt_tx_interface_add_del)
-
-/* *INDENT-OFF* */
-VLIB_PLUGIN_REGISTER () = {
-    .version = VPP_BUILD_VER,
-    .description = "Flow per Packet",
-};
-/* *INDENT-ON* */
-
-static clib_error_t *
-flowperpkt_tx_interface_add_del_feature_command_fn (vlib_main_t * vm,
-						    unformat_input_t * input,
-						    vlib_cli_command_t * cmd)
-{
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  u32 sw_if_index = ~0;
-  int is_add = 1;
-  u8 which = FLOW_VARIANT_IPV4;
-
-  int rv;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "disable"))
-	is_add = 0;
-      else if (unformat (input, "%U", unformat_vnet_sw_interface,
-			 fm->vnet_main, &sw_if_index));
-      else if (unformat (input, "l2"))
-	which = FLOW_VARIANT_L2;
-      else
-	break;
-    }
-
-  if (sw_if_index == ~0)
-    return clib_error_return (0, "Please specify an interface...");
-
-  rv =
-    flowperpkt_tx_interface_add_del_feature (fm, sw_if_index, which, is_add);
-  switch (rv)
-    {
-    case 0:
-      break;
-
-    case VNET_API_ERROR_INVALID_SW_IF_INDEX:
-      return clib_error_return
-	(0, "Invalid interface, only works on physical ports");
-      break;
-
-    case VNET_API_ERROR_UNIMPLEMENTED:
-      return clib_error_return (0, "ip6 not supported");
-      break;
-
-    default:
-      return clib_error_return (0, "flowperpkt_enable_disable returned %d",
-				rv);
-    }
-  return 0;
-}
-
-/*?
- * 'flowperpkt feature add-del' commands to enable/disable
- * per-packet IPFIX flow record generation on an interface
- *
- * @cliexpar
- * @parblock
- * To enable per-packet IPFIX flow-record generation on an interface:
- * @cliexcmd{flowperpkt feature add-del GigabitEthernet2/0/0}
- *
- * To disable per-packet IPFIX flow-record generation on an interface:
- * @cliexcmd{flowperpkt feature add-del GigabitEthernet2/0/0 disable}
- * @cliexend
- * @endparblock
-?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (flowperpkt_enable_disable_command, static) = {
-    .path = "flowperpkt feature add-del",
-    .short_help =
-    "flowperpkt feature add-del  [disable]",
-    .function = flowperpkt_tx_interface_add_del_feature_command_fn,
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Set up the API message handling tables
- * @param vm vlib_main_t * vlib main data structure pointer
- * @returns 0 to indicate all is well
- */
-static clib_error_t *
-flowperpkt_plugin_api_hookup (vlib_main_t * vm)
-{
-  flowperpkt_main_t *fm = &flowperpkt_main;
-#define _(N,n)                                                  \
-    vl_msg_api_set_handlers((VL_API_##N + fm->msg_id_base),     \
-                           #n,					\
-                           vl_api_##n##_t_handler,              \
-                           vl_noop_handler,                     \
-                           vl_api_##n##_t_endian,               \
-                           vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
-  foreach_flowperpkt_plugin_api_msg;
-#undef _
-
-  return 0;
-}
-
-#define vl_msg_name_crc_list
-#include 
-#undef vl_msg_name_crc_list
-
-static void
-setup_message_id_table (flowperpkt_main_t * fm, api_main_t * am)
-{
-#define _(id,n,crc) \
-  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + fm->msg_id_base);
-  foreach_vl_msg_name_crc_flowperpkt;
-#undef _
-}
-
-/**
- * @brief Set up the API message handling tables
- * @param vm vlib_main_t * vlib main data structure pointer
- * @returns 0 to indicate all is well, or a clib_error_t
- */
-static clib_error_t *
-flowperpkt_init (vlib_main_t * vm)
-{
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  vlib_thread_main_t *tm = &vlib_thread_main;
-  clib_error_t *error = 0;
-  u32 num_threads;
-  u8 *name;
-
-  fm->vnet_main = vnet_get_main ();
-
-  /* Construct the API name */
-  name = format (0, "flowperpkt_%08x%c", api_version, 0);
-
-  /* Ask for a correctly-sized block of API message decode slots */
-  fm->msg_id_base = vl_msg_api_get_msg_ids
-    ((char *) name, VL_MSG_FIRST_AVAILABLE);
-
-  /* Hook up message handlers */
-  error = flowperpkt_plugin_api_hookup (vm);
-
-  /* Add our API messages to the global name_crc hash table */
-  setup_message_id_table (fm, &api_main);
-
-  vec_free (name);
-
-  /* Decide how many worker threads we have */
-  num_threads = 1 /* main thread */  + tm->n_threads;
-
-  /* Allocate per worker thread vectors */
-  vec_validate (fm->ipv4_buffers_per_worker, num_threads - 1);
-  vec_validate (fm->l2_buffers_per_worker, num_threads - 1);
-  vec_validate (fm->ipv4_frames_per_worker, num_threads - 1);
-  vec_validate (fm->l2_frames_per_worker, num_threads - 1);
-  vec_validate (fm->ipv4_next_record_offset_per_worker, num_threads - 1);
-  vec_validate (fm->l2_next_record_offset_per_worker, num_threads - 1);
-
-  /* Set up time reference pair */
-  fm->vlib_time_0 = vlib_time_now (vm);
-  fm->nanosecond_time_0 = unix_time_now_nsec ();
-
-  return error;
-}
-
-VLIB_INIT_FUNCTION (flowperpkt_init);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/flowperpkt/flowperpkt.h b/src/plugins/flowperpkt/flowperpkt.h
deleted file mode 100644
index 20f6939d..00000000
--- a/src/plugins/flowperpkt/flowperpkt.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * flowperpkt.h - skeleton vpp engine plug-in header file
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_flowperpkt_h__
-#define __included_flowperpkt_h__
-
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-
-/**
- * @file
- * @brief flow-per-packet plugin header file
- */
-typedef struct
-{
-  /** API message ID base */
-  u16 msg_id_base;
-
-  /** Have the reports [templates] been created? */
-  int ipv4_report_created;
-  int l2_report_created;
-
-  /** stream/template IDs */
-  u16 ipv4_report_id;
-  u16 l2_report_id;
-
-  /** ipfix buffers under construction, per-worker thread */
-  vlib_buffer_t **ipv4_buffers_per_worker;
-  vlib_buffer_t **l2_buffers_per_worker;
-
-  /** frames containing ipfix buffers, per-worker thread */
-  vlib_frame_t **ipv4_frames_per_worker;
-  vlib_frame_t **l2_frames_per_worker;
-
-  /** next record offset, per worker thread */
-  u16 *ipv4_next_record_offset_per_worker;
-  u16 *l2_next_record_offset_per_worker;
-
-  /** Time reference pair */
-  u64 nanosecond_time_0;
-  f64 vlib_time_0;
-
-  /** convenience vlib_main_t pointer */
-  vlib_main_t *vlib_main;
-  /** convenience vnet_main_t pointer */
-  vnet_main_t *vnet_main;
-} flowperpkt_main_t;
-
-typedef enum
-{
-  FLOW_VARIANT_IPV4,
-  FLOW_VARIANT_L2,
-  FLOW_N_VARIANTS,
-} flowperpkt_variant_t;
-
-extern flowperpkt_main_t flowperpkt_main;
-
-extern vlib_node_registration_t flowperpkt_ipv4_node;
-
-void flowperpkt_flush_callback_ipv4 (void);
-void flowperpkt_flush_callback_l2 (void);
-
-#endif /* __included_flowperpkt_h__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/flowperpkt/flowperpkt_all_api_h.h b/src/plugins/flowperpkt/flowperpkt_all_api_h.h
deleted file mode 100644
index 329c375a..00000000
--- a/src/plugins/flowperpkt/flowperpkt_all_api_h.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * flowperpkt_all_api_h.h - plug-in api #include file
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/* Include the generated file, see BUILT_SOURCES in Makefile.am */
-#include 
diff --git a/src/plugins/flowperpkt/flowperpkt_msg_enum.h b/src/plugins/flowperpkt/flowperpkt_msg_enum.h
deleted file mode 100644
index 3177e77a..00000000
--- a/src/plugins/flowperpkt/flowperpkt_msg_enum.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * flowperpkt_msg_enum.h - vpp engine plug-in message enumeration
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef included_flowperpkt_msg_enum_h
-#define included_flowperpkt_msg_enum_h
-
-#include 
-
-#define vl_msg_id(n,h) n,
-typedef enum
-{
-#include 
-  /* We'll want to know how many messages IDs we need... */
-  VL_MSG_FIRST_AVAILABLE,
-} vl_msg_id_t;
-#undef vl_msg_id
-
-#endif /* included_flowperpkt_msg_enum_h */
diff --git a/src/plugins/flowperpkt/flowperpkt_plugin_doc.md b/src/plugins/flowperpkt/flowperpkt_plugin_doc.md
deleted file mode 100644
index ed76c45c..00000000
--- a/src/plugins/flowperpkt/flowperpkt_plugin_doc.md
+++ /dev/null
@@ -1,13 +0,0 @@
-Per-packet IPFIX flow record plugin    {#flowperpkt_plugin_doc}
-===================================
-
-## Introduction
-
-This plugin generates one ipfix record entry per packet transmitted
-on interfaces which have the feature enabled
-
-## Sample configuration
-
-set ipfix exporter collector 192.168.6.2 src 192.168.6.1 template-interval 20 port 4739 path-mtu 1500
-
-flowperpkt feature add-del GigabitEthernet2/3/0
diff --git a/src/plugins/flowperpkt/flowperpkt_test.c b/src/plugins/flowperpkt/flowperpkt_test.c
deleted file mode 100644
index 972a3b07..00000000
--- a/src/plugins/flowperpkt/flowperpkt_test.c
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * flowperpkt.c - skeleton vpp-api-test plug-in
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define __plugin_msg_base flowperpkt_test_main.msg_id_base
-#include 
-
-/**
- * @file vpp_api_test plugin
- */
-
-uword unformat_sw_if_index (unformat_input_t * input, va_list * args);
-
-/* Declare message IDs */
-#include 
-
-/* define message structures */
-#define vl_typedefs
-#include 
-#undef vl_typedefs
-
-/* declare message handlers for each api */
-
-#define vl_endianfun		/* define message structures */
-#include 
-#undef vl_endianfun
-
-/* instantiate all the print functions we know about */
-#define vl_print(handle, ...)
-#define vl_printfun
-#include 
-#undef vl_printfun
-
-/* Get the API version number. */
-#define vl_api_version(n,v) static u32 api_version=(v);
-#include 
-#undef vl_api_version
-
-typedef struct
-{
-    /** API message ID base */
-  u16 msg_id_base;
-    /** vat_main_t pointer */
-  vat_main_t *vat_main;
-} flowperpkt_test_main_t;
-
-flowperpkt_test_main_t flowperpkt_test_main;
-
-#define foreach_standard_reply_retval_handler   \
-_(flowperpkt_tx_interface_add_del_reply)
-
-#define _(n)                                            \
-    static void vl_api_##n##_t_handler                  \
-    (vl_api_##n##_t * mp)                               \
-    {                                                   \
-        vat_main_t * vam = flowperpkt_test_main.vat_main;   \
-        i32 retval = ntohl(mp->retval);                 \
-        if (vam->async_mode) {                          \
-            vam->async_errors += (retval < 0);          \
-        } else {                                        \
-            vam->retval = retval;                       \
-            vam->result_ready = 1;                      \
-        }                                               \
-    }
-foreach_standard_reply_retval_handler;
-#undef _
-
-/*
- * Table of message reply handlers, must include boilerplate handlers
- * we just generated
- */
-#define foreach_vpe_api_reply_msg               \
-_(FLOWPERPKT_TX_INTERFACE_ADD_DEL_REPLY,        \
-  flowperpkt_tx_interface_add_del_reply)
-
-static int
-api_flowperpkt_tx_interface_add_del (vat_main_t * vam)
-{
-  unformat_input_t *i = vam->input;
-  int enable_disable = 1;
-  u8 which = 0;			/* ipv4 by default */
-  u32 sw_if_index = ~0;
-  vl_api_flowperpkt_tx_interface_add_del_t *mp;
-  int ret;
-
-  /* Parse args required to build the message */
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
-	;
-      else if (unformat (i, "sw_if_index %d", &sw_if_index))
-	;
-      else if (unformat (i, "disable"))
-	enable_disable = 0;
-      else if (unformat (i, "l2"))
-	which = 1;
-      else
-	break;
-    }
-
-  if (sw_if_index == ~0)
-    {
-      errmsg ("missing interface name / explicit sw_if_index number \n");
-      return -99;
-    }
-
-  /* Construct the API message */
-  M (FLOWPERPKT_TX_INTERFACE_ADD_DEL, mp);
-  mp->sw_if_index = ntohl (sw_if_index);
-  mp->is_add = enable_disable;
-  mp->which = which;
-
-  /* send it... */
-  S (mp);
-
-  /* Wait for a reply... */
-  W (ret);
-  return ret;
-}
-
-/*
- * List of messages that the api test plugin sends,
- * and that the data plane plugin processes
- */
-#define foreach_vpe_api_msg \
-_(flowperpkt_tx_interface_add_del, " [disable]")
-
-static void
-flowperpkt_vat_api_hookup (vat_main_t * vam)
-{
-  flowperpkt_test_main_t *sm = &flowperpkt_test_main;
-  /* Hook up handlers for replies from the data plane plug-in */
-#define _(N,n)                                                  \
-    vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),     \
-                           #n,                                  \
-                           vl_api_##n##_t_handler,              \
-                           vl_noop_handler,                     \
-                           vl_api_##n##_t_endian,               \
-                           vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
-  foreach_vpe_api_reply_msg;
-#undef _
-
-  /* API messages we can send */
-#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
-  foreach_vpe_api_msg;
-#undef _
-
-  /* Help strings */
-#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
-  foreach_vpe_api_msg;
-#undef _
-}
-
-clib_error_t *
-vat_plugin_register (vat_main_t * vam)
-{
-  flowperpkt_test_main_t *sm = &flowperpkt_test_main;
-  u8 *name;
-
-  sm->vat_main = vam;
-
-  /* Ask the vpp engine for the first assigned message-id */
-  name = format (0, "flowperpkt_%08x%c", api_version, 0);
-  sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
-
-  /* Don't attempt to hook up API messages if the data plane plugin is AWOL */
-  if (sm->msg_id_base != (u16) ~ 0)
-    flowperpkt_vat_api_hookup (vam);
-
-  vec_free (name);
-
-  return 0;
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/flowperpkt/l2_node.c b/src/plugins/flowperpkt/l2_node.c
deleted file mode 100644
index db80e990..00000000
--- a/src/plugins/flowperpkt/l2_node.c
+++ /dev/null
@@ -1,561 +0,0 @@
-/*
- * l2_node.c - l2 ipfix-per-packet graph node
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-/**
- * @file l2 flow record generator graph node
- */
-
-typedef struct
-{
-  /** interface handle */
-  u32 rx_sw_if_index;
-  u32 tx_sw_if_index;
-  /** src and dst L2 addresses */
-  u8 src_mac[6];
-  u8 dst_mac[6];
-  /** Ethertype */
-  u16 ethertype;
-  /** packet timestamp */
-  u64 timestamp;
-  /** size of the buffer */
-  u16 buffer_size;
-} flowperpkt_l2_trace_t;
-
-/* packet trace format function */
-static u8 *
-format_flowperpkt_l2_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  flowperpkt_l2_trace_t *t = va_arg (*args, flowperpkt_l2_trace_t *);
-
-  s = format (s,
-	      "FLOWPERPKT-L2: rx_sw_if_index %d, tx_sw_if_index %d, src %U dst %U ethertype %0x2, timestamp %lld, size %d",
-	      t->rx_sw_if_index, t->tx_sw_if_index,
-	      format_ethernet_address, &t->src_mac,
-	      format_ethernet_address, &t->dst_mac,
-	      t->ethertype, t->timestamp, t->buffer_size);
-  return s;
-}
-
-vlib_node_registration_t flowperpkt_l2_node;
-
-/* No counters at the moment */
-#define foreach_flowperpkt_l2_error
-
-typedef enum
-{
-#define _(sym,str) FLOWPERPKT_ERROR_##sym,
-  foreach_flowperpkt_l2_error
-#undef _
-    FLOWPERPKT_N_ERROR,
-} flowperpkt_l2_error_t;
-
-static char *flowperpkt_l2_error_strings[] = {
-#define _(sym,string) string,
-  foreach_flowperpkt_l2_error
-#undef _
-};
-
-typedef enum
-{
-  FLOWPERPKT_L2_NEXT_DROP,
-  FLOWPERPKT_L2_NEXT_IP4_LOOKUP,
-  FLOWPERPKT_L2_N_NEXT,
-} flowperpkt_l2_next_t;
-
-/**
- * @brief add an entry to the flow record under construction
- * @param vm vlib_main_t * current worker thread main structure pointer
- * @param fm flowperpkt_main_t * flow-per-packet main structure pointer
- * @param sw_if_index u32 interface handle
- * @param tos u8 ToS bits from the packet
- * @param timestamp u64 timestamp, nanoseconds since 1/1/70
- * @param length u16 ip length of the packet
- * @param do_flush int 1 = flush all cached records, 0 = construct a record
- */
-
-static inline void
-add_to_flow_record_l2 (vlib_main_t * vm,
-		       vlib_node_runtime_t * node,
-		       flowperpkt_main_t * fm,
-		       u32 rx_sw_if_index, u32 tx_sw_if_index,
-		       u8 * src_mac, u8 * dst_mac,
-		       u16 ethertype, u64 timestamp, u16 length, int do_flush)
-{
-  u32 my_thread_index = vm->thread_index;
-  flow_report_main_t *frm = &flow_report_main;
-  ip4_header_t *ip;
-  udp_header_t *udp;
-  ip4_ipfix_template_packet_t *tp;
-  ipfix_message_header_t *h;
-  ipfix_set_header_t *s;
-  vlib_frame_t *f;
-  vlib_buffer_t *b0;
-  u16 offset;
-  u32 bi0;
-  vlib_buffer_free_list_t *fl;
-
-  /* Find or allocate a buffer */
-  b0 = fm->l2_buffers_per_worker[my_thread_index];
-
-  /* Need to allocate a buffer? */
-  if (PREDICT_FALSE (b0 == 0))
-    {
-      /* Nothing to flush */
-      if (do_flush)
-	return;
-
-      /* $$$$ drop counter? */
-      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
-	return;
-
-      /* Initialize the buffer */
-      b0 = fm->l2_buffers_per_worker[my_thread_index] =
-	vlib_get_buffer (vm, bi0);
-      fl =
-	vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-      vlib_buffer_init_for_free_list (b0, fl);
-      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-      offset = 0;
-    }
-  else
-    {
-      /* use the current buffer */
-      bi0 = vlib_get_buffer_index (vm, b0);
-      offset = fm->l2_next_record_offset_per_worker[my_thread_index];
-    }
-
-  /* Find or allocate a frame */
-  f = fm->l2_frames_per_worker[my_thread_index];
-  if (PREDICT_FALSE (f == 0))
-    {
-      u32 *to_next;
-      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      fm->l2_frames_per_worker[my_thread_index] = f;
-
-      /* Enqueue the buffer */
-      to_next = vlib_frame_vector_args (f);
-      to_next[0] = bi0;
-      f->n_vectors = 1;
-    }
-
-  /* Fresh packet, construct header */
-  if (PREDICT_FALSE (offset == 0))
-    {
-      flow_report_stream_t *stream;
-
-      stream = &frm->streams[0];
-
-      b0->current_data = 0;
-      b0->current_length = sizeof (*ip) + sizeof (*udp) + sizeof (*h) +
-	sizeof (*s);
-      b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_FLOW_REPORT);
-      vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
-      vnet_buffer (b0)->sw_if_index[VLIB_TX] = frm->fib_index;
-
-      tp = vlib_buffer_get_current (b0);
-      ip = (ip4_header_t *) & tp->ip4;
-      udp = (udp_header_t *) (ip + 1);
-      h = (ipfix_message_header_t *) (udp + 1);
-      s = (ipfix_set_header_t *) (h + 1);
-
-      ip->ip_version_and_header_length = 0x45;
-      ip->ttl = 254;
-      ip->protocol = IP_PROTOCOL_UDP;
-      ip->flags_and_fragment_offset = 0;
-      ip->src_address.as_u32 = frm->src_address.as_u32;
-      ip->dst_address.as_u32 = frm->ipfix_collector.as_u32;
-      udp->src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix);
-      udp->dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix);
-      udp->checksum = 0;
-
-      /* FIXUP: message header export_time */
-      h->export_time = (u32)
-	(((f64) frm->unix_time_0) +
-	 (vlib_time_now (frm->vlib_main) - frm->vlib_time_0));
-      h->export_time = clib_host_to_net_u32 (h->export_time);
-      h->domain_id = clib_host_to_net_u32 (stream->domain_id);
-
-      /* FIXUP: message header sequence_number */
-      h->sequence_number = stream->sequence_number++;
-      h->sequence_number = clib_host_to_net_u32 (h->sequence_number);
-
-      offset = (u32) (((u8 *) (s + 1)) - (u8 *) tp);
-    }
-
-  /* Add data, unless we're flushing stale data */
-  if (PREDICT_TRUE (do_flush == 0))
-    {
-
-      /* Add data */
-      /* Ingress interface */
-      {
-	u32 ingress_interface = clib_host_to_net_u32 (rx_sw_if_index);
-	clib_memcpy (b0->data + offset, &ingress_interface,
-		     sizeof (ingress_interface));
-	offset += sizeof (ingress_interface);
-      }
-      /* Egress interface */
-      {
-	u32 egress_interface = clib_host_to_net_u32 (tx_sw_if_index);
-	clib_memcpy (b0->data + offset, &egress_interface,
-		     sizeof (egress_interface));
-	offset += sizeof (egress_interface);
-      }
-      /* src mac address */
-      {
-	clib_memcpy (b0->data + offset, src_mac, 6);
-	offset += 6;
-      }
-      /* dst mac address */
-      {
-	clib_memcpy (b0->data + offset, dst_mac, 6);
-	offset += 6;
-      }
-
-      /* ethertype */
-      b0->data[offset++] = ethertype >> 8;
-      b0->data[offset++] = ethertype & 0xFF;
-
-      /* Timestamp */
-      clib_memcpy (b0->data + offset, ×tamp, sizeof (f64));
-      offset += sizeof (f64);
-
-      /* pkt size */
-      {
-	u16 pkt_size = clib_host_to_net_u16 (length);
-	clib_memcpy (b0->data + offset, &pkt_size, sizeof (pkt_size));
-	offset += sizeof (pkt_size);
-      }
-
-      b0->current_length +=
-	/* 2*sw_if_index + 2*mac + ethertype + timestamp + length = 32 */
-	2 * sizeof (u32) + 12 + sizeof (u16) + sizeof (f64) + sizeof (u16);
-
-    }
-  /* Time to flush the buffer? */
-  if (PREDICT_FALSE
-      (do_flush || (offset + 2 * sizeof (u32) + 12 + sizeof (u16) +
-		    +sizeof (f64) + sizeof (u16)) > frm->path_mtu))
-    {
-      tp = vlib_buffer_get_current (b0);
-      ip = (ip4_header_t *) & tp->ip4;
-      udp = (udp_header_t *) (ip + 1);
-      h = (ipfix_message_header_t *) (udp + 1);
-      s = (ipfix_set_header_t *) (h + 1);
-
-      s->set_id_length = ipfix_set_id_length (fm->l2_report_id,
-					      b0->current_length -
-					      (sizeof (*ip) + sizeof (*udp) +
-					       sizeof (*h)));
-      h->version_length = version_length (b0->current_length -
-					  (sizeof (*ip) + sizeof (*udp)));
-
-      ip->length = clib_host_to_net_u16 (b0->current_length);
-
-      ip->checksum = ip4_header_checksum (ip);
-      udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
-
-      if (frm->udp_checksum)
-	{
-	  /* RFC 7011 section 10.3.2. */
-	  udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
-	  if (udp->checksum == 0)
-	    udp->checksum = 0xffff;
-	}
-
-      ASSERT (ip->checksum == ip4_header_checksum (ip));
-
-      if (PREDICT_FALSE (vlib_get_trace_count (vm, node) > 0))
-	{
-	  vlib_trace_buffer (vm, node, FLOWPERPKT_L2_NEXT_IP4_LOOKUP, b0,
-			     0 /* follow chain */ );
-	  flowperpkt_l2_trace_t *t =
-	    vlib_add_trace (vm, node, b0, sizeof (*t));
-	  memset (t, 0, sizeof (*t));
-	  t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	  t->tx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-	  t->buffer_size = b0->current_length;
-	}
-
-      vlib_put_frame_to_node (vm, ip4_lookup_node.index,
-			      fm->l2_frames_per_worker[my_thread_index]);
-      fm->l2_frames_per_worker[my_thread_index] = 0;
-      fm->l2_buffers_per_worker[my_thread_index] = 0;
-      offset = 0;
-    }
-
-  fm->l2_next_record_offset_per_worker[my_thread_index] = offset;
-}
-
-void
-flowperpkt_flush_callback_l2 (void)
-{
-  vlib_main_t *vm = vlib_get_main ();
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  vlib_node_runtime_t *node;
-  node = vlib_node_get_runtime (vm, flowperpkt_l2_node.index);
-
-  add_to_flow_record_l2 (vm, node, fm, 0 /* rx_sw_if_index */ ,
-			 0 /* tx_sw_if_index */ ,
-			 0 /* src mac */ ,
-			 0 /* dst mac */ ,
-			 0 /* ethertype */ ,
-			 0ULL /* timestamp */ ,
-			 0 /* length */ ,
-			 1 /* do_flush */ );
-}
-
-
-static uword
-flowperpkt_l2_node_fn (vlib_main_t * vm,
-		       vlib_node_runtime_t * node, vlib_frame_t * frame)
-{
-  u32 n_left_from, *from, *to_next;
-  flowperpkt_l2_next_t next_index;
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  u64 now;
-
-  now = (u64) ((vlib_time_now (vm) - fm->vlib_time_0) * 1e9);
-  now += fm->nanosecond_time_0;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-	  u32 next0 = FLOWPERPKT_L2_NEXT_DROP;
-	  u32 next1 = FLOWPERPKT_L2_NEXT_DROP;
-	  ethernet_header_t *eh0, *eh1;
-	  u16 len0, len1;
-	  u32 bi0, bi1;
-	  vlib_buffer_t *b0, *b1;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t *p2, *p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-	  /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-	  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_TX],
-			     &next0, b0);
-	  vnet_feature_next (vnet_buffer (b1)->sw_if_index[VLIB_TX],
-			     &next1, b1);
-
-	  eh0 = vlib_buffer_get_current (b0);
-	  len0 = vlib_buffer_length_in_chain (vm, b0);
-
-	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
-	    add_to_flow_record_l2 (vm, node, fm,
-				   vnet_buffer (b0)->sw_if_index[VLIB_RX],
-				   vnet_buffer (b0)->sw_if_index[VLIB_TX],
-				   eh0->src_address,
-				   eh0->dst_address,
-				   eh0->type, now, len0, 0 /* flush */ );
-
-	  eh1 = vlib_buffer_get_current (b0);
-	  len1 = vlib_buffer_length_in_chain (vm, b0);
-
-	  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
-	    add_to_flow_record_l2 (vm, node, fm,
-				   vnet_buffer (b1)->sw_if_index[VLIB_RX],
-				   vnet_buffer (b1)->sw_if_index[VLIB_TX],
-				   eh1->src_address,
-				   eh1->dst_address,
-				   eh1->type, now, len1, 0 /* flush */ );
-
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
-	    {
-	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
-		{
-		  flowperpkt_l2_trace_t *t =
-		    vlib_add_trace (vm, node, b0, sizeof (*t));
-		  t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-		  t->tx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-		  clib_memcpy (t->src_mac, eh0->src_address, 6);
-		  clib_memcpy (t->dst_mac, eh0->dst_address, 6);
-		  t->ethertype = clib_net_to_host_u16 (eh0->type);
-		  t->timestamp = now;
-		  t->buffer_size = len0;
-		}
-	      if (b1->flags & VLIB_BUFFER_IS_TRACED)
-		{
-		  flowperpkt_l2_trace_t *t =
-		    vlib_add_trace (vm, node, b1, sizeof (*t));
-		  t->rx_sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
-		  t->tx_sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX];
-		  clib_memcpy (t->src_mac, eh1->src_address, 6);
-		  clib_memcpy (t->dst_mac, eh1->dst_address, 6);
-		  t->ethertype = clib_net_to_host_u16 (eh1->type);
-		  t->timestamp = now;
-		  t->buffer_size = len1;
-		}
-	    }
-
-	  /* verify speculative enqueues, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, bi1, next0, next1);
-	}
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  u32 next0 = FLOWPERPKT_L2_NEXT_DROP;
-	  ethernet_header_t *eh0;
-	  u16 len0;
-
-	  /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_TX],
-			     &next0, b0);
-
-	  eh0 = vlib_buffer_get_current (b0);
-	  len0 = vlib_buffer_length_in_chain (vm, b0);
-
-	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
-	    add_to_flow_record_l2 (vm, node, fm,
-				   vnet_buffer (b0)->sw_if_index[VLIB_RX],
-				   vnet_buffer (b0)->sw_if_index[VLIB_TX],
-				   eh0->src_address,
-				   eh0->dst_address,
-				   eh0->type, now, len0, 0 /* flush */ );
-
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	    {
-	      flowperpkt_l2_trace_t *t =
-		vlib_add_trace (vm, node, b0, sizeof (*t));
-	      t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	      t->tx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-	      clib_memcpy (t->src_mac, eh0->src_address, 6);
-	      clib_memcpy (t->dst_mac, eh0->dst_address, 6);
-	      t->ethertype = clib_net_to_host_u16 (eh0->type);
-	      t->timestamp = now;
-	      t->buffer_size = len0;
-	    }
-
-	  /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  return frame->n_vectors;
-}
-
-/**
- * @brief IPFIX l2 flow-per-packet graph node
- * @node flowperpkt-l2
- *
- * This is the IPFIX flow-record-per-packet node.
- *
- * @param vm    vlib_main_t corresponding to the current thread.
- * @param node  vlib_node_runtime_t data for this node.
- * @param frame vlib_frame_t whose contents should be dispatched.
- *
- * @par Graph mechanics: buffer metadata, next index usage
- *
- * Uses:
- * - vnet_buffer(b)->ip.save_rewrite_length
- *     - tells the node the length of the rewrite which was applied in
- *       ip4/6_rewrite_inline, allows the code to find the IP header without
- *       having to parse L2 headers, or make stupid assumptions about their
- *       length.
- * - vnet_buffer(b)->flags & VLIB_BUFFER_FLOW_REPORT
- *     - Used to suppress flow record generation for flow record packets.
- *
- * Sets:
- * - vnet_buffer(b)->flags & VLIB_BUFFER_FLOW_REPORT
- *     - To suppress flow record generation for flow record packets
- *
- * Next Index:
- * - Next configured output feature on the interface, usually
- *   "interface-output." Generated flow records head for ip4-lookup
- */
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (flowperpkt_l2_node) = {
-  .function = flowperpkt_l2_node_fn,
-  .name = "flowperpkt-l2",
-  .vector_size = sizeof (u32),
-  .format_trace = format_flowperpkt_l2_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(flowperpkt_l2_error_strings),
-  .error_strings = flowperpkt_l2_error_strings,
-
-  .n_next_nodes = FLOWPERPKT_L2_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [FLOWPERPKT_L2_NEXT_DROP] = "error-drop",
-    [FLOWPERPKT_L2_NEXT_IP4_LOOKUP] = "ip4-lookup",
-  },
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/flowperpkt/node.c b/src/plugins/flowperpkt/node.c
deleted file mode 100644
index 9bac4166..00000000
--- a/src/plugins/flowperpkt/node.c
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * node.c - ipv4 ipfix-per-packet graph node
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-
-/**
- * @file ipv4 flow record generator graph node
- */
-
-typedef struct
-{
-  /** interface handle */
-  u32 rx_sw_if_index;
-  u32 tx_sw_if_index;
-  u32 src_address;
-  u32 dst_address;
-  /** ToS bits */
-  u8 tos;
-  /** packet timestamp */
-  u64 timestamp;
-  /** size of the buffer */
-  u16 buffer_size;
-} flowperpkt_ipv4_trace_t;
-
-/* packet trace format function */
-static u8 *
-format_flowperpkt_ipv4_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  flowperpkt_ipv4_trace_t *t = va_arg (*args, flowperpkt_ipv4_trace_t *);
-
-  s = format (s,
-	      "FLOWPERPKT-V4: rx_sw_if_index %d, tx_sw_if_index %d, src %U dst %U tos %0x2, timestamp %lld, size %d",
-	      t->rx_sw_if_index, t->tx_sw_if_index,
-	      format_ip4_address, &t->src_address,
-	      format_ip4_address, &t->dst_address,
-	      t->tos, t->timestamp, t->buffer_size);
-  return s;
-}
-
-vlib_node_registration_t flowperpkt_ipv4_node;
-
-/* No counters at the moment */
-#define foreach_flowperpkt_ipv4_error
-
-typedef enum
-{
-#define _(sym,str) FLOWPERPKT_ERROR_##sym,
-  foreach_flowperpkt_ipv4_error
-#undef _
-    FLOWPERPKT_N_ERROR,
-} flowperpkt_ipv4_error_t;
-
-static char *flowperpkt_ipv4_error_strings[] = {
-#define _(sym,string) string,
-  foreach_flowperpkt_ipv4_error
-#undef _
-};
-
-typedef enum
-{
-  FLOWPERPKT_IPV4_NEXT_DROP,
-  FLOWPERPKT_IPV4_NEXT_LOOKUP,
-  FLOWPERPKT_IPV4_N_NEXT,
-} flowperpkt_ipv4_next_t;
-
-/**
- * @brief add an entry to the flow record under construction
- * @param vm vlib_main_t * current worker thread main structure pointer
- * @param fm flowperpkt_main_t * flow-per-packet main structure pointer
- * @param sw_if_index u32 interface handle
- * @param tos u8 ToS bits from the packet
- * @param timestamp u64 timestamp, nanoseconds since 1/1/70
- * @param length u16 ip length of the packet
- * @param do_flush int 1 = flush all cached records, 0 = construct a record
- */
-
-static inline void
-add_to_flow_record_ipv4 (vlib_main_t * vm,
-			 vlib_node_runtime_t * node,
-			 flowperpkt_main_t * fm,
-			 u32 rx_sw_if_index, u32 tx_sw_if_index,
-			 u32 src_address, u32 dst_address,
-			 u8 tos, u64 timestamp, u16 length, int do_flush)
-{
-  u32 my_thread_index = vm->thread_index;
-  flow_report_main_t *frm = &flow_report_main;
-  ip4_header_t *ip;
-  udp_header_t *udp;
-  ip4_ipfix_template_packet_t *tp;
-  ipfix_message_header_t *h;
-  ipfix_set_header_t *s;
-  vlib_frame_t *f;
-  vlib_buffer_t *b0;
-  u16 offset;
-  u32 bi0;
-  vlib_buffer_free_list_t *fl;
-
-  /* Find or allocate a buffer */
-  b0 = fm->ipv4_buffers_per_worker[my_thread_index];
-
-  /* Need to allocate a buffer? */
-  if (PREDICT_FALSE (b0 == 0))
-    {
-      /* Nothing to flush */
-      if (do_flush)
-	return;
-
-      /* $$$$ drop counter? */
-      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
-	return;
-
-      /* Initialize the buffer */
-      b0 = fm->ipv4_buffers_per_worker[my_thread_index] =
-	vlib_get_buffer (vm, bi0);
-      fl =
-	vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-      vlib_buffer_init_for_free_list (b0, fl);
-      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-      offset = 0;
-    }
-  else
-    {
-      /* use the current buffer */
-      bi0 = vlib_get_buffer_index (vm, b0);
-      offset = fm->ipv4_next_record_offset_per_worker[my_thread_index];
-    }
-
-  /* Find or allocate a frame */
-  f = fm->ipv4_frames_per_worker[my_thread_index];
-  if (PREDICT_FALSE (f == 0))
-    {
-      u32 *to_next;
-      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      fm->ipv4_frames_per_worker[my_thread_index] = f;
-
-      /* Enqueue the buffer */
-      to_next = vlib_frame_vector_args (f);
-      to_next[0] = bi0;
-      f->n_vectors = 1;
-    }
-
-  /* Fresh packet, construct header */
-  if (PREDICT_FALSE (offset == 0))
-    {
-      flow_report_stream_t *stream;
-
-      stream = &frm->streams[0];
-
-      b0->current_data = 0;
-      b0->current_length = sizeof (*ip) + sizeof (*udp) + sizeof (*h) +
-	sizeof (*s);
-      b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_FLOW_REPORT);
-      vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
-      vnet_buffer (b0)->sw_if_index[VLIB_TX] = frm->fib_index;
-
-      tp = vlib_buffer_get_current (b0);
-      ip = (ip4_header_t *) & tp->ip4;
-      udp = (udp_header_t *) (ip + 1);
-      h = (ipfix_message_header_t *) (udp + 1);
-      s = (ipfix_set_header_t *) (h + 1);
-
-      ip->ip_version_and_header_length = 0x45;
-      ip->ttl = 254;
-      ip->protocol = IP_PROTOCOL_UDP;
-      ip->flags_and_fragment_offset = 0;
-      ip->src_address.as_u32 = frm->src_address.as_u32;
-      ip->dst_address.as_u32 = frm->ipfix_collector.as_u32;
-      udp->src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix);
-      udp->dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix);
-      udp->checksum = 0;
-
-      /* FIXUP: message header export_time */
-      h->export_time = (u32)
-	(((f64) frm->unix_time_0) +
-	 (vlib_time_now (frm->vlib_main) - frm->vlib_time_0));
-      h->export_time = clib_host_to_net_u32 (h->export_time);
-      h->domain_id = clib_host_to_net_u32 (stream->domain_id);
-
-      /* FIXUP: message header sequence_number */
-      h->sequence_number = stream->sequence_number++;
-      h->sequence_number = clib_host_to_net_u32 (h->sequence_number);
-
-      offset = (u32) (((u8 *) (s + 1)) - (u8 *) tp);
-    }
-
-  /* Add data, unless we're flushing stale data */
-  if (PREDICT_TRUE (do_flush == 0))
-    {
-
-      /* Add data */
-      /* Ingress interface */
-      {
-	u32 ingress_interface = clib_host_to_net_u32 (rx_sw_if_index);
-	clib_memcpy (b0->data + offset, &ingress_interface,
-		     sizeof (ingress_interface));
-	offset += sizeof (ingress_interface);
-      }
-      /* Egress interface */
-      {
-	u32 egress_interface = clib_host_to_net_u32 (tx_sw_if_index);
-	clib_memcpy (b0->data + offset, &egress_interface,
-		     sizeof (egress_interface));
-	offset += sizeof (egress_interface);
-      }
-      /* ip4 src address */
-      {
-	clib_memcpy (b0->data + offset, &src_address, sizeof (src_address));
-	offset += sizeof (src_address);
-      }
-      /* ip4 dst address */
-      {
-	clib_memcpy (b0->data + offset, &dst_address, sizeof (dst_address));
-	offset += sizeof (dst_address);
-      }
-
-      /* ToS */
-      b0->data[offset++] = tos;
-
-      /* Timestamp */
-      clib_memcpy (b0->data + offset, ×tamp, sizeof (f64));
-      offset += sizeof (f64);
-
-      /* pkt size */
-      {
-	u16 pkt_size = clib_host_to_net_u16 (length);
-	clib_memcpy (b0->data + offset, &pkt_size, sizeof (pkt_size));
-	offset += sizeof (pkt_size);
-      }
-
-      b0->current_length +=
-	/* sw_if_index + tos + timestamp + length = 15 */
-	4 * sizeof (u32) + sizeof (u8) + sizeof (f64) + sizeof (u16);
-
-    }
-  /* Time to flush the buffer? */
-  if (PREDICT_FALSE
-      (do_flush || (offset + 4 * sizeof (u32) + sizeof (u8)
-		    + sizeof (f64) + sizeof (u16)) > frm->path_mtu))
-    {
-      tp = vlib_buffer_get_current (b0);
-      ip = (ip4_header_t *) & tp->ip4;
-      udp = (udp_header_t *) (ip + 1);
-      h = (ipfix_message_header_t *) (udp + 1);
-      s = (ipfix_set_header_t *) (h + 1);
-
-      s->set_id_length = ipfix_set_id_length (fm->ipv4_report_id,
-					      b0->current_length -
-					      (sizeof (*ip) + sizeof (*udp) +
-					       sizeof (*h)));
-      h->version_length = version_length (b0->current_length -
-					  (sizeof (*ip) + sizeof (*udp)));
-
-      ip->length = clib_host_to_net_u16 (b0->current_length);
-
-      ip->checksum = ip4_header_checksum (ip);
-      udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
-
-      if (frm->udp_checksum)
-	{
-	  /* RFC 7011 section 10.3.2. */
-	  udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
-	  if (udp->checksum == 0)
-	    udp->checksum = 0xffff;
-	}
-
-      ASSERT (ip->checksum == ip4_header_checksum (ip));
-
-      if (PREDICT_FALSE (vlib_get_trace_count (vm, node) > 0))
-	{
-	  vlib_trace_buffer (vm, node, FLOWPERPKT_IPV4_NEXT_LOOKUP, b0,
-			     0 /* follow chain */ );
-	  flowperpkt_ipv4_trace_t *t =
-	    vlib_add_trace (vm, node, b0, sizeof (*t));
-	  t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	  t->tx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-	  t->src_address = 0;
-	  t->dst_address = 0;
-	  t->tos = 0;
-	  t->timestamp = 0;
-	  t->buffer_size = b0->current_length;
-	}
-
-      vlib_put_frame_to_node (vm, ip4_lookup_node.index,
-			      fm->ipv4_frames_per_worker[my_thread_index]);
-      fm->ipv4_frames_per_worker[my_thread_index] = 0;
-      fm->ipv4_buffers_per_worker[my_thread_index] = 0;
-      offset = 0;
-    }
-
-  fm->ipv4_next_record_offset_per_worker[my_thread_index] = offset;
-}
-
-void
-flowperpkt_flush_callback_ipv4 (void)
-{
-  vlib_main_t *vm = vlib_get_main ();
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  vlib_node_runtime_t *node;
-  node = vlib_node_get_runtime (vm, flowperpkt_ipv4_node.index);
-
-  add_to_flow_record_ipv4 (vm, node, fm, 0 /* rx_sw_if_index */ ,
-			   0 /* tx_sw_if_index */ ,
-			   0 /* src_address */ ,
-			   0 /* dst_address */ ,
-			   0 /* ToS */ ,
-			   0ULL /* timestamp */ ,
-			   0 /* length */ ,
-			   1 /* do_flush */ );
-}
-
-
-static uword
-flowperpkt_ipv4_node_fn (vlib_main_t * vm,
-			 vlib_node_runtime_t * node, vlib_frame_t * frame)
-{
-  u32 n_left_from, *from, *to_next;
-  flowperpkt_ipv4_next_t next_index;
-  flowperpkt_main_t *fm = &flowperpkt_main;
-  u64 now;
-
-  now = (u64) ((vlib_time_now (vm) - fm->vlib_time_0) * 1e9);
-  now += fm->nanosecond_time_0;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-	  u32 next0 = FLOWPERPKT_IPV4_NEXT_DROP;
-	  u32 next1 = FLOWPERPKT_IPV4_NEXT_DROP;
-	  ip4_header_t *ip0, *ip1;
-	  u16 len0, len1;
-	  u32 bi0, bi1;
-	  vlib_buffer_t *b0, *b1;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t *p2, *p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-	  /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-	  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_TX],
-			     &next0, b0);
-	  vnet_feature_next (vnet_buffer (b1)->sw_if_index[VLIB_TX],
-			     &next1, b1);
-
-	  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
-				  vnet_buffer (b0)->ip.save_rewrite_length);
-
-	  len0 = vlib_buffer_length_in_chain (vm, b0);
-
-	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
-	    add_to_flow_record_ipv4 (vm, node, fm,
-				     vnet_buffer (b0)->sw_if_index[VLIB_RX],
-				     vnet_buffer (b0)->sw_if_index[VLIB_TX],
-				     ip0->src_address.as_u32,
-				     ip0->dst_address.as_u32,
-				     ip0->tos, now, len0, 0 /* flush */ );
-
-	  ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
-				  vnet_buffer (b1)->ip.save_rewrite_length);
-	  len1 = vlib_buffer_length_in_chain (vm, b1);
-
-	  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
-	    add_to_flow_record_ipv4 (vm, node, fm,
-				     vnet_buffer (b1)->sw_if_index[VLIB_RX],
-				     vnet_buffer (b1)->sw_if_index[VLIB_TX],
-				     ip1->src_address.as_u32,
-				     ip1->dst_address.as_u32,
-				     ip1->tos, now, len1, 0 /* flush */ );
-
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
-	    {
-	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
-		{
-		  flowperpkt_ipv4_trace_t *t =
-		    vlib_add_trace (vm, node, b0, sizeof (*t));
-		  t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-		  t->tx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-		  t->src_address = ip0->src_address.as_u32;
-		  t->dst_address = ip0->dst_address.as_u32;
-		  t->tos = ip0->tos;
-		  t->timestamp = now;
-		  t->buffer_size = len0;
-		}
-	      if (b1->flags & VLIB_BUFFER_IS_TRACED)
-		{
-		  flowperpkt_ipv4_trace_t *t =
-		    vlib_add_trace (vm, node, b1, sizeof (*t));
-		  t->rx_sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
-		  t->tx_sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX];
-		  t->src_address = ip1->src_address.as_u32;
-		  t->dst_address = ip1->dst_address.as_u32;
-		  t->tos = ip1->tos;
-		  t->timestamp = now;
-		  t->buffer_size = len1;
-		}
-	    }
-
-	  /* verify speculative enqueues, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, bi1, next0, next1);
-	}
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  u32 next0 = FLOWPERPKT_IPV4_NEXT_DROP;
-	  ip4_header_t *ip0;
-	  u16 len0;
-
-	  /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-	  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_TX],
-			     &next0, b0);
-
-	  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
-				  vnet_buffer (b0)->ip.save_rewrite_length);
-	  /*
-	   * egressInterface, TLV type 14, u32
-	   * ipClassOfService, TLV type 5, u8
-	   * flowStartNanoseconds, TLV type 156, dateTimeNanoseconds (f64)
-	   *   Implementation: f64 nanoseconds since VPP started
-	   * dataLinkFrameSize, TLV type 312, u16
-	   */
-	  len0 = vlib_buffer_length_in_chain (vm, b0);
-
-	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
-	    add_to_flow_record_ipv4 (vm, node, fm,
-				     vnet_buffer (b0)->sw_if_index[VLIB_RX],
-				     vnet_buffer (b0)->sw_if_index[VLIB_TX],
-				     ip0->src_address.as_u32,
-				     ip0->dst_address.as_u32,
-				     ip0->tos, now, len0, 0 /* flush */ );
-
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	    {
-	      flowperpkt_ipv4_trace_t *t =
-		vlib_add_trace (vm, node, b0, sizeof (*t));
-	      t->rx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	      t->tx_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-	      t->src_address = ip0->src_address.as_u32;
-	      t->dst_address = ip0->dst_address.as_u32;
-	      t->tos = ip0->tos;
-	      t->timestamp = now;
-	      t->buffer_size = len0;
-	    }
-
-	  /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  return frame->n_vectors;
-}
-
-/**
- * @brief IPFIX ipv4 flow-per-packet graph node
- * @node flowperpkt-ipv4
- *
- * This is the IPFIX flow-record-per-packet node.
- *
- * @param vm    vlib_main_t corresponding to the current thread.
- * @param node  vlib_node_runtime_t data for this node.
- * @param frame vlib_frame_t whose contents should be dispatched.
- *
- * @par Graph mechanics: buffer metadata, next index usage
- *
- * Uses:
- * - vnet_buffer(b)->ip.save_rewrite_length
- *     - tells the node the length of the rewrite which was applied in
- *       ip4/6_rewrite_inline, allows the code to find the IP header without
- *       having to parse L2 headers, or make stupid assumptions about their
- *       length.
- * - vnet_buffer(b)->flags & VLIB_BUFFER_FLOW_REPORT
- *     - Used to suppress flow record generation for flow record packets.
- *
- * Sets:
- * - vnet_buffer(b)->flags & VLIB_BUFFER_FLOW_REPORT
- *     - To suppress flow record generation for flow record packets
- *
- * Next Index:
- * - Next configured output feature on the interface, usually
- *   "interface-output." Generated flow records head for ip4-lookup
- */
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (flowperpkt_ipv4_node) = {
-  .function = flowperpkt_ipv4_node_fn,
-  .name = "flowperpkt-ipv4",
-  .vector_size = sizeof (u32),
-  .format_trace = format_flowperpkt_ipv4_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(flowperpkt_ipv4_error_strings),
-  .error_strings = flowperpkt_ipv4_error_strings,
-
-  .n_next_nodes = FLOWPERPKT_IPV4_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [FLOWPERPKT_IPV4_NEXT_DROP] = "error-drop",
-    /* Used only to trace ipfix data packets */
-    [FLOWPERPKT_IPV4_NEXT_LOOKUP] = "ip4-lookup",
-  },
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/flowprobe.am b/src/plugins/flowprobe.am
new file mode 100644
index 00000000..c56e246d
--- /dev/null
+++ b/src/plugins/flowprobe.am
@@ -0,0 +1,37 @@
+
+# Copyright (c)  
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppplugins_LTLIBRARIES += flowprobe_plugin.la
+vppapitestplugins_LTLIBRARIES += flowprobe_test_plugin.la
+
+flowprobe_plugin_la_SOURCES = flowprobe/flowprobe.c	\
+        flowprobe/node.c				\
+	flowprobe/flowprobe_plugin.api.h
+
+BUILT_SOURCES +=				\
+	flowprobe/flowprobe.api.h		\
+	flowprobe/flowprobe.api.json
+
+noinst_HEADERS +=				\
+  flowprobe/flowprobe_all_api_h.h		\
+  flowprobe/flowprobe_msg_enum.h		\
+  flowprobe/flowprobe.api.h
+
+flowprobe_test_plugin_la_SOURCES = \
+  flowprobe/flowprobe_test.c			\
+  flowprobe/flowprobe_plugin.api.h
+
+API_FILES += flowprobe/flowprobe.api
+
+# vi:syntax=automake
diff --git a/src/plugins/flowprobe/flowprobe.api b/src/plugins/flowprobe/flowprobe.api
new file mode 100644
index 00000000..3f8c583b
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe.api
@@ -0,0 +1,40 @@
+/* Define a simple enable-disable binary API to control the feature */
+
+/** \file
+    This file defines the vpp control-plane API messages
+    used to control the flowprobe plugin
+*/
+
+/** \brief Enable / disable per-packet IPFIX recording on an interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - add address if non-zero, else delete
+    @param is_ipv6 - if non-zero the address is ipv6, else ipv4
+    @param sw_if_index - index of the interface
+*/
+autoreply manual_print define flowprobe_tx_interface_add_del
+{
+  /* Client identifier, set from api_main.my_client_index */
+  u32 client_index;
+
+  /* Arbitrary context, so client can match reply to request */
+  u32 context;
+
+  /* Enable / disable the feature */
+  u8 is_add;
+  u8 which;  /* 0 = ipv4, 1 = l2, 2 = ipv6 */
+
+  /* Interface handle */
+  u32 sw_if_index;
+};
+
+autoreply define flowprobe_params
+{
+  u32 client_index;
+  u32 context;
+  u8 record_l2;
+  u8 record_l3;
+  u8 record_l4;
+  u32 active_timer;  /* ~0 is off, 0 is default */
+  u32 passive_timer; /* ~0 is off, 0 is default */
+};
diff --git a/src/plugins/flowprobe/flowprobe.c b/src/plugins/flowprobe/flowprobe.c
new file mode 100644
index 00000000..8975f89c
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe.c
@@ -0,0 +1,1122 @@
+/*
+ * flowprobe.c - ipfix probe plugin
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ * @brief Per-packet IPFIX flow record generator plugin
+ *
+ * This file implements vpp plugin registration mechanics,
+ * debug CLI, and binary API handling.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+/* define message IDs */
+#include 
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+flowprobe_main_t flowprobe_main;
+vlib_node_registration_t flowprobe_walker_node;
+static vlib_node_registration_t flowprobe_timer_node;
+uword flowprobe_walker_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
+				vlib_frame_t * f);
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#define REPLY_MSG_ID_BASE fm->msg_id_base
+#include 
+
+/* Define the per-interface configurable features */
+/* *INDENT-OFF* */
+VNET_FEATURE_INIT (flow_perpacket_ip4, static) =
+{
+  .arc_name = "ip4-output",
+  .node_name = "flowprobe-ip4",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+
+VNET_FEATURE_INIT (flow_perpacket_ip6, static) =
+{
+  .arc_name = "ip6-output",
+  .node_name = "flowprobe-ip6",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+
+VNET_FEATURE_INIT (flow_perpacket_l2, static) =
+{
+  .arc_name = "interface-output",
+  .node_name = "flowprobe-l2",
+  .runs_before = VNET_FEATURES ("interface-tx"),
+};
+/* *INDENT-ON* */
+
+/* Macro to finish up custom dump fns */
+#define FINISH                                  \
+    vec_add1 (s, 0);                            \
+    vl_print (handle, (char *)s);               \
+    vec_free (s);                               \
+    return handle;
+
+static inline ipfix_field_specifier_t *
+flowprobe_template_ip4_fields (ipfix_field_specifier_t * f)
+{
+#define flowprobe_template_ip4_field_count() 4
+  /* sourceIpv4Address, TLV type 8, u32 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      sourceIPv4Address, 4);
+  f++;
+  /* destinationIPv4Address, TLV type 12, u32 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      destinationIPv4Address, 4);
+  f++;
+  /* protocolIdentifier, TLV type 4, u8 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      protocolIdentifier, 1);
+  f++;
+  /* octetDeltaCount, TLV type 1, u64 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      octetDeltaCount, 8);
+  f++;
+  return f;
+}
+
+static inline ipfix_field_specifier_t *
+flowprobe_template_ip6_fields (ipfix_field_specifier_t * f)
+{
+#define flowprobe_template_ip6_field_count() 4
+  /* sourceIpv6Address, TLV type 27, 16 octets */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      sourceIPv6Address, 16);
+  f++;
+  /* destinationIPv6Address, TLV type 28, 16 octets */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      destinationIPv6Address, 16);
+  f++;
+  /* protocolIdentifier, TLV type 4, u8 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      protocolIdentifier, 1);
+  f++;
+  /* octetDeltaCount, TLV type 1, u64 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      octetDeltaCount, 8);
+  f++;
+  return f;
+}
+
+static inline ipfix_field_specifier_t *
+flowprobe_template_l2_fields (ipfix_field_specifier_t * f)
+{
+#define flowprobe_template_l2_field_count() 3
+  /* sourceMacAddress, TLV type 56, u8[6] we hope */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      sourceMacAddress, 6);
+  f++;
+  /* destinationMacAddress, TLV type 80, u8[6] we hope */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      destinationMacAddress, 6);
+  f++;
+  /* ethernetType, TLV type 256, u16 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      ethernetType, 2);
+  f++;
+  return f;
+}
+
+static inline ipfix_field_specifier_t *
+flowprobe_template_common_fields (ipfix_field_specifier_t * f)
+{
+#define flowprobe_template_common_field_count() 3
+  /* ingressInterface, TLV type 10, u32 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      ingressInterface, 4);
+  f++;
+
+  /* egressInterface, TLV type 14, u32 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      egressInterface, 4);
+  f++;
+
+  /* packetDeltaCount, TLV type 2, u64 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      packetDeltaCount, 8);
+  f++;
+
+  return f;
+}
+
+static inline ipfix_field_specifier_t *
+flowprobe_template_l4_fields (ipfix_field_specifier_t * f)
+{
+#define flowprobe_template_l4_field_count() 2
+  /* sourceTransportPort, TLV type 7, u16 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      sourceTransportPort, 2);
+  f++;
+  /* destinationTransportPort, TLV type 11, u16 */
+  f->e_id_length = ipfix_e_id_length (0 /* enterprise */ ,
+				      destinationTransportPort, 2);
+  f++;
+  return f;
+}
+
+/**
+ * @brief Create an IPFIX template packet rewrite string
+ * @param frm flow_report_main_t *
+ * @param fr flow_report_t *
+ * @param collector_address ip4_address_t * the IPFIX collector address
+ * @param src_address ip4_address_t * the source address we should use
+ * @param collector_port u16 the collector port we should use, host byte order
+ * @returns u8 * vector containing the indicated IPFIX template packet
+ */
+static inline u8 *
+flowprobe_template_rewrite_inline (flow_report_main_t * frm,
+				   flow_report_t * fr,
+				   ip4_address_t * collector_address,
+				   ip4_address_t * src_address,
+				   u16 collector_port,
+				   flowprobe_variant_t which)
+{
+  ip4_header_t *ip;
+  udp_header_t *udp;
+  ipfix_message_header_t *h;
+  ipfix_set_header_t *s;
+  ipfix_template_header_t *t;
+  ipfix_field_specifier_t *f;
+  ipfix_field_specifier_t *first_field;
+  u8 *rewrite = 0;
+  ip4_ipfix_template_packet_t *tp;
+  u32 field_count = 0;
+  flow_report_stream_t *stream;
+  flowprobe_main_t *fm = &flowprobe_main;
+  flowprobe_record_t flags = fr->opaque.as_uword;
+  bool collect_ip4 = false, collect_ip6 = false;
+
+  stream = &frm->streams[fr->stream_index];
+
+  if (flags & FLOW_RECORD_L3)
+    {
+      collect_ip4 = which == FLOW_VARIANT_L2_IP4 || which == FLOW_VARIANT_IP4;
+      collect_ip6 = which == FLOW_VARIANT_L2_IP6 || which == FLOW_VARIANT_IP6;
+      if (which == FLOW_VARIANT_L2_IP4)
+	flags |= FLOW_RECORD_L2_IP4;
+      if (which == FLOW_VARIANT_L2_IP6)
+	flags |= FLOW_RECORD_L2_IP6;
+    }
+
+  field_count += flowprobe_template_common_field_count ();
+  if (flags & FLOW_RECORD_L2)
+    field_count += flowprobe_template_l2_field_count ();
+  if (collect_ip4)
+    field_count += flowprobe_template_ip4_field_count ();
+  if (collect_ip6)
+    field_count += flowprobe_template_ip6_field_count ();
+  if (flags & FLOW_RECORD_L4)
+    field_count += flowprobe_template_l4_field_count ();
+
+  /* allocate rewrite space */
+  vec_validate_aligned
+    (rewrite, sizeof (ip4_ipfix_template_packet_t)
+     + field_count * sizeof (ipfix_field_specifier_t) - 1,
+     CLIB_CACHE_LINE_BYTES);
+
+  tp = (ip4_ipfix_template_packet_t *) rewrite;
+  ip = (ip4_header_t *) & tp->ip4;
+  udp = (udp_header_t *) (ip + 1);
+  h = (ipfix_message_header_t *) (udp + 1);
+  s = (ipfix_set_header_t *) (h + 1);
+  t = (ipfix_template_header_t *) (s + 1);
+  first_field = f = (ipfix_field_specifier_t *) (t + 1);
+
+  ip->ip_version_and_header_length = 0x45;
+  ip->ttl = 254;
+  ip->protocol = IP_PROTOCOL_UDP;
+  ip->src_address.as_u32 = src_address->as_u32;
+  ip->dst_address.as_u32 = collector_address->as_u32;
+  udp->src_port = clib_host_to_net_u16 (stream->src_port);
+  udp->dst_port = clib_host_to_net_u16 (collector_port);
+  udp->length = clib_host_to_net_u16 (vec_len (rewrite) - sizeof (*ip));
+
+  /* FIXUP: message header export_time */
+  /* FIXUP: message header sequence_number */
+  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
+
+  /* Add TLVs to the template */
+  f = flowprobe_template_common_fields (f);
+
+  if (flags & FLOW_RECORD_L2)
+    f = flowprobe_template_l2_fields (f);
+  if (collect_ip4)
+    f = flowprobe_template_ip4_fields (f);
+  if (collect_ip6)
+    f = flowprobe_template_ip6_fields (f);
+  if (flags & FLOW_RECORD_L4)
+    f = flowprobe_template_l4_fields (f);
+
+  /* Back to the template packet... */
+  ip = (ip4_header_t *) & tp->ip4;
+  udp = (udp_header_t *) (ip + 1);
+
+  ASSERT (f - first_field);
+  /* Field count in this template */
+  t->id_count = ipfix_id_count (fr->template_id, f - first_field);
+
+  fm->template_size[flags] = (u8 *) f - (u8 *) s;
+
+  /* set length in octets */
+  s->set_id_length =
+    ipfix_set_id_length (2 /* set_id */ , (u8 *) f - (u8 *) s);
+
+  /* message length in octets */
+  h->version_length = version_length ((u8 *) f - (u8 *) h);
+
+  ip->length = clib_host_to_net_u16 ((u8 *) f - (u8 *) ip);
+  ip->checksum = ip4_header_checksum (ip);
+
+  return rewrite;
+}
+
+static u8 *
+flowprobe_template_rewrite_ip6 (flow_report_main_t * frm,
+				flow_report_t * fr,
+				ip4_address_t * collector_address,
+				ip4_address_t * src_address,
+				u16 collector_port)
+{
+  return flowprobe_template_rewrite_inline
+    (frm, fr, collector_address, src_address, collector_port,
+     FLOW_VARIANT_IP6);
+}
+
+static u8 *
+flowprobe_template_rewrite_ip4 (flow_report_main_t * frm,
+				flow_report_t * fr,
+				ip4_address_t * collector_address,
+				ip4_address_t * src_address,
+				u16 collector_port)
+{
+  return flowprobe_template_rewrite_inline
+    (frm, fr, collector_address, src_address, collector_port,
+     FLOW_VARIANT_IP4);
+}
+
+static u8 *
+flowprobe_template_rewrite_l2 (flow_report_main_t * frm,
+			       flow_report_t * fr,
+			       ip4_address_t * collector_address,
+			       ip4_address_t * src_address,
+			       u16 collector_port)
+{
+  return flowprobe_template_rewrite_inline
+    (frm, fr, collector_address, src_address, collector_port,
+     FLOW_VARIANT_L2);
+}
+
+static u8 *
+flowprobe_template_rewrite_l2_ip4 (flow_report_main_t * frm,
+				   flow_report_t * fr,
+				   ip4_address_t * collector_address,
+				   ip4_address_t * src_address,
+				   u16 collector_port)
+{
+  return flowprobe_template_rewrite_inline
+    (frm, fr, collector_address, src_address, collector_port,
+     FLOW_VARIANT_L2_IP4);
+}
+
+static u8 *
+flowprobe_template_rewrite_l2_ip6 (flow_report_main_t * frm,
+				   flow_report_t * fr,
+				   ip4_address_t * collector_address,
+				   ip4_address_t * src_address,
+				   u16 collector_port)
+{
+  return flowprobe_template_rewrite_inline
+    (frm, fr, collector_address, src_address, collector_port,
+     FLOW_VARIANT_L2_IP6);
+}
+
+/**
+ * @brief Flush accumulated data
+ * @param frm flow_report_main_t *
+ * @param fr flow_report_t *
+ * @param f vlib_frame_t *
+ *
+ * Notes:
+ * This function must simply return the incoming frame, or no template packets
+ * will be sent.
+ */
+vlib_frame_t *
+flowprobe_data_callback_ip4 (flow_report_main_t * frm,
+			     flow_report_t * fr,
+			     vlib_frame_t * f, u32 * to_next, u32 node_index)
+{
+  flowprobe_flush_callback_ip4 ();
+  return f;
+}
+
+vlib_frame_t *
+flowprobe_data_callback_ip6 (flow_report_main_t * frm,
+			     flow_report_t * fr,
+			     vlib_frame_t * f, u32 * to_next, u32 node_index)
+{
+  flowprobe_flush_callback_ip6 ();
+  return f;
+}
+
+vlib_frame_t *
+flowprobe_data_callback_l2 (flow_report_main_t * frm,
+			    flow_report_t * fr,
+			    vlib_frame_t * f, u32 * to_next, u32 node_index)
+{
+  flowprobe_flush_callback_l2 ();
+  return f;
+}
+
+static int
+flowprobe_template_add_del (u32 domain_id, u16 src_port,
+			    flowprobe_record_t flags,
+			    vnet_flow_data_callback_t * flow_data_callback,
+			    vnet_flow_rewrite_callback_t * rewrite_callback,
+			    bool is_add, u16 * template_id)
+{
+  flow_report_main_t *frm = &flow_report_main;
+  vnet_flow_report_add_del_args_t a = {
+    .rewrite_callback = rewrite_callback,
+    .flow_data_callback = flow_data_callback,
+    .is_add = is_add,
+    .domain_id = domain_id,
+    .src_port = src_port,
+    .opaque.as_uword = flags,
+  };
+  return vnet_flow_report_add_del (frm, &a, template_id);
+}
+
+static void
+flowprobe_expired_timer_callback (u32 * expired_timers)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  flowprobe_main_t *fm = &flowprobe_main;
+  u32 my_cpu_number = vm->thread_index;
+  int i;
+  u32 poolindex;
+
+  for (i = 0; i < vec_len (expired_timers); i++)
+    {
+      poolindex = expired_timers[i] & 0x7FFFFFFF;
+      vec_add1 (fm->expired_passive_per_worker[my_cpu_number], poolindex);
+    }
+}
+
+static clib_error_t *
+flowprobe_create_state_tables (u32 active_timer)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  vlib_thread_main_t *tm = &vlib_thread_main;
+  vlib_main_t *vm = vlib_get_main ();
+  clib_error_t *error = 0;
+  u32 num_threads;
+  int i;
+
+  /* Decide how many worker threads we have */
+  num_threads = 1 /* main thread */  + tm->n_threads;
+
+  /* Hash table per worker */
+  fm->ht_log2len = FLOWPROBE_LOG2_HASHSIZE;
+
+  /* Init per worker flow state and timer wheels */
+  if (active_timer)
+    {
+      vec_validate (fm->timers_per_worker, num_threads - 1);
+      vec_validate (fm->expired_passive_per_worker, num_threads - 1);
+      vec_validate (fm->hash_per_worker, num_threads - 1);
+      vec_validate (fm->pool_per_worker, num_threads - 1);
+
+      for (i = 0; i < num_threads; i++)
+	{
+	  int j;
+	  pool_alloc (fm->pool_per_worker[i], 1 << fm->ht_log2len);
+	  vec_resize (fm->hash_per_worker[i], 1 << fm->ht_log2len);
+	  for (j = 0; j < (1 << fm->ht_log2len); j++)
+	    fm->hash_per_worker[i][j] = ~0;
+	  fm->timers_per_worker[i] =
+	    clib_mem_alloc (sizeof (TWT (tw_timer_wheel)));
+	  tw_timer_wheel_init_2t_1w_2048sl (fm->timers_per_worker[i],
+					    flowprobe_expired_timer_callback,
+					    1.0, 1024);
+	}
+      fm->disabled = true;
+    }
+  else
+    {
+      f64 now = vlib_time_now (vm);
+      vec_validate (fm->stateless_entry, num_threads - 1);
+      for (i = 0; i < num_threads; i++)
+	fm->stateless_entry[i].last_exported = now;
+      fm->disabled = false;
+    }
+  fm->initialized = true;
+  return error;
+}
+
+static int
+validate_feature_on_interface (flowprobe_main_t * fm, u32 sw_if_index,
+			       u8 which)
+{
+  vec_validate_init_empty (fm->flow_per_interface, sw_if_index, ~0);
+
+  if (fm->flow_per_interface[sw_if_index] == (u8) ~ 0)
+    return -1;
+  else if (fm->flow_per_interface[sw_if_index] != which)
+    return 0;
+  else
+    return 1;
+}
+
+/**
+ * @brief configure / deconfigure the IPFIX flow-per-packet
+ * @param fm flowprobe_main_t * fm
+ * @param sw_if_index u32 the desired interface
+ * @param is_add int 1 to enable the feature, 0 to disable it
+ * @returns 0 if successful, non-zero otherwise
+ */
+
+static int
+flowprobe_tx_interface_add_del_feature (flowprobe_main_t * fm,
+					u32 sw_if_index, u8 which, int is_add)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  int rv = 0;
+  u16 template_id = 0;
+  flowprobe_record_t flags = fm->record;
+
+  fm->flow_per_interface[sw_if_index] = (is_add) ? which : (u8) ~ 0;
+  fm->template_per_flow[which] += (is_add) ? 1 : -1;
+  if (is_add && fm->template_per_flow[which] > 1)
+    template_id = fm->template_reports[flags];
+
+  if ((is_add && fm->template_per_flow[which] == 1) ||
+      (!is_add && fm->template_per_flow[which] == 0))
+    {
+      if (which == FLOW_VARIANT_L2)
+	{
+	  if (fm->record & FLOW_RECORD_L2)
+	    {
+	      rv = flowprobe_template_add_del (1, UDP_DST_PORT_ipfix, flags,
+					       flowprobe_data_callback_l2,
+					       flowprobe_template_rewrite_l2,
+					       is_add, &template_id);
+	    }
+	  if (fm->record & FLOW_RECORD_L3 || fm->record & FLOW_RECORD_L4)
+	    {
+	      rv = flowprobe_template_add_del (1, UDP_DST_PORT_ipfix, flags,
+					       flowprobe_data_callback_l2,
+					       flowprobe_template_rewrite_l2_ip4,
+					       is_add, &template_id);
+	      fm->template_reports[flags | FLOW_RECORD_L2_IP4] =
+		(is_add) ? template_id : 0;
+	      rv =
+		flowprobe_template_add_del (1, UDP_DST_PORT_ipfix, flags,
+					    flowprobe_data_callback_l2,
+					    flowprobe_template_rewrite_l2_ip6,
+					    is_add, &template_id);
+	      fm->template_reports[flags | FLOW_RECORD_L2_IP6] =
+		(is_add) ? template_id : 0;
+
+	      /* Special case L2 */
+	      fm->context[FLOW_VARIANT_L2_IP4].flags =
+		flags | FLOW_RECORD_L2_IP4;
+	      fm->context[FLOW_VARIANT_L2_IP6].flags =
+		flags | FLOW_RECORD_L2_IP6;
+
+	      fm->template_reports[flags] = template_id;
+	    }
+	}
+      else if (which == FLOW_VARIANT_IP4)
+	rv = flowprobe_template_add_del (1, UDP_DST_PORT_ipfix, flags,
+					 flowprobe_data_callback_ip4,
+					 flowprobe_template_rewrite_ip4,
+					 is_add, &template_id);
+      else if (which == FLOW_VARIANT_IP6)
+	rv = flowprobe_template_add_del (1, UDP_DST_PORT_ipfix, flags,
+					 flowprobe_data_callback_ip6,
+					 flowprobe_template_rewrite_ip6,
+					 is_add, &template_id);
+    }
+  if (rv && rv != VNET_API_ERROR_VALUE_EXIST)
+    {
+      clib_warning ("vnet_flow_report_add_del returned %d", rv);
+      return -1;
+    }
+
+  if (which != (u8) ~ 0)
+    {
+      fm->context[which].flags = fm->record;
+      fm->template_reports[flags] = (is_add) ? template_id : 0;
+    }
+
+  if (which == FLOW_VARIANT_IP4)
+    vnet_feature_enable_disable ("ip4-output", "flowprobe-ip4",
+				 sw_if_index, is_add, 0, 0);
+  else if (which == FLOW_VARIANT_IP6)
+    vnet_feature_enable_disable ("ip6-output", "flowprobe-ip6",
+				 sw_if_index, is_add, 0, 0);
+  else if (which == FLOW_VARIANT_L2)
+    vnet_feature_enable_disable ("interface-output", "flowprobe-l2",
+				 sw_if_index, is_add, 0, 0);
+
+  /* Stateful flow collection */
+  if (is_add && !fm->initialized)
+    {
+      flowprobe_create_state_tables (fm->active_timer);
+      if (fm->active_timer)
+	vlib_process_signal_event (vm, flowprobe_timer_node.index, 1, 0);
+    }
+
+  return 0;
+}
+
+/**
+ * @brief API message handler
+ * @param mp vl_api_flowprobe_tx_interface_add_del_t * mp the api message
+ */
+void vl_api_flowprobe_tx_interface_add_del_t_handler
+  (vl_api_flowprobe_tx_interface_add_del_t * mp)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  vl_api_flowprobe_tx_interface_add_del_reply_t *rmp;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  if (mp->which != FLOW_VARIANT_IP4 && mp->which != FLOW_VARIANT_L2
+      && mp->which != FLOW_VARIANT_IP6)
+    {
+      rv = VNET_API_ERROR_UNIMPLEMENTED;
+      goto out;
+    }
+
+  if (fm->record == 0)
+    {
+      clib_warning ("Please specify flowprobe params record first...");
+      rv = VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE;
+      goto out;
+    }
+
+  rv = validate_feature_on_interface (fm, sw_if_index, mp->which);
+  if ((rv == 1 && mp->is_add == 1) || rv == 0)
+    {
+      rv = VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE;
+      goto out;
+    }
+
+  rv = flowprobe_tx_interface_add_del_feature
+    (fm, sw_if_index, mp->which, mp->is_add);
+
+out:
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_FLOWPROBE_TX_INTERFACE_ADD_DEL_REPLY);
+}
+
+/**
+ * @brief API message custom-dump function
+ * @param mp vl_api_flowprobe_tx_interface_add_del_t * mp the api message
+ * @param handle void * print function handle
+ * @returns u8 * output string
+ */
+static void *vl_api_flowprobe_tx_interface_add_del_t_print
+  (vl_api_flowprobe_tx_interface_add_del_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: flowprobe_tx_interface_add_del ");
+  s = format (s, "sw_if_index %d is_add %d which %d ",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      (int) mp->is_add, (int) mp->which);
+  FINISH;
+}
+
+#define vec_neg_search(v,E)         \
+({              \
+  word _v(i) = 0;         \
+  while (_v(i) < vec_len(v) && v[_v(i)] == E)        \
+  {             \
+    _v(i)++;            \
+  }             \
+  if (_v(i) == vec_len(v))        \
+    _v(i) = ~0;                 \
+  _v(i);            \
+})
+
+static int
+flowprobe_params (flowprobe_main_t * fm, u8 record_l2,
+		  u8 record_l3, u8 record_l4,
+		  u32 active_timer, u32 passive_timer)
+{
+  flowprobe_record_t flags = 0;
+
+  if (vec_neg_search (fm->flow_per_interface, (u8) ~ 0) != ~0)
+    return ~0;
+
+  if (record_l2)
+    flags |= FLOW_RECORD_L2;
+  if (record_l3)
+    flags |= FLOW_RECORD_L3;
+  if (record_l4)
+    flags |= FLOW_RECORD_L4;
+
+  fm->record = flags;
+
+  /*
+   * Timers: ~0 is default, 0 is off
+   */
+  fm->active_timer =
+    (active_timer == (u32) ~ 0 ? FLOWPROBE_TIMER_ACTIVE : active_timer);
+  fm->passive_timer =
+    (passive_timer == (u32) ~ 0 ? FLOWPROBE_TIMER_PASSIVE : passive_timer);
+
+  return 0;
+}
+
+void
+vl_api_flowprobe_params_t_handler (vl_api_flowprobe_params_t * mp)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  vl_api_flowprobe_params_reply_t *rmp;
+  int rv = 0;
+
+  rv = flowprobe_params
+    (fm, mp->record_l2, mp->record_l3, mp->record_l4,
+     clib_net_to_host_u32 (mp->active_timer),
+     clib_net_to_host_u32 (mp->passive_timer));
+
+  REPLY_MACRO (VL_API_FLOWPROBE_PARAMS_REPLY);
+}
+
+/* List of message types that this plugin understands */
+#define foreach_flowprobe_plugin_api_msg				\
+_(FLOWPROBE_TX_INTERFACE_ADD_DEL, flowprobe_tx_interface_add_del)	\
+_(FLOWPROBE_PARAMS, flowprobe_params)
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .description = "Flow per Packet",
+};
+/* *INDENT-ON* */
+
+u8 *
+format_flowprobe_entry (u8 * s, va_list * args)
+{
+  flowprobe_entry_t *e = va_arg (*args, flowprobe_entry_t *);
+  s = format (s, " %d/%d", e->key.rx_sw_if_index, e->key.tx_sw_if_index);
+
+  s = format (s, " %U %U", format_ethernet_address, &e->key.src_mac,
+	      format_ethernet_address, &e->key.dst_mac);
+  s = format (s, " %U -> %U",
+	      format_ip46_address, &e->key.src_address, IP46_TYPE_ANY,
+	      format_ip46_address, &e->key.dst_address, IP46_TYPE_ANY);
+  s = format (s, " %d", e->key.protocol);
+  s = format (s, " %d %d\n", clib_net_to_host_u16 (e->key.src_port),
+	      clib_net_to_host_u16 (e->key.dst_port));
+
+  return s;
+}
+
+static clib_error_t *
+flowprobe_show_table_fn (vlib_main_t * vm,
+			 unformat_input_t * input, vlib_cli_command_t * cm)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  int i;
+  flowprobe_entry_t *e;
+
+  vlib_cli_output (vm, "Dumping IPFIX table");
+
+  for (i = 0; i < vec_len (fm->pool_per_worker); i++)
+    {
+      /* *INDENT-OFF* */
+      pool_foreach (e, fm->pool_per_worker[i], (
+	{
+	  vlib_cli_output (vm, "%U",
+			   format_flowprobe_entry,
+			   e);
+	}));
+      /* *INDENT-ON* */
+
+    }
+  return 0;
+}
+
+static clib_error_t *
+flowprobe_show_stats_fn (vlib_main_t * vm,
+			 unformat_input_t * input, vlib_cli_command_t * cm)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  int i;
+
+  vlib_cli_output (vm, "IPFIX table statistics");
+  vlib_cli_output (vm, "Flow entry size: %d\n", sizeof (flowprobe_entry_t));
+  vlib_cli_output (vm, "Flow pool size per thread: %d\n",
+		   0x1 << FLOWPROBE_LOG2_HASHSIZE);
+
+  for (i = 0; i < vec_len (fm->pool_per_worker); i++)
+    vlib_cli_output (vm, "Pool utilisation thread %d is %d%%\n", i,
+		     (100 * pool_elts (fm->pool_per_worker[i])) /
+		     (0x1 << FLOWPROBE_LOG2_HASHSIZE));
+  return 0;
+}
+
+static clib_error_t *
+flowprobe_tx_interface_add_del_feature_command_fn (vlib_main_t * vm,
+						   unformat_input_t * input,
+						   vlib_cli_command_t * cmd)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  u32 sw_if_index = ~0;
+  int is_add = 1;
+  u8 which = FLOW_VARIANT_IP4;
+  int rv;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "disable"))
+	is_add = 0;
+      else if (unformat (input, "%U", unformat_vnet_sw_interface,
+			 fm->vnet_main, &sw_if_index));
+      else if (unformat (input, "ip4"))
+	which = FLOW_VARIANT_IP4;
+      else if (unformat (input, "ip6"))
+	which = FLOW_VARIANT_IP6;
+      else if (unformat (input, "l2"))
+	which = FLOW_VARIANT_L2;
+      else
+	break;
+    }
+
+  if (fm->record == 0)
+    return clib_error_return (0,
+			      "Please specify flowprobe params record first...");
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0, "Please specify an interface...");
+
+  rv = validate_feature_on_interface (fm, sw_if_index, which);
+  if (rv == 1)
+    {
+      if (is_add)
+	return clib_error_return (0,
+				  "Datapath is already enabled for given interface...");
+    }
+  else if (rv == 0)
+    return clib_error_return (0,
+			      "Interface has enable different datapath ...");
+
+  rv =
+    flowprobe_tx_interface_add_del_feature (fm, sw_if_index, which, is_add);
+  switch (rv)
+    {
+    case 0:
+      break;
+
+    case VNET_API_ERROR_INVALID_SW_IF_INDEX:
+      return clib_error_return
+	(0, "Invalid interface, only works on physical ports");
+      break;
+
+    case VNET_API_ERROR_UNIMPLEMENTED:
+      return clib_error_return (0, "ip6 not supported");
+      break;
+
+    default:
+      return clib_error_return (0, "flowprobe_enable_disable returned %d",
+				rv);
+    }
+  return 0;
+}
+
+static clib_error_t *
+flowprobe_params_command_fn (vlib_main_t * vm,
+			     unformat_input_t * input,
+			     vlib_cli_command_t * cmd)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  bool record_l2 = false, record_l3 = false, record_l4 = false;
+  u32 active_timer = ~0;
+  u32 passive_timer = ~0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "active %d", &active_timer))
+	;
+      else if (unformat (input, "passive %d", &passive_timer))
+	;
+      else if (unformat (input, "record"))
+	while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+	  {
+	    if (unformat (input, "l2"))
+	      record_l2 = true;
+	    else if (unformat (input, "l3"))
+	      record_l3 = true;
+	    else if (unformat (input, "l4"))
+	      record_l4 = true;
+	    else
+	      break;
+	  }
+      else
+	break;
+    }
+
+  if (passive_timer > 0 && active_timer > passive_timer)
+    return clib_error_return (0,
+			      "Passive timer has to be greater than active one...");
+
+  if (flowprobe_params (fm, record_l2, record_l3, record_l4,
+			active_timer, passive_timer))
+    return clib_error_return (0,
+			      "Couldn't change flowperpacket params when feature is enabled on some interface ...");
+  return 0;
+}
+
+/*?
+ * 'flowprobe feature add-del' commands to enable/disable
+ * per-packet IPFIX flow record generation on an interface
+ *
+ * @cliexpar
+ * @parblock
+ * To enable per-packet IPFIX flow-record generation on an interface:
+ * @cliexcmd{flowprobe feature add-del GigabitEthernet2/0/0}
+ *
+ * To disable per-packet IPFIX flow-record generation on an interface:
+ * @cliexcmd{flowprobe feature add-del GigabitEthernet2/0/0 disable}
+ * @cliexend
+ * @endparblock
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (flowprobe_enable_disable_command, static) = {
+    .path = "flowprobe feature add-del",
+    .short_help =
+    "flowprobe feature add-del   disable",
+    .function = flowprobe_tx_interface_add_del_feature_command_fn,
+};
+VLIB_CLI_COMMAND (flowprobe_params_command, static) = {
+    .path = "flowprobe params",
+    .short_help =
+    "flowprobe params record <[l2] [l3] [l4]> [active  passive ]",
+    .function = flowprobe_params_command_fn,
+};
+VLIB_CLI_COMMAND (flowprobe_show_table_command, static) = {
+    .path = "show flowprobe table",
+    .short_help = "show flowprobe table",
+    .function = flowprobe_show_table_fn,
+};
+VLIB_CLI_COMMAND (flowprobe_show_stats_command, static) = {
+    .path = "show flowprobe statistics",
+    .short_help = "show flowprobe statistics",
+    .function = flowprobe_show_stats_fn,
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Set up the API message handling tables
+ * @param vm vlib_main_t * vlib main data structure pointer
+ * @returns 0 to indicate all is well
+ */
+static clib_error_t *
+flowprobe_plugin_api_hookup (vlib_main_t * vm)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + fm->msg_id_base),     \
+                           #n,					\
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_flowprobe_plugin_api_msg;
+#undef _
+
+  return 0;
+}
+
+#define vl_msg_name_crc_list
+#include 
+#undef vl_msg_name_crc_list
+
+static void
+setup_message_id_table (flowprobe_main_t * fm, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + fm->msg_id_base);
+  foreach_vl_msg_name_crc_flowprobe;
+#undef _
+}
+
+/*
+ * Main-core process, sending an interrupt to the per worker input
+ * process that spins the per worker timer wheel.
+ */
+static uword
+timer_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
+{
+  uword *event_data = 0;
+  vlib_main_t **worker_vms = 0, *worker_vm;
+  flowprobe_main_t *fm = &flowprobe_main;
+
+  /* Wait for Godot... */
+  vlib_process_wait_for_event_or_clock (vm, 1e9);
+  uword event_type = vlib_process_get_events (vm, &event_data);
+  if (event_type != 1)
+    clib_warning ("bogus kickoff event received, %d", event_type);
+  vec_reset_length (event_data);
+
+  int i;
+  if (vec_len (vlib_mains) == 0)
+    vec_add1 (worker_vms, vm);
+  else
+    {
+      for (i = 0; i < vec_len (vlib_mains); i++)
+	{
+	  worker_vm = vlib_mains[i];
+	  if (worker_vm)
+	    vec_add1 (worker_vms, worker_vm);
+	}
+    }
+  f64 sleep_duration = 0.1;
+
+  while (1)
+    {
+      /* Send an interrupt to each timer input node */
+      sleep_duration = 0.1;
+      for (i = 0; i < vec_len (worker_vms); i++)
+	{
+	  worker_vm = worker_vms[i];
+	  if (worker_vm)
+	    {
+	      vlib_node_set_interrupt_pending (worker_vm,
+					       flowprobe_walker_node.index);
+	      sleep_duration =
+		(fm->expired_passive_per_worker[i] > 0) ? 1e-4 : 0.1;
+	    }
+	}
+      vlib_process_suspend (vm, sleep_duration);
+    }
+  return 0;			/* or not */
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (flowprobe_timer_node,static) = {
+  .function = timer_process,
+  .name = "flowprobe-timer-process",
+  .type = VLIB_NODE_TYPE_PROCESS,
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Set up the API message handling tables
+ * @param vm vlib_main_t * vlib main data structure pointer
+ * @returns 0 to indicate all is well, or a clib_error_t
+ */
+static clib_error_t *
+flowprobe_init (vlib_main_t * vm)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  vlib_thread_main_t *tm = &vlib_thread_main;
+  clib_error_t *error = 0;
+  u8 *name;
+  u32 num_threads;
+  int i;
+
+  fm->vnet_main = vnet_get_main ();
+
+  /* Construct the API name */
+  name = format (0, "flowprobe_%08x%c", api_version, 0);
+
+  /* Ask for a correctly-sized block of API message decode slots */
+  fm->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+  /* Hook up message handlers */
+  error = flowprobe_plugin_api_hookup (vm);
+
+  /* Add our API messages to the global name_crc hash table */
+  setup_message_id_table (fm, &api_main);
+
+  vec_free (name);
+
+  /* Set up time reference pair */
+  fm->vlib_time_0 = vlib_time_now (vm);
+  fm->nanosecond_time_0 = unix_time_now_nsec ();
+
+  memset (fm->template_reports, 0, sizeof (fm->template_reports));
+  memset (fm->template_size, 0, sizeof (fm->template_size));
+  memset (fm->template_per_flow, 0, sizeof (fm->template_per_flow));
+
+  /* Decide how many worker threads we have */
+  num_threads = 1 /* main thread */  + tm->n_threads;
+
+  /* Allocate per worker thread vectors per flavour */
+  for (i = 0; i < FLOW_N_VARIANTS; i++)
+    {
+      vec_validate (fm->context[i].buffers_per_worker, num_threads - 1);
+      vec_validate (fm->context[i].frames_per_worker, num_threads - 1);
+      vec_validate (fm->context[i].next_record_offset_per_worker,
+		    num_threads - 1);
+    }
+
+  fm->active_timer = FLOWPROBE_TIMER_ACTIVE;
+  fm->passive_timer = FLOWPROBE_TIMER_PASSIVE;
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (flowprobe_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/flowprobe/flowprobe.h b/src/plugins/flowprobe/flowprobe.h
new file mode 100644
index 00000000..196c92a7
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe.h
@@ -0,0 +1,167 @@
+/*
+ * flowprobe.h - ipfix probe plug-in header file
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_flowprobe_h__
+#define __included_flowprobe_h__
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Default timers in seconds */
+#define FLOWPROBE_TIMER_ACTIVE   (15)
+#define FLOWPROBE_TIMER_PASSIVE  120	// XXXX: FOR TESTING (30*60)
+#define FLOWPROBE_LOG2_HASHSIZE  (18)
+
+typedef enum
+{
+  FLOW_RECORD_L2 = 1 << 0,
+  FLOW_RECORD_L3 = 1 << 1,
+  FLOW_RECORD_L4 = 1 << 2,
+  FLOW_RECORD_L2_IP4 = 1 << 3,
+  FLOW_RECORD_L2_IP6 = 1 << 4,
+  FLOW_N_RECORDS = 1 << 5,
+} flowprobe_record_t;
+
+/* *INDENT-OFF* */
+typedef enum __attribute__ ((__packed__))
+{
+  FLOW_VARIANT_IP4,
+  FLOW_VARIANT_IP6,
+  FLOW_VARIANT_L2,
+  FLOW_VARIANT_L2_IP4,
+  FLOW_VARIANT_L2_IP6,
+  FLOW_N_VARIANTS,
+} flowprobe_variant_t;
+/* *INDENT-ON* */
+
+STATIC_ASSERT (sizeof (flowprobe_variant_t) == 1,
+	       "flowprobe_variant_t is expected to be 1 byte, "
+	       "revisit padding in flowprobe_key_t");
+
+#define FLOW_MAXIMUM_EXPORT_ENTRIES	(1024)
+
+typedef struct
+{
+  /* what to collect per variant */
+  flowprobe_record_t flags;
+  /** ipfix buffers under construction, per-worker thread */
+  vlib_buffer_t **buffers_per_worker;
+  /** frames containing ipfix buffers, per-worker thread */
+  vlib_frame_t **frames_per_worker;
+  /** next record offset, per worker thread */
+  u16 *next_record_offset_per_worker;
+} flowprobe_protocol_context_t;
+
+#define FLOWPROBE_KEY_IN_U32 22
+/* *INDENT-OFF* */
+typedef CLIB_PACKED (union
+{
+  struct {
+    u32 rx_sw_if_index;
+    u32 tx_sw_if_index;
+    u8 src_mac[6];
+    u8 dst_mac[6];
+    u16 ethertype;
+    ip46_address_t src_address;
+    ip46_address_t dst_address;
+    u8 protocol;
+    u16 src_port;
+    u16 dst_port;
+    flowprobe_variant_t which;
+  };
+  u32 as_u32[FLOWPROBE_KEY_IN_U32];
+}) flowprobe_key_t;
+/* *INDENT-ON* */
+
+STATIC_ASSERT (sizeof (flowprobe_key_t) == FLOWPROBE_KEY_IN_U32 *
+	       sizeof (u32), "flowprobe_key_t padding is wrong");
+
+typedef struct
+{
+  flowprobe_key_t key;
+  u64 packetcount;
+  u64 octetcount;
+  f64 last_updated;
+  f64 last_exported;
+  u32 passive_timer_handle;
+} flowprobe_entry_t;
+
+/**
+ * @file
+ * @brief flow-per-packet plugin header file
+ */
+typedef struct
+{
+  /** API message ID base */
+  u16 msg_id_base;
+
+  flowprobe_protocol_context_t context[FLOW_N_VARIANTS];
+  u16 template_reports[FLOW_N_RECORDS];
+  u16 template_size[FLOW_N_RECORDS];
+
+  /** Time reference pair */
+  u64 nanosecond_time_0;
+  f64 vlib_time_0;
+
+  /** Per CPU flow-state */
+  u8 ht_log2len;		/* Hash table size is 2^log2len */
+  u32 **hash_per_worker;
+  flowprobe_entry_t **pool_per_worker;
+  /* *INDENT-OFF* */
+  TWT (tw_timer_wheel) ** timers_per_worker;
+  /* *INDENT-ON* */
+  u32 **expired_passive_per_worker;
+
+  flowprobe_record_t record;
+  u32 active_timer;
+  u32 passive_timer;
+  flowprobe_entry_t *stateless_entry;
+
+  bool initialized;
+  bool disabled;
+
+  u16 template_per_flow[FLOW_N_VARIANTS];
+  u8 *flow_per_interface;
+
+  /** convenience vlib_main_t pointer */
+  vlib_main_t *vlib_main;
+  /** convenience vnet_main_t pointer */
+  vnet_main_t *vnet_main;
+} flowprobe_main_t;
+
+extern flowprobe_main_t flowprobe_main;
+
+void flowprobe_flush_callback_ip4 (void);
+void flowprobe_flush_callback_ip6 (void);
+void flowprobe_flush_callback_l2 (void);
+u8 *format_flowprobe_entry (u8 * s, va_list * args);
+
+#endif
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/flowprobe/flowprobe_all_api_h.h b/src/plugins/flowprobe/flowprobe_all_api_h.h
new file mode 100644
index 00000000..1f30eccc
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe_all_api_h.h
@@ -0,0 +1,18 @@
+/*
+ * flowprobe_all_api_h.h - plug-in api #include file
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include 
diff --git a/src/plugins/flowprobe/flowprobe_msg_enum.h b/src/plugins/flowprobe/flowprobe_msg_enum.h
new file mode 100644
index 00000000..bc0b21c9
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe_msg_enum.h
@@ -0,0 +1,31 @@
+/*
+ * flowprobe_msg_enum.h - vpp engine plug-in message enumeration
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_flowprobe_msg_enum_h
+#define included_flowprobe_msg_enum_h
+
+#include 
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include 
+  /* We'll want to know how many messages IDs we need... */
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_flowprobe_msg_enum_h */
diff --git a/src/plugins/flowprobe/flowprobe_plugin_doc.md b/src/plugins/flowprobe/flowprobe_plugin_doc.md
new file mode 100644
index 00000000..4c9b2342
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe_plugin_doc.md
@@ -0,0 +1,13 @@
+IPFIX flow record plugin    {#flowprobe_plugin_doc}
+========================
+
+## Introduction
+
+This plugin generates ipfix flow records on interfaces which have the feature enabled
+
+## Sample configuration
+
+set ipfix exporter collector 192.168.6.2 src 192.168.6.1 template-interval 20 port 4739 path-mtu 1500
+
+flowprobe params record l3 active 20 passive 120
+flowprobe feature add-del GigabitEthernet2/3/0 l2
\ No newline at end of file
diff --git a/src/plugins/flowprobe/flowprobe_test.c b/src/plugins/flowprobe/flowprobe_test.c
new file mode 100644
index 00000000..91793f55
--- /dev/null
+++ b/src/plugins/flowprobe/flowprobe_test.c
@@ -0,0 +1,263 @@
+/*
+ * flowprobe.c - skeleton vpp-api-test plug-in
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define __plugin_msg_base flowprobe_test_main.msg_id_base
+#include 
+
+/**
+ * @file vpp_api_test plugin
+ */
+
+uword unformat_sw_if_index (unformat_input_t * input, va_list * args);
+
+/* Declare message IDs */
+#include 
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+
+#define vl_endianfun		/* define message structures */
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+typedef struct
+{
+    /** API message ID base */
+  u16 msg_id_base;
+    /** vat_main_t pointer */
+  vat_main_t *vat_main;
+} flowprobe_test_main_t;
+
+flowprobe_test_main_t flowprobe_test_main;
+
+#define foreach_standard_reply_retval_handler   \
+_(flowprobe_tx_interface_add_del_reply)        \
+_(flowprobe_params_reply)
+
+#define _(n)                                            \
+    static void vl_api_##n##_t_handler                  \
+    (vl_api_##n##_t * mp)                               \
+    {                                                   \
+        vat_main_t * vam = flowprobe_test_main.vat_main;   \
+        i32 retval = ntohl(mp->retval);                 \
+        if (vam->async_mode) {                          \
+            vam->async_errors += (retval < 0);          \
+        } else {                                        \
+            vam->retval = retval;                       \
+            vam->result_ready = 1;                      \
+        }                                               \
+    }
+foreach_standard_reply_retval_handler;
+#undef _
+
+/*
+ * Table of message reply handlers, must include boilerplate handlers
+ * we just generated
+ */
+#define foreach_vpe_api_reply_msg               \
+_(FLOWPROBE_TX_INTERFACE_ADD_DEL_REPLY,        \
+  flowprobe_tx_interface_add_del_reply)        \
+_(FLOWPROBE_PARAMS_REPLY, flowprobe_params_reply)
+
+static int
+api_flowprobe_tx_interface_add_del (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  int enable_disable = 1;
+  u8 which = FLOW_VARIANT_IP4;
+  u32 sw_if_index = ~0;
+  vl_api_flowprobe_tx_interface_add_del_t *mp;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
+	;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (i, "disable"))
+	enable_disable = 0;
+      else if (unformat (i, "ip4"))
+	which = FLOW_VARIANT_IP4;
+      else if (unformat (i, "ip6"))
+	which = FLOW_VARIANT_IP6;
+      else if (unformat (i, "l2"))
+	which = FLOW_VARIANT_L2;
+      else
+	break;
+    }
+
+  if (sw_if_index == ~0)
+    {
+      errmsg ("missing interface name / explicit sw_if_index number \n");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (FLOWPROBE_TX_INTERFACE_ADD_DEL, mp);
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->is_add = enable_disable;
+  mp->which = which;
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply... */
+  W (ret);
+  return ret;
+}
+
+static int
+api_flowprobe_params (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  u8 record_l2 = 0, record_l3 = 0, record_l4 = 0;
+  u32 active_timer = ~0;
+  u32 passive_timer = ~0;
+  vl_api_flowprobe_params_t *mp;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "active %d", &active_timer))
+	;
+      else if (unformat (i, "passive %d", &passive_timer))
+	;
+      else if (unformat (i, "record"))
+	while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+	  {
+	    if (unformat (i, "l2"))
+	      record_l2 = 1;
+	    else if (unformat (i, "l3"))
+	      record_l3 = 1;
+	    else if (unformat (i, "l4"))
+	      record_l4 = 1;
+	    else
+	      break;
+	  }
+      else
+	break;
+    }
+
+  if (passive_timer > 0 && active_timer > passive_timer)
+    {
+      errmsg ("Passive timer has to be greater than active one...\n");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (FLOWPROBE_PARAMS, mp);
+  mp->record_l2 = record_l2;
+  mp->record_l3 = record_l3;
+  mp->record_l4 = record_l4;
+  mp->active_timer = ntohl (active_timer);
+  mp->passive_timer = ntohl (passive_timer);
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply... */
+  W (ret);
+
+  return ret;
+}
+
+/*
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg \
+_(flowprobe_tx_interface_add_del, " [disable]") \
+_(flowprobe_params, "record <[l2] [l3] [l4]> [active  passive ]")
+
+static void
+flowprobe_vat_api_hookup (vat_main_t * vam)
+{
+  flowprobe_test_main_t *sm = &flowprobe_test_main;
+  /* Hook up handlers for replies from the data plane plug-in */
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),     \
+                           #n,                                  \
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_vpe_api_reply_msg;
+#undef _
+
+  /* API messages we can send */
+#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _
+
+  /* Help strings */
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t *
+vat_plugin_register (vat_main_t * vam)
+{
+  flowprobe_test_main_t *sm = &flowprobe_test_main;
+  u8 *name;
+
+  sm->vat_main = vam;
+
+  /* Ask the vpp engine for the first assigned message-id */
+  name = format (0, "flowprobe_%08x%c", api_version, 0);
+  sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  /* Don't attempt to hook up API messages if the data plane plugin is AWOL */
+  if (sm->msg_id_base != (u16) ~ 0)
+    flowprobe_vat_api_hookup (vam);
+
+  vec_free (name);
+
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/flowprobe/node.c b/src/plugins/flowprobe/node.c
new file mode 100644
index 00000000..6a539db9
--- /dev/null
+++ b/src/plugins/flowprobe/node.c
@@ -0,0 +1,998 @@
+/*
+ * node.c - ipfix probe graph node
+ *
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+static void flowprobe_export_entry (vlib_main_t * vm, flowprobe_entry_t * e);
+
+/**
+ * @file flow record generator graph node
+ */
+
+typedef struct
+{
+  /** interface handle */
+  u32 rx_sw_if_index;
+  u32 tx_sw_if_index;
+  /** packet timestamp */
+  u64 timestamp;
+  /** size of the buffer */
+  u16 buffer_size;
+
+  /** L2 information */
+  u8 src_mac[6];
+  u8 dst_mac[6];
+  /** Ethertype */
+  u16 ethertype;
+
+  /** L3 information */
+  ip46_address_t src_address;
+  ip46_address_t dst_address;
+  u8 protocol;
+  u8 tos;
+
+  /** L4 information */
+  u16 src_port;
+  u16 dst_port;
+
+  flowprobe_variant_t which;
+} flowprobe_trace_t;
+
+static char *flowprobe_variant_strings[] = {
+  [FLOW_VARIANT_IP4] = "IP4",
+  [FLOW_VARIANT_IP6] = "IP6",
+  [FLOW_VARIANT_L2] = "L2",
+  [FLOW_VARIANT_L2_IP4] = "L2-IP4",
+  [FLOW_VARIANT_L2_IP6] = "L2-IP6",
+};
+
+/* packet trace format function */
+static u8 *
+format_flowprobe_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  flowprobe_trace_t *t = va_arg (*args, flowprobe_trace_t *);
+  uword indent = format_get_indent (s);
+
+  s = format (s,
+	      "FLOWPROBE[%s]: rx_sw_if_index %d, tx_sw_if_index %d, "
+	      "timestamp %lld, size %d", flowprobe_variant_strings[t->which],
+	      t->rx_sw_if_index, t->tx_sw_if_index,
+	      t->timestamp, t->buffer_size);
+
+  if (t->which == FLOW_VARIANT_L2)
+    s = format (s, "\n%U -> %U", format_white_space, indent,
+		format_ethernet_address, &t->src_mac,
+		format_ethernet_address, &t->dst_mac);
+
+  if (t->protocol > 0
+      && (t->which == FLOW_VARIANT_L2_IP4 || t->which == FLOW_VARIANT_IP4
+	  || t->which == FLOW_VARIANT_L2_IP6 || t->which == FLOW_VARIANT_IP6))
+    s =
+      format (s, "\n%U%U: %U -> %U", format_white_space, indent,
+	      format_ip_protocol, t->protocol, format_ip46_address,
+	      &t->src_address, IP46_TYPE_ANY, format_ip46_address,
+	      &t->dst_address, IP46_TYPE_ANY);
+  return s;
+}
+
+vlib_node_registration_t flowprobe_ip4_node;
+vlib_node_registration_t flowprobe_ip6_node;
+vlib_node_registration_t flowprobe_l2_node;
+
+/* No counters at the moment */
+#define foreach_flowprobe_error			\
+_(COLLISION, "Hash table collisions")		\
+_(BUFFER, "Buffer allocation error")		\
+_(EXPORTED_PACKETS, "Exported packets")		\
+_(INPATH, "Exported packets in path")
+
+typedef enum
+{
+#define _(sym,str) FLOWPROBE_ERROR_##sym,
+  foreach_flowprobe_error
+#undef _
+    FLOWPROBE_N_ERROR,
+} flowprobe_error_t;
+
+static char *flowprobe_error_strings[] = {
+#define _(sym,string) string,
+  foreach_flowprobe_error
+#undef _
+};
+
+typedef enum
+{
+  FLOWPROBE_NEXT_DROP,
+  FLOWPROBE_NEXT_IP4_LOOKUP,
+  FLOWPROBE_N_NEXT,
+} flowprobe_next_t;
+
+#define FLOWPROBE_NEXT_NODES {					\
+    [FLOWPROBE_NEXT_DROP] = "error-drop",			\
+    [FLOWPROBE_NEXT_IP4_LOOKUP] = "ip4-lookup",		\
+}
+
+static inline flowprobe_variant_t
+flowprobe_get_variant (flowprobe_variant_t which,
+		       flowprobe_record_t flags, u16 ethertype)
+{
+  if (which == FLOW_VARIANT_L2
+      && (flags & FLOW_RECORD_L3 || flags & FLOW_RECORD_L4))
+    return ethertype == ETHERNET_TYPE_IP6 ? FLOW_VARIANT_L2_IP6 : ethertype ==
+      ETHERNET_TYPE_IP4 ? FLOW_VARIANT_L2_IP4 : FLOW_VARIANT_L2;
+  return which;
+}
+
+static inline u32
+flowprobe_common_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset)
+{
+  u16 start = offset;
+
+  /* Ingress interface */
+  u32 rx_if = clib_host_to_net_u32 (e->key.rx_sw_if_index);
+  clib_memcpy (to_b->data + offset, &rx_if, sizeof (rx_if));
+  offset += sizeof (rx_if);
+
+  /* Egress interface */
+  u32 tx_if = clib_host_to_net_u32 (e->key.tx_sw_if_index);
+  clib_memcpy (to_b->data + offset, &tx_if, sizeof (tx_if));
+  offset += sizeof (tx_if);
+
+  /* packet delta count */
+  u64 packetdelta = clib_host_to_net_u64 (e->packetcount);
+  clib_memcpy (to_b->data + offset, &packetdelta, sizeof (u64));
+  offset += sizeof (u64);
+
+  return offset - start;
+}
+
+static inline u32
+flowprobe_l2_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset)
+{
+  u16 start = offset;
+
+  /* src mac address */
+  clib_memcpy (to_b->data + offset, &e->key.src_mac, 6);
+  offset += 6;
+
+  /* dst mac address */
+  clib_memcpy (to_b->data + offset, &e->key.dst_mac, 6);
+  offset += 6;
+
+  /* ethertype */
+  clib_memcpy (to_b->data + offset, &e->key.ethertype, 2);
+  offset += 2;
+
+  return offset - start;
+}
+
+static inline u32
+flowprobe_l3_ip6_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset)
+{
+  u16 start = offset;
+
+  /* ip6 src address */
+  clib_memcpy (to_b->data + offset, &e->key.src_address,
+	       sizeof (ip6_address_t));
+  offset += sizeof (ip6_address_t);
+
+  /* ip6 dst address */
+  clib_memcpy (to_b->data + offset, &e->key.dst_address,
+	       sizeof (ip6_address_t));
+  offset += sizeof (ip6_address_t);
+
+  /* Protocol */
+  to_b->data[offset++] = e->key.protocol;
+
+  /* octetDeltaCount */
+  u64 octetdelta = clib_host_to_net_u64 (e->octetcount);
+  clib_memcpy (to_b->data + offset, &octetdelta, sizeof (u64));
+  offset += sizeof (u64);
+
+  return offset - start;
+}
+
+static inline u32
+flowprobe_l3_ip4_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset)
+{
+  u16 start = offset;
+
+  /* ip4 src address */
+  clib_memcpy (to_b->data + offset, &e->key.src_address.ip4,
+	       sizeof (ip4_address_t));
+  offset += sizeof (ip4_address_t);
+
+  /* ip4 dst address */
+  clib_memcpy (to_b->data + offset, &e->key.dst_address.ip4,
+	       sizeof (ip4_address_t));
+  offset += sizeof (ip4_address_t);
+
+  /* Protocol */
+  to_b->data[offset++] = e->key.protocol;
+
+  /* octetDeltaCount */
+  u64 octetdelta = clib_host_to_net_u64 (e->octetcount);
+  clib_memcpy (to_b->data + offset, &octetdelta, sizeof (u64));
+  offset += sizeof (u64);
+
+  return offset - start;
+}
+
+static inline u32
+flowprobe_l4_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset)
+{
+  u16 start = offset;
+
+  /* src port */
+  clib_memcpy (to_b->data + offset, &e->key.src_port, 2);
+  offset += 2;
+
+  /* dst port */
+  clib_memcpy (to_b->data + offset, &e->key.dst_port, 2);
+  offset += 2;
+
+  return offset - start;
+}
+
+static inline u32
+flowprobe_hash (flowprobe_key_t * k)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  int i;
+  u32 h = 0;
+  for (i = 0; i < sizeof (k->as_u32) / sizeof (u32); i++)
+    h = crc_u32 (k->as_u32[i], h);
+  return h >> (32 - fm->ht_log2len);
+}
+
+flowprobe_entry_t *
+flowprobe_lookup (u32 my_cpu_number, flowprobe_key_t * k, u32 * poolindex,
+		  bool * collision)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  flowprobe_entry_t *e;
+  u32 h;
+
+  h = (fm->active_timer) ? flowprobe_hash (k) : 0;
+
+  /* Lookup in the flow state pool */
+  *poolindex = fm->hash_per_worker[my_cpu_number][h];
+  if (*poolindex != ~0)
+    {
+      e = pool_elt_at_index (fm->pool_per_worker[my_cpu_number], *poolindex);
+      if (e)
+	{
+	  /* Verify key or report collision */
+	  if (memcmp (k, &e->key, sizeof (flowprobe_key_t)))
+	    *collision = true;
+	  return e;
+	}
+    }
+
+  return 0;
+}
+
+flowprobe_entry_t *
+flowprobe_create (u32 my_cpu_number, flowprobe_key_t * k, u32 * poolindex)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  u32 h;
+
+  flowprobe_entry_t *e;
+
+  /* Get my index */
+  h = (fm->active_timer) ? flowprobe_hash (k) : 0;
+
+  pool_get (fm->pool_per_worker[my_cpu_number], e);
+  *poolindex = e - fm->pool_per_worker[my_cpu_number];
+  fm->hash_per_worker[my_cpu_number][h] = *poolindex;
+
+  e->key = *k;
+
+  if (fm->passive_timer > 0)
+    {
+      e->passive_timer_handle = tw_timer_start_2t_1w_2048sl
+	(fm->timers_per_worker[my_cpu_number], *poolindex, 0,
+	 fm->passive_timer);
+    }
+  return e;
+}
+
+static inline void
+add_to_flow_record_state (vlib_main_t * vm, vlib_node_runtime_t * node,
+			  flowprobe_main_t * fm, vlib_buffer_t * b,
+			  u64 timestamp, u16 length,
+			  flowprobe_variant_t which, flowprobe_trace_t * t)
+{
+  if (fm->disabled)
+    return;
+
+  u32 my_cpu_number = vm->thread_index;
+  u16 octets = 0;
+
+  flowprobe_record_t flags = fm->context[which].flags;
+  bool collect_ip4 = false, collect_ip6 = false;
+  ASSERT (b);
+  ethernet_header_t *eth = vlib_buffer_get_current (b);
+  u16 ethertype = clib_net_to_host_u16 (eth->type);
+  /* *INDENT-OFF* */
+  flowprobe_key_t k = { {0} };
+  /* *INDENT-ON* */
+  ip4_header_t *ip4 = 0;
+  ip6_header_t *ip6 = 0;
+  udp_header_t *udp = 0;
+
+  if (flags & FLOW_RECORD_L3 || flags & FLOW_RECORD_L4)
+    {
+      collect_ip4 = which == FLOW_VARIANT_L2_IP4 || which == FLOW_VARIANT_IP4;
+      collect_ip6 = which == FLOW_VARIANT_L2_IP6 || which == FLOW_VARIANT_IP6;
+    }
+
+  k.rx_sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
+  k.tx_sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_TX];
+
+  k.which = which;
+
+  if (flags & FLOW_RECORD_L2)
+    {
+      clib_memcpy (k.src_mac, eth->src_address, 6);
+      clib_memcpy (k.dst_mac, eth->dst_address, 6);
+      k.ethertype = ethertype;
+    }
+  if (collect_ip6 && ethertype == ETHERNET_TYPE_IP6)
+    {
+      ip6 = (ip6_header_t *) (eth + 1);
+      udp = (udp_header_t *) (ip6 + 1);
+      if (flags & FLOW_RECORD_L3)
+	{
+	  k.src_address.as_u64[0] = ip6->src_address.as_u64[0];
+	  k.src_address.as_u64[1] = ip6->src_address.as_u64[1];
+	  k.dst_address.as_u64[0] = ip6->dst_address.as_u64[0];
+	  k.dst_address.as_u64[1] = ip6->dst_address.as_u64[1];
+	}
+      k.protocol = ip6->protocol;
+      octets = clib_net_to_host_u16 (ip6->payload_length)
+	+ sizeof (ip6_header_t);
+    }
+  if (collect_ip4 && ethertype == ETHERNET_TYPE_IP4)
+    {
+      ip4 = (ip4_header_t *) (eth + 1);
+      udp = (udp_header_t *) (ip4 + 1);
+      if (flags & FLOW_RECORD_L3)
+	{
+	  k.src_address.ip4.as_u32 = ip4->src_address.as_u32;
+	  k.dst_address.ip4.as_u32 = ip4->dst_address.as_u32;
+	}
+      k.protocol = ip4->protocol;
+      octets = clib_net_to_host_u16 (ip4->length);
+    }
+  if ((flags & FLOW_RECORD_L4) && udp &&
+      (k.protocol == IP_PROTOCOL_TCP || k.protocol == IP_PROTOCOL_UDP))
+    {
+      k.src_port = udp->src_port;
+      k.dst_port = udp->dst_port;
+    }
+
+  if (t)
+    {
+      t->rx_sw_if_index = k.rx_sw_if_index;
+      t->tx_sw_if_index = k.tx_sw_if_index;
+      clib_memcpy (t->src_mac, k.src_mac, 6);
+      clib_memcpy (t->dst_mac, k.dst_mac, 6);
+      t->ethertype = k.ethertype;
+      t->src_address.ip4.as_u32 = k.src_address.ip4.as_u32;
+      t->dst_address.ip4.as_u32 = k.dst_address.ip4.as_u32;
+      t->protocol = k.protocol;
+      t->src_port = k.src_port;
+      t->dst_port = k.dst_port;
+      t->which = k.which;
+    }
+
+  flowprobe_entry_t *e = 0;
+  f64 now = vlib_time_now (vm);
+  if (fm->active_timer > 0)
+    {
+      u32 poolindex = ~0;
+      bool collision = false;
+
+      e = flowprobe_lookup (my_cpu_number, &k, &poolindex, &collision);
+      if (collision)
+	{
+	  /* Flush data and clean up entry for reuse. */
+	  if (e->packetcount)
+	    flowprobe_export_entry (vm, e);
+	  e->key = k;
+	  vlib_node_increment_counter (vm, node->node_index,
+				       FLOWPROBE_ERROR_COLLISION, 1);
+	}
+      if (!e)			/* Create new entry */
+	{
+	  e = flowprobe_create (my_cpu_number, &k, &poolindex);
+	  e->last_exported = now;
+	}
+    }
+  else
+    {
+      e = &fm->stateless_entry[my_cpu_number];
+      e->key = k;
+    }
+
+  if (e)
+    {
+      /* Updating entry */
+      e->packetcount++;
+      e->octetcount += octets;
+      e->last_updated = now;
+
+      if (fm->active_timer == 0
+	  || (now > e->last_exported + fm->active_timer))
+	flowprobe_export_entry (vm, e);
+    }
+}
+
+static u16
+flowprobe_get_headersize (void)
+{
+  return sizeof (ip4_header_t) + sizeof (udp_header_t) +
+    sizeof (ipfix_message_header_t) + sizeof (ipfix_set_header_t);
+}
+
+static void
+flowprobe_export_send (vlib_main_t * vm, vlib_buffer_t * b0,
+		       flowprobe_variant_t which)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vlib_frame_t *f;
+  ip4_ipfix_template_packet_t *tp;
+  ipfix_set_header_t *s;
+  ipfix_message_header_t *h;
+  ip4_header_t *ip;
+  udp_header_t *udp;
+  flowprobe_record_t flags = fm->context[which].flags;
+  u32 my_cpu_number = vm->thread_index;
+
+  /* Fill in header */
+  flow_report_stream_t *stream;
+
+  /* Nothing to send */
+  if (fm->context[which].next_record_offset_per_worker[my_cpu_number] <=
+      flowprobe_get_headersize ())
+    return;
+
+  u32 i, index = vec_len (frm->streams);
+  for (i = 0; i < index; i++)
+    if (frm->streams[i].domain_id == 1)
+      {
+	index = i;
+	break;
+      }
+  if (i == vec_len (frm->streams))
+    {
+      vec_validate (frm->streams, index);
+      frm->streams[index].domain_id = 1;
+    }
+  stream = &frm->streams[index];
+
+  tp = vlib_buffer_get_current (b0);
+  ip = (ip4_header_t *) & tp->ip4;
+  udp = (udp_header_t *) (ip + 1);
+  h = (ipfix_message_header_t *) (udp + 1);
+  s = (ipfix_set_header_t *) (h + 1);
+
+  ip->ip_version_and_header_length = 0x45;
+  ip->ttl = 254;
+  ip->protocol = IP_PROTOCOL_UDP;
+  ip->flags_and_fragment_offset = 0;
+  ip->src_address.as_u32 = frm->src_address.as_u32;
+  ip->dst_address.as_u32 = frm->ipfix_collector.as_u32;
+  udp->src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix);
+  udp->dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix);
+  udp->checksum = 0;
+
+  /* FIXUP: message header export_time */
+  h->export_time = (u32)
+    (((f64) frm->unix_time_0) +
+     (vlib_time_now (frm->vlib_main) - frm->vlib_time_0));
+  h->export_time = clib_host_to_net_u32 (h->export_time);
+  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
+
+  /* FIXUP: message header sequence_number */
+  h->sequence_number = stream->sequence_number++;
+  h->sequence_number = clib_host_to_net_u32 (h->sequence_number);
+
+  s->set_id_length = ipfix_set_id_length (fm->template_reports[flags],
+					  b0->current_length -
+					  (sizeof (*ip) + sizeof (*udp) +
+					   sizeof (*h)));
+  h->version_length = version_length (b0->current_length -
+				      (sizeof (*ip) + sizeof (*udp)));
+
+  ip->length = clib_host_to_net_u16 (b0->current_length);
+
+  ip->checksum = ip4_header_checksum (ip);
+  udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
+
+  if (frm->udp_checksum)
+    {
+      /* RFC 7011 section 10.3.2. */
+      udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
+      if (udp->checksum == 0)
+	udp->checksum = 0xffff;
+    }
+
+  ASSERT (ip->checksum == ip4_header_checksum (ip));
+
+  /* Find or allocate a frame */
+  f = fm->context[which].frames_per_worker[my_cpu_number];
+  if (PREDICT_FALSE (f == 0))
+    {
+      u32 *to_next;
+      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
+      fm->context[which].frames_per_worker[my_cpu_number] = f;
+      u32 bi0 = vlib_get_buffer_index (vm, b0);
+
+      /* Enqueue the buffer */
+      to_next = vlib_frame_vector_args (f);
+      to_next[0] = bi0;
+      f->n_vectors = 1;
+    }
+
+  vlib_put_frame_to_node (vm, ip4_lookup_node.index, f);
+  vlib_node_increment_counter (vm, flowprobe_l2_node.index,
+			       FLOWPROBE_ERROR_EXPORTED_PACKETS, 1);
+
+  fm->context[which].frames_per_worker[my_cpu_number] = 0;
+  fm->context[which].buffers_per_worker[my_cpu_number] = 0;
+  fm->context[which].next_record_offset_per_worker[my_cpu_number] =
+    flowprobe_get_headersize ();
+}
+
+static vlib_buffer_t *
+flowprobe_get_buffer (vlib_main_t * vm, flowprobe_variant_t which)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vlib_buffer_t *b0;
+  u32 bi0;
+  vlib_buffer_free_list_t *fl;
+  u32 my_cpu_number = vm->thread_index;
+
+  /* Find or allocate a buffer */
+  b0 = fm->context[which].buffers_per_worker[my_cpu_number];
+
+  /* Need to allocate a buffer? */
+  if (PREDICT_FALSE (b0 == 0))
+    {
+      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
+	{
+	  vlib_node_increment_counter (vm, flowprobe_l2_node.index,
+				       FLOWPROBE_ERROR_BUFFER, 1);
+	  return 0;
+	}
+
+      /* Initialize the buffer */
+      b0 = fm->context[which].buffers_per_worker[my_cpu_number] =
+	vlib_get_buffer (vm, bi0);
+      fl =
+	vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+      vlib_buffer_init_for_free_list (b0, fl);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+
+      b0->current_data = 0;
+      b0->current_length = flowprobe_get_headersize ();
+      b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_FLOW_REPORT);
+      vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
+      vnet_buffer (b0)->sw_if_index[VLIB_TX] = frm->fib_index;
+      fm->context[which].next_record_offset_per_worker[my_cpu_number] =
+	b0->current_length;
+    }
+
+  return b0;
+}
+
+static void
+flowprobe_export_entry (vlib_main_t * vm, flowprobe_entry_t * e)
+{
+  u32 my_cpu_number = vm->thread_index;
+  flowprobe_main_t *fm = &flowprobe_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vlib_buffer_t *b0;
+  bool collect_ip4 = false, collect_ip6 = false;
+  flowprobe_variant_t which = e->key.which;
+  flowprobe_record_t flags = fm->context[which].flags;
+  u16 offset =
+    fm->context[which].next_record_offset_per_worker[my_cpu_number];
+
+  if (offset < flowprobe_get_headersize ())
+    offset = flowprobe_get_headersize ();
+
+  b0 = flowprobe_get_buffer (vm, which);
+  /* No available buffer, what to do... */
+  if (b0 == 0)
+    return;
+
+  if (flags & FLOW_RECORD_L3)
+    {
+      collect_ip4 = which == FLOW_VARIANT_L2_IP4 || which == FLOW_VARIANT_IP4;
+      collect_ip6 = which == FLOW_VARIANT_L2_IP6 || which == FLOW_VARIANT_IP6;
+    }
+
+  offset += flowprobe_common_add (b0, e, offset);
+
+  if (flags & FLOW_RECORD_L2)
+    offset += flowprobe_l2_add (b0, e, offset);
+  if (collect_ip6)
+    offset += flowprobe_l3_ip6_add (b0, e, offset);
+  if (collect_ip4)
+    offset += flowprobe_l3_ip4_add (b0, e, offset);
+  if (flags & FLOW_RECORD_L4)
+    offset += flowprobe_l4_add (b0, e, offset);
+
+  /* Reset per flow-export counters */
+  e->packetcount = 0;
+  e->octetcount = 0;
+  e->last_exported = vlib_time_now (vm);
+
+  b0->current_length = offset;
+
+  fm->context[which].next_record_offset_per_worker[my_cpu_number] = offset;
+  /* Time to flush the buffer? */
+  if (offset + fm->template_size[flags] > frm->path_mtu)
+    flowprobe_export_send (vm, b0, which);
+}
+
+uword
+flowprobe_node_fn (vlib_main_t * vm,
+		   vlib_node_runtime_t * node, vlib_frame_t * frame,
+		   flowprobe_variant_t which)
+{
+  u32 n_left_from, *from, *to_next;
+  flowprobe_next_t next_index;
+  flowprobe_main_t *fm = &flowprobe_main;
+  u64 now;
+
+  now = (u64) ((vlib_time_now (vm) - fm->vlib_time_0) * 1e9);
+  now += fm->nanosecond_time_0;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+	  u32 next0 = FLOWPROBE_NEXT_DROP;
+	  u32 next1 = FLOWPROBE_NEXT_DROP;
+	  u16 len0, len1;
+	  u32 bi0, bi1;
+	  vlib_buffer_t *b0, *b1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p2, *p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+	  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_TX],
+			     &next0, b0);
+	  vnet_feature_next (vnet_buffer (b1)->sw_if_index[VLIB_TX],
+			     &next1, b1);
+
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+	  ethernet_header_t *eh0 = vlib_buffer_get_current (b0);
+	  u16 ethertype0 = clib_net_to_host_u16 (eh0->type);
+
+	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
+	    add_to_flow_record_state (vm, node, fm, b0, now, len0,
+				      flowprobe_get_variant
+				      (which, fm->context[which].flags,
+				       ethertype0), 0);
+
+	  len1 = vlib_buffer_length_in_chain (vm, b1);
+	  ethernet_header_t *eh1 = vlib_buffer_get_current (b1);
+	  u16 ethertype1 = clib_net_to_host_u16 (eh1->type);
+
+	  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
+	    add_to_flow_record_state (vm, node, fm, b1, now, len1,
+				      flowprobe_get_variant
+				      (which, fm->context[which].flags,
+				       ethertype1), 0);
+
+	  /* verify speculative enqueues, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  u32 next0 = FLOWPROBE_NEXT_DROP;
+	  u16 len0;
+
+	  /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  vnet_feature_next (vnet_buffer (b0)->sw_if_index[VLIB_TX],
+			     &next0, b0);
+
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+	  ethernet_header_t *eh0 = vlib_buffer_get_current (b0);
+	  u16 ethertype0 = clib_net_to_host_u16 (eh0->type);
+
+	  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0))
+	    {
+	      flowprobe_trace_t *t = 0;
+	      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+				 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+		t = vlib_add_trace (vm, node, b0, sizeof (*t));
+
+	      add_to_flow_record_state (vm, node, fm, b0, now, len0,
+					flowprobe_get_variant
+					(which, fm->context[which].flags,
+					 ethertype0), t);
+	    }
+
+	  /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  return frame->n_vectors;
+}
+
+static uword
+flowprobe_ip4_node_fn (vlib_main_t * vm,
+		       vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  return flowprobe_node_fn (vm, node, frame, FLOW_VARIANT_IP4);
+}
+
+static uword
+flowprobe_ip6_node_fn (vlib_main_t * vm,
+		       vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  return flowprobe_node_fn (vm, node, frame, FLOW_VARIANT_IP6);
+}
+
+static uword
+flowprobe_l2_node_fn (vlib_main_t * vm,
+		      vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  return flowprobe_node_fn (vm, node, frame, FLOW_VARIANT_L2);
+}
+
+static inline void
+flush_record (flowprobe_variant_t which)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_buffer_t *b = flowprobe_get_buffer (vm, which);
+  if (b)
+    flowprobe_export_send (vm, b, which);
+}
+
+void
+flowprobe_flush_callback_ip4 (void)
+{
+  flush_record (FLOW_VARIANT_IP4);
+}
+
+void
+flowprobe_flush_callback_ip6 (void)
+{
+  flush_record (FLOW_VARIANT_IP6);
+}
+
+void
+flowprobe_flush_callback_l2 (void)
+{
+  flush_record (FLOW_VARIANT_L2);
+  flush_record (FLOW_VARIANT_L2_IP4);
+  flush_record (FLOW_VARIANT_L2_IP6);
+}
+
+
+static void
+flowprobe_delete_by_index (u32 my_cpu_number, u32 poolindex)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  flowprobe_entry_t *e;
+  u32 h;
+
+  e = pool_elt_at_index (fm->pool_per_worker[my_cpu_number], poolindex);
+
+  /* Get my index */
+  h = flowprobe_hash (&e->key);
+
+  /* Reset hash */
+  fm->hash_per_worker[my_cpu_number][h] = ~0;
+
+  pool_put_index (fm->pool_per_worker[my_cpu_number], poolindex);
+}
+
+
+/* Per worker process processing the active/passive expired entries */
+static uword
+flowprobe_walker_process (vlib_main_t * vm,
+			  vlib_node_runtime_t * rt, vlib_frame_t * f)
+{
+  flowprobe_main_t *fm = &flowprobe_main;
+  flow_report_main_t *frm = &flow_report_main;
+  flowprobe_entry_t *e;
+
+  /*
+   * $$$$ Remove this check from here and track FRM status and disable
+   * this process if required.
+   */
+  if (frm->ipfix_collector.as_u32 == 0 || frm->src_address.as_u32 == 0)
+    {
+      fm->disabled = true;
+      return 0;
+    }
+  fm->disabled = false;
+
+  u32 cpu_index = os_get_thread_index ();
+  u32 *to_be_removed = 0, *i;
+
+  /*
+   * Tick the timer when required and process the vector of expired
+   * timers
+   */
+  f64 start_time = vlib_time_now (vm);
+  u32 count = 0;
+
+  tw_timer_expire_timers_2t_1w_2048sl (fm->timers_per_worker[cpu_index],
+				       start_time);
+
+  vec_foreach (i, fm->expired_passive_per_worker[cpu_index])
+  {
+    u32 exported = 0;
+    f64 now = vlib_time_now (vm);
+    if (now > start_time + 100e-6
+	|| exported > FLOW_MAXIMUM_EXPORT_ENTRIES - 1)
+      break;
+
+    if (pool_is_free_index (fm->pool_per_worker[cpu_index], *i))
+      {
+	clib_warning ("Element is %d is freed already\n", *i);
+	continue;
+      }
+    else
+      e = pool_elt_at_index (fm->pool_per_worker[cpu_index], *i);
+
+    /* Check last update timestamp. If it is longer than passive time nuke
+     * entry. Otherwise restart timer with what's left
+     * Premature passive timer by more than 10%
+     */
+    if ((now - e->last_updated) < (fm->passive_timer * 0.9))
+      {
+	f64 delta = fm->passive_timer - (now - e->last_updated);
+	e->passive_timer_handle = tw_timer_start_2t_1w_2048sl
+	  (fm->timers_per_worker[cpu_index], *i, 0, delta);
+      }
+    else			/* Nuke entry */
+      {
+	vec_add1 (to_be_removed, *i);
+      }
+    /* If anything to report send it to the exporter */
+    if (e->packetcount && now > e->last_exported + fm->active_timer)
+      {
+	exported++;
+	flowprobe_export_entry (vm, e);
+      }
+    count++;
+  }
+  if (count)
+    vec_delete (fm->expired_passive_per_worker[cpu_index], count, 0);
+
+  vec_foreach (i, to_be_removed) flowprobe_delete_by_index (cpu_index, *i);
+  vec_free (to_be_removed);
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (flowprobe_ip4_node) = {
+  .function = flowprobe_ip4_node_fn,
+  .name = "flowprobe-ip4",
+  .vector_size = sizeof (u32),
+  .format_trace = format_flowprobe_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(flowprobe_error_strings),
+  .error_strings = flowprobe_error_strings,
+  .n_next_nodes = FLOWPROBE_N_NEXT,
+  .next_nodes = FLOWPROBE_NEXT_NODES,
+};
+VLIB_REGISTER_NODE (flowprobe_ip6_node) = {
+  .function = flowprobe_ip6_node_fn,
+  .name = "flowprobe-ip6",
+  .vector_size = sizeof (u32),
+  .format_trace = format_flowprobe_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(flowprobe_error_strings),
+  .error_strings = flowprobe_error_strings,
+  .n_next_nodes = FLOWPROBE_N_NEXT,
+  .next_nodes = FLOWPROBE_NEXT_NODES,
+};
+VLIB_REGISTER_NODE (flowprobe_l2_node) = {
+  .function = flowprobe_l2_node_fn,
+  .name = "flowprobe-l2",
+  .vector_size = sizeof (u32),
+  .format_trace = format_flowprobe_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(flowprobe_error_strings),
+  .error_strings = flowprobe_error_strings,
+  .n_next_nodes = FLOWPROBE_N_NEXT,
+  .next_nodes = FLOWPROBE_NEXT_NODES,
+};
+VLIB_REGISTER_NODE (flowprobe_walker_node) = {
+  .function = flowprobe_walker_process,
+  .name = "flowprobe-walker",
+  .type = VLIB_NODE_TYPE_INPUT,
+  .state = VLIB_NODE_STATE_INTERRUPT,
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/ioam/analyse/ioam_summary_export.c b/src/plugins/ioam/analyse/ioam_summary_export.c
index 9a2667f6..af2d39ab 100644
--- a/src/plugins/ioam/analyse/ioam_summary_export.c
+++ b/src/plugins/ioam/analyse/ioam_summary_export.c
@@ -398,6 +398,7 @@ ioam_flow_create (u8 del)
   int rv;
   u32 domain_id = 0;
   flow_report_main_t *frm = &flow_report_main;
+  u16 template_id;
 
   memset (&args, 0, sizeof (args));
   args.rewrite_callback = ioam_template_rewrite;
@@ -405,7 +406,7 @@ ioam_flow_create (u8 del)
   del ? (args.is_add = 0) : (args.is_add = 1);
   args.domain_id = domain_id;
 
-  rv = vnet_flow_report_add_del (frm, &args);
+  rv = vnet_flow_report_add_del (frm, &args, &template_id);
 
   switch (rv)
     {
diff --git a/src/plugins/ioam/udp-ping/udp_ping_export.c b/src/plugins/ioam/udp-ping/udp_ping_export.c
index ce64b606..91cbb4bd 100644
--- a/src/plugins/ioam/udp-ping/udp_ping_export.c
+++ b/src/plugins/ioam/udp-ping/udp_ping_export.c
@@ -226,6 +226,7 @@ udp_ping_flow_create (u8 del)
   int rv;
   u32 domain_id = 0;
   flow_report_main_t *frm = &flow_report_main;
+  u16 template_id;
 
   memset (&args, 0, sizeof (args));
   args.rewrite_callback = udp_ping_template_rewrite;
@@ -234,7 +235,7 @@ udp_ping_flow_create (u8 del)
   args.domain_id = domain_id;
   args.src_port = UDP_DST_PORT_ipfix;
 
-  rv = vnet_flow_report_add_del (frm, &args);
+  rv = vnet_flow_report_add_del (frm, &args, &template_id);
 
   switch (rv)
     {
diff --git a/src/plugins/snat/snat_ipfix_logging.c b/src/plugins/snat/snat_ipfix_logging.c
index 1a111863..c68dc540 100644
--- a/src/plugins/snat/snat_ipfix_logging.c
+++ b/src/plugins/snat/snat_ipfix_logging.c
@@ -784,7 +784,7 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
       a.rewrite_callback = snat_template_rewrite_max_entries_per_usr;
       a.flow_data_callback = snat_data_callback_max_entries_per_usr;
 
-      rv = vnet_flow_report_add_del (frm, &a);
+      rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
         {
           clib_warning ("vnet_flow_report_add_del returned %d", rv);
@@ -796,7 +796,7 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
       a.rewrite_callback = snat_template_rewrite_nat44_session;
       a.flow_data_callback = snat_data_callback_nat44_session;
 
-      rv = vnet_flow_report_add_del (frm, &a);
+      rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
         {
           clib_warning ("vnet_flow_report_add_del returned %d", rv);
@@ -806,7 +806,7 @@ snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
       a.rewrite_callback = snat_template_rewrite_addr_exhausted;
       a.flow_data_callback = snat_data_callback_addr_exhausted;
 
-      rv = vnet_flow_report_add_del (frm, &a);
+      rv = vnet_flow_report_add_del (frm, &a, NULL);
       if (rv)
         {
           clib_warning ("vnet_flow_report_add_del returned %d", rv);
diff --git a/src/vnet/flow/flow_api.c b/src/vnet/flow/flow_api.c
index b975dda1..52a608ca 100644
--- a/src/vnet/flow/flow_api.c
+++ b/src/vnet/flow/flow_api.c
@@ -296,7 +296,7 @@ static void
   args.domain_id = fcm->domain_id;
   args.src_port = fcm->src_port;
 
-  rv = vnet_flow_report_add_del (frm, &args);
+  rv = vnet_flow_report_add_del (frm, &args, NULL);
 
   /* If deleting, or add failed */
   if (is_add == 0 || (rv && is_add))
diff --git a/src/vnet/flow/flow_report.c b/src/vnet/flow/flow_report.c
index c78a78a9..ccc84235 100644
--- a/src/vnet/flow/flow_report.c
+++ b/src/vnet/flow/flow_report.c
@@ -87,7 +87,6 @@ int send_template_packet (flow_report_main_t *frm,
       if (frm->ipfix_collector.as_u32 == 0 
           || frm->src_address.as_u32 == 0)
         {
-          clib_warning ("no collector: disabling flow collector process");
           vlib_node_set_state (frm->vlib_main, flow_report_process_node.index,
                                VLIB_NODE_STATE_DISABLED);
           return -1;
@@ -238,7 +237,8 @@ VLIB_REGISTER_NODE (flow_report_process_node) = {
 };
 
 int vnet_flow_report_add_del (flow_report_main_t *frm, 
-                              vnet_flow_report_add_del_args_t *a)
+                              vnet_flow_report_add_del_args_t *a,
+			      u16 *template_id)
 {
   int i;
   int found_index = ~0;
@@ -260,6 +260,8 @@ int vnet_flow_report_add_del (flow_report_main_t *frm,
           && fr->flow_data_callback == a->flow_data_callback)
         {
           found_index = i;
+          if (template_id)
+            *template_id = fr->template_id;
           break;
         }
     }
@@ -304,7 +306,10 @@ int vnet_flow_report_add_del (flow_report_main_t *frm,
   fr->opaque = a->opaque;
   fr->rewrite_callback = a->rewrite_callback;
   fr->flow_data_callback = a->flow_data_callback;
-  
+
+  if (template_id)
+    *template_id = fr->template_id;
+
   return 0;
 }
 
@@ -415,10 +420,7 @@ set_ipfix_exporter_command_fn (vlib_main_t * vm,
       break;
   }
   
-  if (collector.as_u32 == 0)
-    return clib_error_return (0, "collector address required");
-
-  if (src.as_u32 == 0)
+  if (collector.as_u32 != 0 && src.as_u32 == 0)
     return clib_error_return (0, "src address required");
 
   if (path_mtu > 1450 /* vpp does not support fragmentation */)
@@ -441,7 +443,8 @@ set_ipfix_exporter_command_fn (vlib_main_t * vm,
   frm->template_interval = template_interval;
   frm->udp_checksum = udp_checksum;
   
-  vlib_cli_output (vm, "Collector %U, src address %U, "
+  if (collector.as_u32)
+    vlib_cli_output (vm, "Collector %U, src address %U, "
 		           "fib index %d, path MTU %u, "
 		           "template resend interval %us, "
 		           "udp checksum %s",
@@ -449,6 +452,8 @@ set_ipfix_exporter_command_fn (vlib_main_t * vm,
                    format_ip4_address, &frm->src_address,
                    fib_index, path_mtu, template_interval,
                    udp_checksum ? "enabled" : "disabled");
+  else
+    vlib_cli_output (vm, "IPFIX Collector is disabled");
 
   /* Turn on the flow reporting process */
   vlib_process_signal_event (vm, flow_report_process_node.index,
diff --git a/src/vnet/flow/flow_report.h b/src/vnet/flow/flow_report.h
index e8ed3818..01859ce5 100644
--- a/src/vnet/flow/flow_report.h
+++ b/src/vnet/flow/flow_report.h
@@ -130,7 +130,8 @@ typedef struct {
 } vnet_flow_report_add_del_args_t;  
 
 int vnet_flow_report_add_del (flow_report_main_t *frm, 
-                              vnet_flow_report_add_del_args_t *a);
+                              vnet_flow_report_add_del_args_t *a,
+			      u16 *template_id);
 
 clib_error_t * flow_report_add_del_error_to_clib_error (int error);
 
diff --git a/src/vnet/flow/flow_report_classify.c b/src/vnet/flow/flow_report_classify.c
index 27f03ddc..d4c30492 100644
--- a/src/vnet/flow/flow_report_classify.c
+++ b/src/vnet/flow/flow_report_classify.c
@@ -458,7 +458,7 @@ ipfix_classify_table_add_del_command_fn (vlib_main_t * vm,
   args.domain_id = fcm->domain_id;
   args.src_port = fcm->src_port;
 
-  rv = vnet_flow_report_add_del (frm, &args);
+  rv = vnet_flow_report_add_del (frm, &args, NULL);
 
   error = flow_report_add_del_error_to_clib_error(rv);
 
diff --git a/src/vnet/ip/ip6_packet.h b/src/vnet/ip/ip6_packet.h
index cdd7eed5..c0c745e2 100644
--- a/src/vnet/ip/ip6_packet.h
+++ b/src/vnet/ip/ip6_packet.h
@@ -341,6 +341,12 @@ typedef struct
   ip6_address_t src_address, dst_address;
 } ip6_header_t;
 
+always_inline u8
+ip6_traffic_class (ip6_header_t * i)
+{
+  return (i->ip_version_traffic_class_and_flow_label & 0x0FF00000) >> 20;
+}
+
 always_inline void *
 ip6_next_header (ip6_header_t * i)
 {
diff --git a/src/vpp-api/java/jvpp/gen/jvpp_gen.py b/src/vpp-api/java/jvpp/gen/jvpp_gen.py
index 7932741d..6648a4f7 100755
--- a/src/vpp-api/java/jvpp/gen/jvpp_gen.py
+++ b/src/vpp-api/java/jvpp/gen/jvpp_gen.py
@@ -30,7 +30,7 @@ from jvppgen import jvpp_impl_gen
 from jvppgen import jvpp_c_gen
 from jvppgen import util
 
-blacklist = [ "memclnt.api", "flowperpkt.api" ]
+blacklist = [ "memclnt.api", "flowprobe.api" ]
 
 # Invocation:
 # ~/Projects/vpp/vpp-api/jvpp/gen$ mkdir -p java/io/fd/vpp/jvpp && cd java/io/fd/vpp/jvpp
diff --git a/test/test_flowperpkt.py b/test/test_flowperpkt.py
deleted file mode 100644
index b13d0c63..00000000
--- a/test/test_flowperpkt.py
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env python
-
-import unittest
-
-from framework import VppTestCase, VppTestRunner
-
-from scapy.packet import Raw
-from scapy.layers.l2 import Ether
-from scapy.layers.inet import IP, UDP
-
-
-class TestFlowperpkt(VppTestCase):
-    """ Flow-per-packet plugin: test both L2 and IP4 reporting """
-
-    def setUp(self):
-        """
-        Set up
-
-        **Config:**
-            - create three PG interfaces
-        """
-        super(TestFlowperpkt, self).setUp()
-
-        self.create_pg_interfaces(range(3))
-
-        self.pg_if_packet_sizes = [150]
-
-        self.interfaces = list(self.pg_interfaces)
-
-        for intf in self.interfaces:
-            intf.admin_up()
-            intf.config_ip4()
-            intf.resolve_arp()
-
-    def create_stream(self, src_if, dst_if, packet_sizes):
-        """Create a packet stream to tickle the plugin
-
-        :param VppInterface src_if: Source interface for packet stream
-        :param VppInterface src_if: Dst interface for packet stream
-        :param list packet_sizes: Sizes to test
-        """
-        pkts = []
-        for size in packet_sizes:
-            info = self.create_packet_info(src_if, dst_if)
-            payload = self.info_to_payload(info)
-            p = (Ether(src=src_if.remote_mac, dst=src_if.local_mac) /
-                 IP(src=src_if.remote_ip4, dst=dst_if.remote_ip4) /
-                 UDP(sport=1234, dport=4321) /
-                 Raw(payload))
-            info.data = p.copy()
-            self.extend_packet(p, size)
-            pkts.append(p)
-        return pkts
-
-    @staticmethod
-    def compare_with_mask(payload, masked_expected_data):
-        if len(payload) * 2 != len(masked_expected_data):
-            return False
-
-        # iterate over pairs: raw byte from payload and ASCII code for that
-        # byte from masked payload (or XX if masked)
-        for i in range(len(payload)):
-            p = payload[i]
-            m = masked_expected_data[2 * i:2 * i + 2]
-            if m != "XX":
-                if "%02x" % ord(p) != m:
-                    return False
-        return True
-
-    def verify_ipfix(self, collector_if):
-        """Check the ipfix capture"""
-        found_data_packet = False
-        found_template_packet = False
-        found_l2_data_packet = False
-        found_l2_template_packet = False
-
-        # Scapy, of course, understands ipfix not at all...
-        # These data vetted by manual inspection in wireshark
-        # X'ed out fields are timestamps, which will absolutely
-        # fail to compare.
-
-        data_udp_string = "1283128300370000000a002fXXXXXXXX000000000000000101"\
-            "00001f0000000100000002ac100102ac10020200XXXXXXXXXXXXXXXX0092"
-
-        template_udp_string = "12831283003c0000000a0034XXXXXXXX00000002000000"\
-            "010002002401000007000a0004000e000400080004000c000400050001009c00"\
-            "0801380002"
-
-        l2_data_udp_string = "12831283003c0000000a0034XXXXXXXX000000010000000"\
-            "1010100240000000100000002%s02020000ff020008XXXXXXXXXXX"\
-            "XXXXX0092" % self.pg1.local_mac.translate(None, ":")
-
-        l2_template_udp_string = "12831283003c0000000a0034XXXXXXXX00000002000"\
-            "000010002002401010007000a0004000e0004003800060050000601000002009"\
-            "c000801380002"
-
-        self.logger.info("Look for ipfix packets on %s sw_if_index %d "
-                         % (collector_if.name, collector_if.sw_if_index))
-        # expecting 4 packets on collector interface based on traffic on other
-        # interfaces
-        capture = collector_if.get_capture(4)
-
-        for p in capture:
-            ip = p[IP]
-            udp = p[UDP]
-            self.logger.info("src %s dst %s" % (ip.src, ip.dst))
-            self.logger.info(" udp src_port %s dst_port %s"
-                             % (udp.sport, udp.dport))
-
-            payload = str(udp)
-
-            if self.compare_with_mask(payload, data_udp_string):
-                self.logger.info("found ip4 data packet")
-                found_data_packet = True
-            elif self.compare_with_mask(payload, template_udp_string):
-                self.logger.info("found ip4 template packet")
-                found_template_packet = True
-            elif self.compare_with_mask(payload, l2_data_udp_string):
-                self.logger.info("found l2 data packet")
-                found_l2_data_packet = True
-            elif self.compare_with_mask(payload, l2_template_udp_string):
-                self.logger.info("found l2 template packet")
-                found_l2_template_packet = True
-            else:
-                unmasked_payload = "".join(["%02x" % ord(c) for c in payload])
-                self.logger.error("unknown pkt '%s'" % unmasked_payload)
-
-        self.assertTrue(found_data_packet, "Data packet not found")
-        self.assertTrue(found_template_packet, "Template packet not found")
-        self.assertTrue(found_l2_data_packet, "L2 data packet not found")
-        self.assertTrue(found_l2_template_packet,
-                        "L2 template packet not found")
-
-    def test_L3_fpp(self):
-        """ Flow per packet L3 test """
-
-        # Configure an ipfix report on the [nonexistent] collector
-        # 172.16.3.2, as if it was connected to the pg2 interface
-        # Install a FIB entry, so the exporter's work won't turn into
-        # an ARP request
-
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg2.configure_ipv4_neighbors()
-        self.vapi.set_ipfix_exporter(collector_address=self.pg2.remote_ip4n,
-                                     src_address=self.pg2.local_ip4n,
-                                     path_mtu=1450,
-                                     template_interval=1)
-
-        # Export flow records for all pkts transmitted on pg1
-        self.vapi.cli("flowperpkt feature add-del pg1")
-        self.vapi.cli("flowperpkt feature add-del pg1 l2")
-
-        # Arrange to minimally trace generated ipfix packets
-        self.vapi.cli("trace add flowperpkt-ipv4 10")
-        self.vapi.cli("trace add flowperpkt-l2 10")
-
-        # Create a stream from pg0 -> pg1, which causes
-        # an ipfix packet to be transmitted on pg2
-
-        pkts = self.create_stream(self.pg0, self.pg1,
-                                  self.pg_if_packet_sizes)
-        self.pg0.add_stream(pkts)
-        self.pg_start()
-
-        # Flush the ipfix collector, so we don't need any
-        # asinine time.sleep(5) action
-        self.vapi.cli("ipfix flush")
-
-        # Make sure the 4 pkts we expect actually showed up
-        self.verify_ipfix(self.pg2)
-
-if __name__ == '__main__':
-    unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py
new file mode 100644
index 00000000..560b44cc
--- /dev/null
+++ b/test/test_flowprobe.py
@@ -0,0 +1,967 @@
+#!/usr/bin/env python
+import random
+import socket
+import unittest
+import time
+
+from scapy.packet import Raw
+from scapy.layers.l2 import Ether
+from scapy.layers.inet import IP, UDP
+from scapy.layers.inet6 import IPv6
+
+from framework import VppTestCase, VppTestRunner
+from vpp_object import VppObject
+from vpp_pg_interface import CaptureTimeoutError
+from util import ppp
+from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
+
+
+class VppCFLOW(VppObject):
+    """CFLOW object for IPFIX exporter and Flowprobe feature"""
+
+    def __init__(self, test, intf='pg2', active=0, passive=0, timeout=300,
+                 mtu=512, datapath='l2', layer='l2 l3 l4'):
+        self._test = test
+        self._intf = intf
+        self._active = active
+        if passive == 0 or passive < active:
+            self._passive = active+5
+        else:
+            self._passive = passive
+        self._datapath = datapath           # l2 ip4 ip6
+        self._collect = layer               # l2 l3 l4
+        self._timeout = timeout
+        self._mtu = mtu
+        self._configured = False
+
+    def add_vpp_config(self):
+        self.enable_exporter()
+        self._test.vapi.ppcli("flowprobe params record %s active %s "
+                              "passive %s" % (self._collect, self._active,
+                                              self._passive))
+        self.enable_flowprobe_feature()
+        self._test.vapi.cli("ipfix flush")
+        self._configured = True
+
+    def remove_vpp_config(self):
+        self.disable_exporter()
+        self.disable_flowprobe_feature()
+        self._test.vapi.cli("ipfix flush")
+        self._configured = False
+
+    def enable_exporter(self):
+        self._test.vapi.set_ipfix_exporter(
+            collector_address=self._test.pg0.remote_ip4n,
+            src_address=self._test.pg0.local_ip4n,
+            path_mtu=self._mtu,
+            template_interval=self._timeout)
+
+    def enable_flowprobe_feature(self):
+        self._test.vapi.ppcli("flowprobe feature add-del %s %s" %
+                              (self._intf, self._datapath))
+
+    def disable_exporter(self):
+        self._test.vapi.cli("set ipfix exporter collector 0.0.0.0")
+
+    def disable_flowprobe_feature(self):
+        self._test.vapi.cli("flowprobe feature add-del %s %s disable" %
+                            (self._intf, self._datapath))
+
+    def object_id(self):
+        return "ipfix-collector-%s" % (self._src, self.dst)
+
+    def query_vpp_config(self):
+        return self._configured
+
+    def verify_templates(self, decoder=None, timeout=1, count=3):
+        templates = []
+        p = self._test.wait_for_cflow_packet(self._test.collector, 2, timeout)
+        self._test.assertTrue(p.haslayer(IPFIX))
+        if decoder is not None and p.haslayer(Template):
+            templates.append(p[Template].templateID)
+            decoder.add_template(p.getlayer(Template))
+        if count > 1:
+            p = self._test.wait_for_cflow_packet(self._test.collector, 2)
+            self._test.assertTrue(p.haslayer(IPFIX))
+            if decoder is not None and p.haslayer(Template):
+                templates.append(p[Template].templateID)
+                decoder.add_template(p.getlayer(Template))
+        if count > 2:
+            p = self._test.wait_for_cflow_packet(self._test.collector, 2)
+            self._test.assertTrue(p.haslayer(IPFIX))
+            if decoder is not None and p.haslayer(Template):
+                templates.append(p[Template].templateID)
+                decoder.add_template(p.getlayer(Template))
+        return templates
+
+
+class MethodHolder(VppTestCase):
+    """ Flow-per-packet plugin: test L2, IP4, IP6 reporting """
+
+    # Test variables
+    debug_print = False
+    max_number_of_packets = 16
+    pkts = []
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Perform standard class setup (defined by class method setUpClass in
+        class VppTestCase) before running the test case, set test case related
+        variables and configure VPP.
+        """
+        super(MethodHolder, cls).setUpClass()
+        try:
+            # Create pg interfaces
+            cls.create_pg_interfaces(range(7))
+
+            # Packet sizes
+            cls.pg_if_packet_sizes = [64, 512, 1518, 9018]
+
+            # Create BD with MAC learning and unknown unicast flooding disabled
+            # and put interfaces to this BD
+            cls.vapi.bridge_domain_add_del(bd_id=1, uu_flood=1, learn=1)
+            cls.vapi.sw_interface_set_l2_bridge(cls.pg1._sw_if_index, bd_id=1)
+            cls.vapi.sw_interface_set_l2_bridge(cls.pg2._sw_if_index, bd_id=1)
+
+            # Set up all interfaces
+            for i in cls.pg_interfaces:
+                i.admin_up()
+
+            cls.pg0.config_ip4()
+            cls.pg0.configure_ipv4_neighbors()
+            cls.collector = cls.pg0
+
+            cls.pg1.config_ip4()
+            cls.pg1.resolve_arp()
+            cls.pg2.config_ip4()
+            cls.pg2.resolve_arp()
+            cls.pg3.config_ip4()
+            cls.pg3.resolve_arp()
+            cls.pg4.config_ip4()
+            cls.pg4.resolve_arp()
+
+            cls.pg5.config_ip6()
+            cls.pg5.resolve_ndp()
+            cls.pg5.disable_ipv6_ra()
+            cls.pg6.config_ip6()
+            cls.pg6.resolve_ndp()
+            cls.pg6.disable_ipv6_ra()
+        except Exception:
+            super(MethodHolder, cls).tearDownClass()
+            raise
+
+    def create_stream(self, src_if=None, dst_if=None, packets=None,
+                      size=None, ip_ver='v4'):
+        """Create a packet stream to tickle the plugin
+
+        :param VppInterface src_if: Source interface for packet stream
+        :param VppInterface src_if: Dst interface for packet stream
+        """
+        if src_if is None:
+            src_if = self.pg1
+        if dst_if is None:
+            dst_if = self.pg2
+        self.pkts = []
+        if packets is None:
+            packets = random.randint(1, self.max_number_of_packets)
+        pkt_size = size
+        for p in range(0, packets):
+            if size is None:
+                pkt_size = random.choice(self.pg_if_packet_sizes)
+            info = self.create_packet_info(src_if, dst_if)
+            payload = self.info_to_payload(info)
+            p = Ether(src=src_if.remote_mac, dst=src_if.local_mac)
+            if ip_ver == 'v4':
+                p /= IP(src=src_if.remote_ip4, dst=dst_if.remote_ip4)
+            else:
+                p /= IPv6(src=src_if.remote_ip6, dst=dst_if.remote_ip6)
+            p /= (UDP(sport=1234, dport=4321) / Raw(payload))
+            info.data = p.copy()
+            self.extend_packet(p, pkt_size)
+            self.pkts.append(p)
+
+    def verify_cflow_data(self, decoder, capture, cflow):
+        octets = 0
+        packets = 0
+        for p in capture:
+            octets += p[IP].len
+            packets += 1
+        if cflow.haslayer(Data):
+            data = decoder.decode_data_set(cflow.getlayer(Set))
+            for record in data:
+                self.assertEqual(int(record[1].encode('hex'), 16), octets)
+                self.assertEqual(int(record[2].encode('hex'), 16), packets)
+
+    def verify_cflow_data_detail(self, decoder, capture, cflow,
+                                 data_set={1: 'octets', 2: 'packets'},
+                                 ip_ver='v4'):
+        if self.debug_print:
+            print capture[0].show()
+        if cflow.haslayer(Data):
+            data = decoder.decode_data_set(cflow.getlayer(Set))
+            if self.debug_print:
+                print data
+            if ip_ver == 'v4':
+                ip_layer = capture[0][IP]
+            else:
+                ip_layer = capture[0][IPv6]
+            if data_set is not None:
+                for record in data:
+                    # skip flow if in/out gress interface is 0
+                    if int(record[10].encode('hex'), 16) == 0:
+                        continue
+                    if int(record[14].encode('hex'), 16) == 0:
+                        continue
+
+                    for field in data_set:
+                        if field not in record.keys():
+                            continue
+                        value = data_set[field]
+                        if value == 'octets':
+                            value = ip_layer.len
+                            if ip_ver == 'v6':
+                                value += 40        # ??? is this correct
+                        elif value == 'packets':
+                            value = 1
+                        elif value == 'src_ip':
+                            if ip_ver == 'v4':
+                                ip = socket.inet_pton(socket.AF_INET,
+                                                      ip_layer.src)
+                            else:
+                                ip = socket.inet_pton(socket.AF_INET6,
+                                                      ip_layer.src)
+                            value = int(ip.encode('hex'), 16)
+                        elif value == 'dst_ip':
+                            if ip_ver == 'v4':
+                                ip = socket.inet_pton(socket.AF_INET,
+                                                      ip_layer.dst)
+                            else:
+                                ip = socket.inet_pton(socket.AF_INET6,
+                                                      ip_layer.dst)
+                            value = int(ip.encode('hex'), 16)
+                        elif value == 'sport':
+                            value = int(capture[0][UDP].sport)
+                        elif value == 'dport':
+                            value = int(capture[0][UDP].dport)
+                        self.assertEqual(int(record[field].encode('hex'), 16),
+                                         value)
+
+    def verify_cflow_data_notimer(self, decoder, capture, cflows):
+        idx = 0
+        for cflow in cflows:
+            if cflow.haslayer(Data):
+                data = decoder.decode_data_set(cflow.getlayer(Set))
+            else:
+                raise Exception("No CFLOW data")
+
+            for rec in data:
+                p = capture[idx]
+                idx += 1
+                self.assertEqual(p[IP].len, int(rec[1].encode('hex'), 16))
+                self.assertEqual(1, int(rec[2].encode('hex'), 16))
+        self.assertEqual(len(capture), idx)
+
+    def wait_for_cflow_packet(self, collector_intf, set_id=2, timeout=1,
+                              expected=True):
+        """ wait for CFLOW packet and verify its correctness
+
+        :param timeout: how long to wait
+
+        :returns: tuple (packet, time spent waiting for packet)
+        """
+        self.logger.info("IPFIX: Waiting for CFLOW packet")
+        deadline = time.time() + timeout
+        counter = 0
+        # self.logger.debug(self.vapi.ppcli("show flow table"))
+        while True:
+            counter += 1
+            # sanity check
+            self.assert_in_range(counter, 0, 100, "number of packets ignored")
+            time_left = deadline - time.time()
+            try:
+                if time_left < 0 and expected:
+                    # self.logger.debug(self.vapi.ppcli("show flow table"))
+                    raise CaptureTimeoutError(
+                          "Packet did not arrive within timeout")
+                p = collector_intf.wait_for_packet(timeout=time_left)
+            except CaptureTimeoutError:
+                if expected:
+                    # self.logger.debug(self.vapi.ppcli("show flow table"))
+                    raise CaptureTimeoutError(
+                          "Packet did not arrive within timeout")
+                else:
+                    return
+            if not expected:
+                raise CaptureTimeoutError("Packet arrived even not expected")
+            self.assertEqual(p[Set].setID, set_id)
+            # self.logger.debug(self.vapi.ppcli("show flow table"))
+            self.logger.debug(ppp("IPFIX: Got packet:", p))
+            break
+        return p
+
+    def send_packets(self, src_if=None, dst_if=None):
+        self.sleep(3)
+        if src_if is None:
+            src_if = self.pg1
+        if dst_if is None:
+            dst_if = self.pg2
+        self.pg_enable_capture([dst_if])
+        src_if.add_stream(self.pkts)
+        self.pg_start()
+        return dst_if.get_capture(len(self.pkts))
+
+
+class TestFFP_Timers(MethodHolder):
+    """Template verification, timer tests"""
+
+    def test_0001(self):
+        """ receive template data packets"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+
+        ipfix = VppCFLOW(test=self, active=10)
+        ipfix.add_vpp_config()
+
+        # template packet should arrive immediately
+        ipfix.verify_templates(timeout=3)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+    def test_0002(self):
+        """ timer=10s, less than template timeout"""
+        self.logger.info("FFP_TEST_START_0002")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=20)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream()
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[1], 39)
+        self.verify_cflow_data(ipfix_decoder, capture, cflow)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0002")
+
+    def test_0003(self):
+        """ timer=30s, greater than template timeout"""
+        self.logger.info("FFP_TEST_START_0003")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, timeout=20, active=25)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        ipfix.verify_templates(timeout=3)
+
+        self.create_stream()
+        capture = self.send_packets()
+
+        # next set of template packet should arrive after 20 seconds
+        # template packet should arrive within 20 s
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=25)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[1], 55)
+        self.verify_cflow_data(ipfix_decoder, capture, cflow)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0003")
+
+    def test_0004(self):
+        """ sent packet after first cflow packet arrived"""
+        self.logger.info("FFP_TEST_START_0004")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=20)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream()
+        self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 39)
+
+        self.pg_enable_capture([self.pg2])
+
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.verify_cflow_data(ipfix_decoder, capture, cflow)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0004")
+
+
+class TestFFP_DatapathL2(MethodHolder):
+    """collect information on Ethernet datapath"""
+
+    def test_0000(self):
+        """ verify template on L2 datapath"""
+        self.logger.info("FFP_TEST_START_0000")
+        self.pg_enable_capture(self.pg_interfaces)
+
+        ipfix = VppCFLOW(test=self, active=10)
+        ipfix.add_vpp_config()
+
+        # template packet should arrive immediately
+        ipfix.verify_templates(timeout=3, count=3)
+        self.collector.get_capture(3)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0000")
+
+    def test_0001(self):
+        """ L2 data on L2 datapath"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=10, layer='l2')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(packets=1)
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 256: 8})
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+    def test_0002(self):
+        """ L3 data on L2 datapath"""
+        self.logger.info("FFP_TEST_START_0002")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=10, layer='l3')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=2)
+
+        self.create_stream(packets=1)
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 4: 17,
+                                       8: 'src_ip', 12: 'dst_ip'})
+
+        # expected one template and one cflow packet
+        self.collector.get_capture(3)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0002")
+
+    def test_0003(self):
+        """ L4 data on L2 datapath"""
+        self.logger.info("FFP_TEST_START_0003")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=10, layer='l4')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=2)
+
+        self.create_stream(packets=1)
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 7: 'sport', 11: 'dport'})
+
+        # expected one template and one cflow packet
+        self.collector.get_capture(3)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0003")
+
+
+class TestFFP_DatapathIP4(MethodHolder):
+    """collect information on IP4 datapath"""
+
+    def test_0000(self):
+        """ verify templates on IP4 datapath"""
+        self.logger.info("FFP_TEST_START_0000")
+
+        self.pg_enable_capture(self.pg_interfaces)
+
+        ipfix = VppCFLOW(test=self, active=10, datapath='ip4')
+        ipfix.add_vpp_config()
+
+        # template packet should arrive immediately
+        ipfix.verify_templates(timeout=3, count=1)
+        self.collector.get_capture(1)
+
+        ipfix.remove_vpp_config()
+
+        self.logger.info("FFP_TEST_FINISH_0000")
+
+    def test_0001(self):
+        """ L2 data on IP4 datapath"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, intf='pg4', active=10,
+                         layer='l2', datapath='ip4')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(src_if=self.pg3, dst_if=self.pg4, packets=1)
+        capture = self.send_packets(src_if=self.pg3, dst_if=self.pg4)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 256: 8})
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+    def test_0002(self):
+        """ L3 data on IP4 datapath"""
+        self.logger.info("FFP_TEST_START_0002")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, intf='pg4', active=10,
+                         layer='l3', datapath='ip4')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(src_if=self.pg3, dst_if=self.pg4, packets=1)
+        capture = self.send_packets(src_if=self.pg3, dst_if=self.pg4)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {1: 'octets', 2: 'packets',
+                                       8: 'src_ip', 12: 'dst_ip'})
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0002")
+
+    def test_0003(self):
+        """ L4 data on IP4 datapath"""
+        self.logger.info("FFP_TEST_START_0003")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, intf='pg4', active=10,
+                         layer='l4', datapath='ip4')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(src_if=self.pg3, dst_if=self.pg4, packets=1)
+        capture = self.send_packets(src_if=self.pg3, dst_if=self.pg4)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 7: 'sport', 11: 'dport'})
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0003")
+
+
+class TestFFP_DatapathIP6(MethodHolder):
+    """collect information on IP6 datapath"""
+
+    def test_0000(self):
+        """ verify templates on IP6 datapath"""
+        self.logger.info("FFP_TEST_START_0000")
+        self.pg_enable_capture(self.pg_interfaces)
+
+        ipfix = VppCFLOW(test=self, active=10, datapath='ip6')
+        ipfix.add_vpp_config()
+
+        # template packet should arrive immediately
+        ipfix.verify_templates(timeout=3, count=1)
+        self.collector.get_capture(1)
+
+        ipfix.remove_vpp_config()
+
+        self.logger.info("FFP_TEST_FINISH_0000")
+
+    def test_0001(self):
+        """ L2 data on IP6 datapath"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, intf='pg6', active=20,
+                         layer='l2', datapath='ip6')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(src_if=self.pg5, dst_if=self.pg6, packets=1,
+                           ip_ver='IPv6')
+        capture = self.send_packets(src_if=self.pg5, dst_if=self.pg6)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 256: 56710},
+                                      ip_ver='v6')
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+    def test_0002(self):
+        """ L3 data on IP6 datapath"""
+        self.logger.info("FFP_TEST_START_0002")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, intf='pg6', active=10,
+                         layer='l3', datapath='ip6')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(src_if=self.pg5, dst_if=self.pg6, packets=1,
+                           ip_ver='IPv6')
+        capture = self.send_packets(src_if=self.pg5, dst_if=self.pg6)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets',
+                                       27: 'src_ip', 28: 'dst_ip'},
+                                      ip_ver='v6')
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0002")
+
+    def test_0003(self):
+        """ L4 data on IP6 datapath"""
+        self.logger.info("FFP_TEST_START_0003")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, intf='pg6', active=10,
+                         layer='l4', datapath='ip6')
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3, count=1)
+
+        self.create_stream(src_if=self.pg5, dst_if=self.pg6, packets=1,
+                           ip_ver='IPv6')
+        capture = self.send_packets(src_if=self.pg5, dst_if=self.pg6)
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[0], 39)
+        self.verify_cflow_data_detail(ipfix_decoder, capture, cflow,
+                                      {2: 'packets', 7: 'sport', 11: 'dport'},
+                                      ip_ver='v6')
+
+        # expected two templates and one cflow packet
+        self.collector.get_capture(2)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0003")
+
+
+class TestFFP_NoTimers(MethodHolder):
+    """No timers"""
+
+    def test_0001(self):
+        """ no timers, one CFLOW packet, 9 Flows inside"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=0)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream(packets=9)
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflow = self.wait_for_cflow_packet(self.collector, templates[1], 10)
+        self.verify_cflow_data_notimer(ipfix_decoder, capture, [cflow])
+        self.wait_for_cflow_packet(self.collector, templates[1], 10,
+                                   expected=False)
+        self.collector.get_capture(4)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+    def test_0002(self):
+        """ no timers, two CFLOW packets (mtu=256), 3 Flows in each"""
+        self.logger.info("FFP_TEST_START_0002")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=0, mtu=256)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream(packets=6)
+        capture = self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        cflows = []
+        cflows.append(self.wait_for_cflow_packet(self.collector,
+                                                 templates[1], 10))
+        cflows.append(self.wait_for_cflow_packet(self.collector,
+                                                 templates[1], 10))
+        self.verify_cflow_data_notimer(ipfix_decoder, capture, cflows)
+        self.collector.get_capture(5)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0002")
+
+
+class TestFFP_DisableIPFIX(MethodHolder):
+    """Disable IPFIX"""
+
+    def test_0001(self):
+        """ disable IPFIX after first packets"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=10)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=30)
+
+        self.create_stream()
+        self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.collector.get_capture(4)
+
+        # disble IPFIX
+        ipfix.disable_exporter()
+        self.pg_enable_capture([self.collector])
+
+        self.send_packets()
+
+        # make sure no one packet arrived in 1 minute
+        self.wait_for_cflow_packet(self.collector, templates[1], 30,
+                                   expected=False)
+        self.collector.get_capture(0)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+
+class TestFFP_ReenableIPFIX(MethodHolder):
+    """Re-enable IPFIX"""
+
+    def test_0001(self):
+        """ disable IPFIX after first packets and re-enable after few packets
+        """
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=10)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream()
+        self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.collector.get_capture(4)
+
+        # disble IPFIX
+        ipfix.disable_exporter()
+        self.pg_enable_capture([self.collector])
+
+        self.send_packets()
+
+        # make sure no one packet arrived in active timer span
+        self.wait_for_cflow_packet(self.collector, templates[1], 30,
+                                   expected=False)
+        self.collector.get_capture(0)
+
+        # enable IPFIX
+        ipfix.enable_exporter()
+        self.vapi.cli("ipfix flush")
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.send_packets()
+
+        # make sure the next packets (templates and data) we expect actually
+        # showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.collector.get_capture(4)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+
+class TestFFP_DisableFFP(MethodHolder):
+    """Disable Flowprobe feature"""
+
+    def test_0001(self):
+        """ disable flowprobe feature after first packets"""
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+        ipfix = VppCFLOW(test=self, active=10)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream()
+        self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.collector.get_capture(4)
+
+        # disble IPFIX
+        ipfix.disable_flowprobe_feature()
+        self.pg_enable_capture([self.collector])
+
+        self.send_packets()
+
+        # make sure no one packet arrived in active timer span
+        self.wait_for_cflow_packet(self.collector, templates[1], 30,
+                                   expected=False)
+        self.collector.get_capture(0)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+
+class TestFFP_ReenableFFP(MethodHolder):
+    """Re-enable Flowprobe feature"""
+
+    def test_0001(self):
+        """ disable flowprobe feature after first packets and re-enable
+        after few packets """
+        self.logger.info("FFP_TEST_START_0001")
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pkts = []
+
+        ipfix = VppCFLOW(test=self, active=10)
+        ipfix.add_vpp_config()
+
+        ipfix_decoder = IPFIXDecoder()
+        # template packet should arrive immediately
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.create_stream()
+        self.send_packets()
+
+        # make sure the one packet we expect actually showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.collector.get_capture(4)
+
+        # disble FPP feature
+        ipfix.disable_flowprobe_feature()
+        self.pg_enable_capture([self.collector])
+
+        self.send_packets()
+
+        # make sure no one packet arrived in active timer span
+        self.wait_for_cflow_packet(self.collector, templates[1], 30,
+                                   expected=False)
+        self.collector.get_capture(0)
+
+        # enable FPP feature
+        ipfix.enable_flowprobe_feature()
+        self.vapi.cli("ipfix flush")
+        templates = ipfix.verify_templates(ipfix_decoder, timeout=3)
+
+        self.send_packets()
+
+        # make sure the next packets (templates and data) we expect actually
+        # showed up
+        self.wait_for_cflow_packet(self.collector, templates[1], 30)
+        self.collector.get_capture(4)
+
+        ipfix.remove_vpp_config()
+        self.logger.info("FFP_TEST_FINISH_0001")
+
+
+if __name__ == '__main__':
+    unittest.main(testRunner=VppTestRunner)
-- 
cgit 1.2.3-korg


From 62f9cdd82c52dc05cb89a742d21aba013ce526d4 Mon Sep 17 00:00:00 2001
From: Hongjun Ni 
Date: Tue, 4 Jul 2017 20:11:57 +0800
Subject: Add PPPoE Plugin

Supports 64K PPPoE sessions

This plugin adds three graph nodes:
1) pppoe-input for PPPoE decapsulation
2) pppoe-encap for PPPoE encapsulation
3) pppoe-tap-dispatch for control plane process

Below is the configuration to make PPPoE CP and DP work:
vim /etc/vpp/startup.conf
tuntap {
  enable
  ethernet
  name newtap
}

create pppoe tap tap-if-index 1

//Configure it after a subscriber's PPPoE discovery and PPP link establishment succeeds:
create pppoe session client-ip 100.1.2.1 session-id 1 client-mac 00:11:01:00:00:01

show pppoe fib
show pppoe session

Change-Id: I73e724b6bf7c3e4181a9914c5752da1fa72d7e60
Signed-off-by: Hongjun Ni 
---
 MAINTAINERS                              |   5 +
 src/configure.ac                         |   1 +
 src/plugins/Makefile.am                  |   4 +
 src/plugins/pppoe.am                     |  40 ++
 src/plugins/pppoe/pppoe.api              |  90 +++++
 src/plugins/pppoe/pppoe.c                | 660 +++++++++++++++++++++++++++++++
 src/plugins/pppoe/pppoe.h                | 295 ++++++++++++++
 src/plugins/pppoe/pppoe_all_api_h.h      |  18 +
 src/plugins/pppoe/pppoe_api.c            | 224 +++++++++++
 src/plugins/pppoe/pppoe_decap.c          | 422 ++++++++++++++++++++
 src/plugins/pppoe/pppoe_encap.c          | 384 ++++++++++++++++++
 src/plugins/pppoe/pppoe_error.def        |  18 +
 src/plugins/pppoe/pppoe_msg_enum.h       |  31 ++
 src/plugins/pppoe/pppoe_tap.c            |  89 +++++
 src/plugins/pppoe/pppoe_tap_node.c       | 297 ++++++++++++++
 src/plugins/pppoe/pppoe_test.c           | 330 ++++++++++++++++
 src/vpp-api/java/Makefile.am             |  20 +
 src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.c | 107 +++++
 src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.h |  42 ++
 test/test_pppoe.py                       | 605 ++++++++++++++++++++++++++++
 test/vpp_papi_provider.py                |  26 ++
 test/vpp_pppoe_interface.py              |  79 ++++
 22 files changed, 3787 insertions(+)
 create mode 100644 src/plugins/pppoe.am
 create mode 100644 src/plugins/pppoe/pppoe.api
 create mode 100644 src/plugins/pppoe/pppoe.c
 create mode 100644 src/plugins/pppoe/pppoe.h
 create mode 100644 src/plugins/pppoe/pppoe_all_api_h.h
 create mode 100644 src/plugins/pppoe/pppoe_api.c
 create mode 100644 src/plugins/pppoe/pppoe_decap.c
 create mode 100644 src/plugins/pppoe/pppoe_encap.c
 create mode 100644 src/plugins/pppoe/pppoe_error.def
 create mode 100644 src/plugins/pppoe/pppoe_msg_enum.h
 create mode 100644 src/plugins/pppoe/pppoe_tap.c
 create mode 100644 src/plugins/pppoe/pppoe_tap_node.c
 create mode 100644 src/plugins/pppoe/pppoe_test.c
 create mode 100644 src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.c
 create mode 100644 src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.h
 create mode 100644 test/test_pppoe.py
 create mode 100644 test/vpp_pppoe_interface.py

(limited to 'src/plugins/Makefile.am')

diff --git a/MAINTAINERS b/MAINTAINERS
index 55fe00bf..4f2918fa 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -132,6 +132,11 @@ M:	Hongjun Ni 
 F:	src/plugins/gtpu/
 F:	src/plugins/gtpu.am
 
+Plugin - PPPoE
+M:  Hongjun Ni 
+F:  src/plugins/pppoe/
+F:  src/plugins/pppoe.am
+
 Test Infrastructure
 M:	Klement Sekera 
 F:	test/
diff --git a/src/configure.ac b/src/configure.ac
index cb00d0bd..7a038c2e 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -167,6 +167,7 @@ PLUGIN_ENABLED(ioam)
 PLUGIN_ENABLED(ixge)
 PLUGIN_ENABLED(lb)
 PLUGIN_ENABLED(memif)
+PLUGIN_ENABLED(pppoe)
 PLUGIN_ENABLED(sixrd)
 PLUGIN_ENABLED(snat)
 
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index f26d0fd2..8c7b3fac 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -66,6 +66,10 @@ if ENABLE_MEMIF_PLUGIN
 include memif.am
 endif
 
+if ENABLE_PPPOE_PLUGIN
+include pppoe.am
+endif
+
 if ENABLE_SIXRD_PLUGIN
 include sixrd.am
 endif
diff --git a/src/plugins/pppoe.am b/src/plugins/pppoe.am
new file mode 100644
index 00000000..28bd20a0
--- /dev/null
+++ b/src/plugins/pppoe.am
@@ -0,0 +1,40 @@
+# Copyright (c) 2017 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppapitestplugins_LTLIBRARIES += pppoe_test_plugin.la
+vppplugins_LTLIBRARIES += pppoe_plugin.la
+
+pppoe_plugin_la_SOURCES =		\
+    pppoe/pppoe_decap.c		    \
+    pppoe/pppoe_encap.c		    \
+    pppoe/pppoe_tap.c		    \
+    pppoe/pppoe_tap_node.c		\
+    pppoe/pppoe.c		        \
+    pppoe/pppoe_api.c
+
+BUILT_SOURCES +=			   \
+    pppoe/pppoe.api.h		    \
+    pppoe/pppoe.api.json
+
+API_FILES += pppoe/pppoe.api
+
+nobase_apiinclude_HEADERS +=	\
+    pppoe/pppoe_all_api_h.h		\
+    pppoe/pppoe_msg_enum.h		\
+    pppoe/pppoe.api.h
+
+pppoe_test_plugin_la_SOURCES = \
+    pppoe/pppoe_test.c            \
+    pppoe/pppoe_plugin.api.h
+
+# vi:syntax=automake
diff --git a/src/plugins/pppoe/pppoe.api b/src/plugins/pppoe/pppoe.api
new file mode 100644
index 00000000..e8cd989f
--- /dev/null
+++ b/src/plugins/pppoe/pppoe.api
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** \brief Set or delete an PPPOE session
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - add address if non-zero, else delete
+    @param is_ipv6 - client_ip and dst_address is ipv6 or not
+    @param session_id - PPPoE session ID
+    @param client_ip - PPPOE session's client address.
+    @param decap_vrf_id - the vrf index for pppoe decaped packet
+    @param client_mac - the client ethernet address
+*/
+define pppoe_add_del_session
+{
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_ipv6;
+  u16 session_id;
+  u8 client_ip[16];
+  u32 decap_vrf_id;
+  u8 client_mac[6];
+};
+
+/** \brief reply for set or delete an PPPOE session
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param sw_if_index - software index of the interface
+*/
+define pppoe_add_del_session_reply
+{
+  u32 context;
+  i32 retval;
+  u32 sw_if_index;
+};
+
+/** \brief Dump PPPOE session
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface
+*/
+define pppoe_session_dump
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+};
+
+/** \brief dump details of an PPPOE session
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface
+    @param is_ipv6 - client_ip and dst_address is ipv6 or not
+    @param session_id - PPPoE session ID
+    @param client_ip - PPPOE session's client address.
+    @param encap_if_index - the index of tx interface for pppoe encaped packet
+    @param decap_vrf_id - the vrf index for pppoe decaped packet
+    @param local_mac - the local ethernet address
+    @param client_mac - the client ethernet address
+*/
+define pppoe_session_details
+{
+  u32 context;
+  u32 sw_if_index;
+  u8 is_ipv6;
+  u16 session_id;
+  u8 client_ip[16];
+  u32 encap_if_index;
+  u32 decap_vrf_id;
+  u8 local_mac[6];
+  u8 client_mac[6];
+};
+
+/*
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/pppoe/pppoe.c b/src/plugins/pppoe/pppoe.c
new file mode 100644
index 00000000..4c63902a
--- /dev/null
+++ b/src/plugins/pppoe/pppoe.c
@@ -0,0 +1,660 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+pppoe_main_t pppoe_main;
+
+u8 *
+format_pppoe_session (u8 * s, va_list * args)
+{
+  pppoe_session_t *t = va_arg (*args, pppoe_session_t *);
+  pppoe_main_t *pem = &pppoe_main;
+
+  s = format (s, "[%d] sw-if-index %d client-ip %U session-id %d ",
+	      t - pem->sessions, t->sw_if_index,
+	      format_ip46_address, &t->client_ip, IP46_TYPE_ANY,
+	      t->session_id);
+
+  s = format (s, "encap-if-index %d decap-fib-index %d\n",
+	      t->encap_if_index, t->decap_fib_index);
+
+  s = format (s, "    local-mac %U  client-mac %U",
+	      format_ethernet_address, t->local_mac,
+	      format_ethernet_address, t->client_mac);
+
+  return s;
+}
+
+static u8 *
+format_pppoe_name (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  return format (s, "pppoe_session%d", dev_instance);
+}
+
+static uword
+dummy_interface_tx (vlib_main_t * vm,
+		    vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  clib_warning ("you shouldn't be here, leaking buffers...");
+  return frame->n_vectors;
+}
+
+static clib_error_t *
+pppoe_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
+{
+  u32 hw_flags = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ?
+    VNET_HW_INTERFACE_FLAG_LINK_UP : 0;
+  vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags);
+
+  return /* no error */ 0;
+}
+
+/* *INDENT-OFF* */
+VNET_DEVICE_CLASS (pppoe_device_class,static) = {
+  .name = "PPPPOE",
+  .format_device_name = format_pppoe_name,
+  .format_tx_trace = format_pppoe_encap_trace,
+  .tx_function = dummy_interface_tx,
+  .admin_up_down_function = pppoe_interface_admin_up_down,
+};
+/* *INDENT-ON* */
+
+static u8 *
+format_pppoe_header_with_length (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  s = format (s, "unimplemented dev %u", dev_instance);
+  return s;
+}
+
+/* *INDENT-OFF* */
+VNET_HW_INTERFACE_CLASS (pppoe_hw_class) =
+{
+  .name = "PPPPOE",
+  .format_header = format_pppoe_header_with_length,
+  .build_rewrite = default_build_rewrite,
+  .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
+};
+/* *INDENT-ON* */
+
+#define foreach_copy_field                      \
+_(session_id)                                   \
+_(encap_if_index)                               \
+_(decap_fib_index)                              \
+_(client_ip)
+
+static void
+eth_pppoe_rewrite (pppoe_session_t * t, bool is_ip6)
+{
+  u8 *rw = 0;
+  int len = sizeof (pppoe_header_t) + sizeof (ethernet_header_t);
+
+  vec_validate_aligned (rw, len - 1, CLIB_CACHE_LINE_BYTES);
+
+  ethernet_header_t *eth_hdr = (ethernet_header_t *) rw;
+  clib_memcpy (eth_hdr->dst_address, t->client_mac, 6);
+  clib_memcpy (eth_hdr->src_address, t->local_mac, 6);
+  eth_hdr->type = clib_host_to_net_u16 (ETHERNET_TYPE_PPPOE_SESSION);
+
+  pppoe_header_t *pppoe = (pppoe_header_t *) (eth_hdr + 1);
+  pppoe->ver_type = PPPOE_VER_TYPE;
+  pppoe->code = 0;
+  pppoe->session_id = clib_host_to_net_u16 (t->session_id);
+  pppoe->length = 0;		/* To be filled in at run-time */
+
+  if (!is_ip6)
+    {
+      pppoe->ppp_proto = clib_host_to_net_u16 (PPP_PROTOCOL_ip4);
+    }
+  else
+    {
+      pppoe->ppp_proto = clib_host_to_net_u16 (PPP_PROTOCOL_ip6);
+    }
+
+  t->rewrite = rw;
+  _vec_len (t->rewrite) = len;
+
+  return;
+}
+
+static bool
+pppoe_decap_next_is_valid (pppoe_main_t * pem, u32 is_ip6,
+			   u32 decap_fib_index)
+{
+  vlib_main_t *vm = pem->vlib_main;
+  u32 input_idx = (!is_ip6) ? ip4_input_node.index : ip6_input_node.index;
+  vlib_node_runtime_t *r = vlib_node_get_runtime (vm, input_idx);
+
+  return decap_fib_index < r->n_next_nodes;
+}
+
+int vnet_pppoe_add_del_session
+  (vnet_pppoe_add_del_session_args_t * a, u32 * sw_if_indexp)
+{
+  pppoe_main_t *pem = &pppoe_main;
+  pppoe_session_t *t = 0;
+  vnet_main_t *vnm = pem->vnet_main;
+  u32 hw_if_index = ~0;
+  u32 sw_if_index = ~0;
+  u32 is_ip6 = a->is_ip6;
+  pppoe_entry_key_t cached_key;
+  pppoe_entry_result_t cached_result;
+  u32 bucket;
+  pppoe_entry_key_t key;
+  pppoe_entry_result_t result;
+  vnet_hw_interface_t *hi;
+  vnet_sw_interface_t *si;
+  fib_prefix_t pfx;
+
+  cached_key.raw = ~0;
+  cached_result.raw = ~0;	/* warning be gone */
+  memset (&pfx, 0, sizeof (pfx));
+
+  if (!is_ip6)
+    {
+      pfx.fp_addr.ip4.as_u32 = a->client_ip.ip4.as_u32;
+      pfx.fp_len = 32;
+      pfx.fp_proto = FIB_PROTOCOL_IP4;
+    }
+  else
+    {
+      pfx.fp_addr.ip6.as_u64[0] = a->client_ip.ip6.as_u64[0];
+      pfx.fp_addr.ip6.as_u64[1] = a->client_ip.ip6.as_u64[1];
+      pfx.fp_len = 128;
+      pfx.fp_proto = FIB_PROTOCOL_IP6;
+    }
+
+  /* Get encap_if_index and local mac address */
+  pppoe_lookup_1 (&pem->session_table, &cached_key, &cached_result,
+		  a->client_mac, clib_host_to_net_u16 (a->session_id),
+		  &key, &bucket, &result);
+  a->encap_if_index = result.fields.sw_if_index;
+
+  if (a->encap_if_index == ~0)
+    return VNET_API_ERROR_INVALID_SW_IF_INDEX;
+
+  si = vnet_get_sw_interface (vnm, a->encap_if_index);
+  hi = vnet_get_hw_interface (vnm, si->hw_if_index);
+
+
+  if (a->is_add)
+    {
+      /* adding a session: session must not already exist */
+      if (result.fields.session_index != ~0)
+	return VNET_API_ERROR_TUNNEL_EXIST;
+
+      /*if not set explicitly, default to ip4 */
+      if (!pppoe_decap_next_is_valid (pem, is_ip6, a->decap_fib_index))
+	return VNET_API_ERROR_INVALID_DECAP_NEXT;
+
+      pool_get_aligned (pem->sessions, t, CLIB_CACHE_LINE_BYTES);
+      memset (t, 0, sizeof (*t));
+
+      clib_memcpy (t->local_mac, hi->hw_address, 6);
+
+      /* copy from arg structure */
+#define _(x) t->x = a->x;
+      foreach_copy_field;
+#undef _
+
+      clib_memcpy (t->client_mac, a->client_mac, 6);
+
+      eth_pppoe_rewrite (t, is_ip6);
+
+      /* update pppoe fib with session_index */
+      result.fields.session_index = t - pem->sessions;
+      pppoe_update_1 (&pem->session_table,
+		      a->client_mac, clib_host_to_net_u16 (a->session_id),
+		      &key, &bucket, &result);
+
+      vnet_hw_interface_t *hi;
+      if (vec_len (pem->free_pppoe_session_hw_if_indices) > 0)
+	{
+	  vnet_interface_main_t *im = &vnm->interface_main;
+	  hw_if_index = pem->free_pppoe_session_hw_if_indices
+	    [vec_len (pem->free_pppoe_session_hw_if_indices) - 1];
+	  _vec_len (pem->free_pppoe_session_hw_if_indices) -= 1;
+
+	  hi = vnet_get_hw_interface (vnm, hw_if_index);
+	  hi->dev_instance = t - pem->sessions;
+	  hi->hw_instance = hi->dev_instance;
+
+	  /* clear old stats of freed session before reuse */
+	  sw_if_index = hi->sw_if_index;
+	  vnet_interface_counter_lock (im);
+	  vlib_zero_combined_counter
+	    (&im->combined_sw_if_counters[VNET_INTERFACE_COUNTER_TX],
+	     sw_if_index);
+	  vlib_zero_combined_counter (&im->combined_sw_if_counters
+				      [VNET_INTERFACE_COUNTER_RX],
+				      sw_if_index);
+	  vlib_zero_simple_counter (&im->sw_if_counters
+				    [VNET_INTERFACE_COUNTER_DROP],
+				    sw_if_index);
+	  vnet_interface_counter_unlock (im);
+	}
+      else
+	{
+	  hw_if_index = vnet_register_interface
+	    (vnm, pppoe_device_class.index, t - pem->sessions,
+	     pppoe_hw_class.index, t - pem->sessions);
+	  hi = vnet_get_hw_interface (vnm, hw_if_index);
+	}
+
+      t->hw_if_index = hw_if_index;
+      t->sw_if_index = sw_if_index = hi->sw_if_index;
+
+      vec_validate_init_empty (pem->session_index_by_sw_if_index, sw_if_index,
+			       ~0);
+      pem->session_index_by_sw_if_index[sw_if_index] = t - pem->sessions;
+
+      vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index);
+      si->flags &= ~VNET_SW_INTERFACE_FLAG_HIDDEN;
+      vnet_sw_interface_set_flags (vnm, sw_if_index,
+				   VNET_SW_INTERFACE_FLAG_ADMIN_UP);
+
+      /* Set pppoe session output node */
+      hi->output_node_index = pppoe_encap_node.index;
+
+      /* add reverse route for client ip */
+      fib_table_entry_path_add (a->decap_fib_index, &pfx,
+				FIB_SOURCE_PLUGIN_HI, FIB_ENTRY_FLAG_NONE,
+				pfx.fp_proto, &pfx.fp_addr, sw_if_index, ~0,
+				1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
+
+    }
+  else
+    {
+      /* deleting a session: session must exist */
+      if (result.fields.session_index == ~0)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      t = pool_elt_at_index (pem->sessions, result.fields.session_index);
+      sw_if_index = t->sw_if_index;
+
+      vnet_sw_interface_set_flags (vnm, t->sw_if_index, 0 /* down */ );
+      vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, t->sw_if_index);
+      si->flags |= VNET_SW_INTERFACE_FLAG_HIDDEN;
+
+      vec_add1 (pem->free_pppoe_session_hw_if_indices, t->hw_if_index);
+
+      pem->session_index_by_sw_if_index[t->sw_if_index] = ~0;
+
+      /* update pppoe fib with session_inde=~0x */
+      result.fields.session_index = ~0;
+      pppoe_update_1 (&pem->session_table,
+		      a->client_mac, clib_host_to_net_u16 (a->session_id),
+		      &key, &bucket, &result);
+
+
+      /* delete reverse route for client ip */
+      fib_table_entry_path_remove (a->decap_fib_index, &pfx,
+				   FIB_SOURCE_PLUGIN_HI,
+				   pfx.fp_proto,
+				   &pfx.fp_addr,
+				   sw_if_index, ~0, 1,
+				   FIB_ROUTE_PATH_FLAG_NONE);
+
+      vec_free (t->rewrite);
+      pool_put (pem->sessions, t);
+    }
+
+  if (sw_if_indexp)
+    *sw_if_indexp = sw_if_index;
+
+  return 0;
+}
+
+static clib_error_t *
+pppoe_add_del_session_command_fn (vlib_main_t * vm,
+				  unformat_input_t * input,
+				  vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u16 session_id = 0;
+  ip46_address_t client_ip;
+  u8 is_add = 1;
+  u8 client_ip_set = 0;
+  u8 ipv4_set = 0;
+  u8 ipv6_set = 0;
+  u32 encap_if_index = 0;
+  u32 decap_fib_index = 0;
+  u8 client_mac[6] = { 0 };
+  u8 client_mac_set = 0;
+  int rv;
+  u32 tmp;
+  vnet_pppoe_add_del_session_args_t _a, *a = &_a;
+  u32 session_sw_if_index;
+  clib_error_t *error = NULL;
+
+  /* Cant "universally zero init" (={0}) due to GCC bug 53119 */
+  memset (&client_ip, 0, sizeof client_ip);
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "del"))
+	{
+	  is_add = 0;
+	}
+      else if (unformat (line_input, "session-id %d", &session_id))
+	;
+      else if (unformat (line_input, "client-ip %U",
+			 unformat_ip4_address, &client_ip.ip4))
+	{
+	  client_ip_set = 1;
+	  ipv4_set = 1;
+	}
+      else if (unformat (line_input, "client-ip %U",
+			 unformat_ip6_address, &client_ip.ip6))
+	{
+	  client_ip_set = 1;
+	  ipv6_set = 1;
+	}
+      else if (unformat (line_input, "decap-vrf-id %d", &tmp))
+	{
+	  if (ipv6_set)
+	    decap_fib_index = fib_table_find (FIB_PROTOCOL_IP6, tmp);
+	  else
+	    decap_fib_index = fib_table_find (FIB_PROTOCOL_IP4, tmp);
+
+	  if (decap_fib_index == ~0)
+	    {
+	      error =
+		clib_error_return (0, "nonexistent decap fib id %d", tmp);
+	      goto done;
+	    }
+	}
+      else
+	if (unformat
+	    (line_input, "client-mac %U", unformat_ethernet_address,
+	     client_mac))
+	client_mac_set = 1;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (client_ip_set == 0)
+    {
+      error =
+	clib_error_return (0, "session client ip address not specified");
+      goto done;
+    }
+
+  if (ipv4_set && ipv6_set)
+    {
+      error = clib_error_return (0, "both IPv4 and IPv6 addresses specified");
+      goto done;
+    }
+
+  if (client_mac_set == 0)
+    {
+      error = clib_error_return (0, "session client mac not specified");
+      goto done;
+    }
+
+  memset (a, 0, sizeof (*a));
+
+  a->is_add = is_add;
+  a->is_ip6 = ipv6_set;
+
+#define _(x) a->x = x;
+  foreach_copy_field;
+#undef _
+
+  clib_memcpy (a->client_mac, client_mac, 6);
+
+  rv = vnet_pppoe_add_del_session (a, &session_sw_if_index);
+
+  switch (rv)
+    {
+    case 0:
+      if (is_add)
+	vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name,
+			 vnet_get_main (), session_sw_if_index);
+      break;
+
+    case VNET_API_ERROR_TUNNEL_EXIST:
+      error = clib_error_return (0, "session already exists...");
+      goto done;
+
+    case VNET_API_ERROR_NO_SUCH_ENTRY:
+      error = clib_error_return (0, "session does not exist...");
+      goto done;
+
+    default:
+      error = clib_error_return
+	(0, "vnet_pppoe_add_del_session returned %d", rv);
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * Add or delete a PPPPOE Session.
+ *
+ * @cliexpar
+ * Example of how to create a PPPPOE Session:
+ * @cliexcmd{create pppoe session client-ip 10.0.3.1 session-id 13
+ *             client-mac 00:01:02:03:04:05 }
+ * Example of how to delete a PPPPOE Session:
+ * @cliexcmd{create pppoe session client-ip 10.0.3.1 session-id 13
+ *             client-mac 00:01:02:03:04:05 del }
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (create_pppoe_session_command, static) = {
+  .path = "create pppoe session",
+  .short_help =
+  "create pppoe session client-ip  session-id "
+  " client-mac  [decap-vrf-id ] [del]",
+  .function = pppoe_add_del_session_command_fn,
+};
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+static clib_error_t *
+show_pppoe_session_command_fn (vlib_main_t * vm,
+			       unformat_input_t * input,
+			       vlib_cli_command_t * cmd)
+{
+  pppoe_main_t *pem = &pppoe_main;
+  pppoe_session_t *t;
+
+  if (pool_elts (pem->sessions) == 0)
+    vlib_cli_output (vm, "No pppoe sessions configured...");
+
+  pool_foreach (t, pem->sessions,
+		({
+		    vlib_cli_output (vm, "%U",format_pppoe_session, t);
+		}));
+
+  return 0;
+}
+/* *INDENT-ON* */
+
+/*?
+ * Display all the PPPPOE Session entries.
+ *
+ * @cliexpar
+ * Example of how to display the PPPPOE Session entries:
+ * @cliexstart{show pppoe session}
+ * [0] client-ip 10.0.3.1 session_id 13 encap-if-index 0 decap-vrf-id 13 sw_if_index 5
+ *     local-mac a0:b0:c0:d0:e0:f0 client-mac 00:01:02:03:04:05
+ * @cliexend
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_pppoe_session_command, static) = {
+    .path = "show pppoe session",
+    .short_help = "show pppoe session",
+    .function = show_pppoe_session_command_fn,
+};
+/* *INDENT-ON* */
+
+/** Display the contents of the PPPoE Fib. */
+static clib_error_t *
+show_pppoe_fib_command_fn (vlib_main_t * vm,
+			   unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  pppoe_main_t *pem = &pppoe_main;
+  BVT (clib_bihash) * h = &pem->session_table;
+  BVT (clib_bihash_bucket) * b;
+  BVT (clib_bihash_value) * v;
+  pppoe_entry_key_t key;
+  pppoe_entry_result_t result;
+  u32 first_entry = 1;
+  u64 total_entries = 0;
+  int i, j, k;
+  u8 *s = 0;
+
+  for (i = 0; i < h->nbuckets; i++)
+    {
+      b = &h->buckets[i];
+      if (b->offset == 0)
+	continue;
+      v = BV (clib_bihash_get_value) (h, b->offset);
+      for (j = 0; j < (1 << b->log2_pages); j++)
+	{
+	  for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
+	    {
+	      if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
+		continue;
+
+	      if (first_entry)
+		{
+		  first_entry = 0;
+		  vlib_cli_output (vm,
+				   "%=19s%=12s%=13s%=14s",
+				   "Mac-Address", "session_id", "sw_if_index",
+				   "session_index");
+		}
+
+	      key.raw = v->kvp[k].key;
+	      result.raw = v->kvp[k].value;
+
+
+	      vlib_cli_output (vm,
+			       "%=19U%=12d%=13d%=14d",
+			       format_ethernet_address, key.fields.mac,
+			       clib_net_to_host_u16 (key.fields.session_id),
+			       result.fields.sw_if_index == ~0
+			       ? -1 : result.fields.sw_if_index,
+			       result.fields.session_index == ~0
+			       ? -1 : result.fields.session_index);
+	      vec_reset_length (s);
+	      total_entries++;
+	    }
+	  v++;
+	}
+    }
+
+  if (total_entries == 0)
+    vlib_cli_output (vm, "no pppoe fib entries");
+  else
+    vlib_cli_output (vm, "%lld pppoe fib entries", total_entries);
+
+  vec_free (s);
+  return 0;
+}
+
+/*?
+ * This command dispays the MAC Address entries of the PPPoE FIB table.
+ * Output can be filtered to just get the number of MAC Addresses or display
+ * each MAC Address.
+ *
+ * @cliexpar
+ * Example of how to display the number of MAC Address entries in the PPPoE
+ * FIB table:
+ * @cliexstart{show pppoe fib}
+ *     Mac Address      session_id      Interface           sw_if_index  session_index
+ *  52:54:00:53:18:33     1          GigabitEthernet0/8/0        2          0
+ *  52:54:00:53:18:55     2          GigabitEthernet0/8/1        3          1
+ * @cliexend
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (show_pppoe_fib_command, static) = {
+    .path = "show pppoe fib",
+    .short_help = "show pppoe fib",
+    .function = show_pppoe_fib_command_fn,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+pppoe_init (vlib_main_t * vm)
+{
+  pppoe_main_t *pem = &pppoe_main;
+
+  pem->vnet_main = vnet_get_main ();
+  pem->vlib_main = vm;
+
+  /* Create the hash table  */
+  BV (clib_bihash_init) (&pem->session_table, "pppoe session table",
+			 PPPOE_NUM_BUCKETS, PPPOE_MEMORY_SIZE);
+
+  ethernet_register_input_type (vm, ETHERNET_TYPE_PPPOE_SESSION,
+				pppoe_input_node.index);
+
+  ethernet_register_input_type (vm, ETHERNET_TYPE_PPPOE_DISCOVERY,
+				pppoe_tap_dispatch_node.index);
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (pppoe_init);
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .description = "PPPoE",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/pppoe/pppoe.h b/src/plugins/pppoe/pppoe.h
new file mode 100644
index 00000000..37d628eb
--- /dev/null
+++ b/src/plugins/pppoe/pppoe.h
@@ -0,0 +1,295 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#ifndef _PPPOE_H
+#define _PPPOE_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+
+typedef struct
+{
+  u8 ver_type;
+  u8 code;
+  u16 session_id;
+  u16 length;
+  u16 ppp_proto;
+} pppoe_header_t;
+
+#define PPPOE_VER_TYPE 0x11
+#define PPPOE_PADS 0x65
+
+typedef struct
+{
+  /* Rewrite string */
+  u8 *rewrite;
+
+  /* pppoe session_id in HOST byte order */
+  u16 session_id;
+
+  /* session client addresses */
+  ip46_address_t client_ip;
+
+  /* the index of tx interface for pppoe encaped packet */
+  u32 encap_if_index;
+
+  /** FIB indices - inner IP packet lookup here */
+  u32 decap_fib_index;
+
+  u8 local_mac[6];
+  u8 client_mac[6];
+
+  /* vnet intfc index */
+  u32 sw_if_index;
+  u32 hw_if_index;
+
+} pppoe_session_t;
+
+#define foreach_pppoe_input_next        \
+_(DROP, "error-drop")                  \
+_(IP4_INPUT, "ip4-input")              \
+_(IP6_INPUT, "ip6-input" )             \
+_(CP_INPUT, "pppoe-tap-dispatch" )     \
+
+typedef enum
+{
+#define _(s,n) PPPOE_INPUT_NEXT_##s,
+  foreach_pppoe_input_next
+#undef _
+    PPPOE_INPUT_N_NEXT,
+} pppoe_input_next_t;
+
+typedef enum
+{
+#define pppoe_error(n,s) PPPOE_ERROR_##n,
+#include 
+#undef pppoe_error
+  PPPOE_N_ERROR,
+} pppoe_input_error_t;
+
+
+#define MTU 1500
+#define MTU_BUFFERS ((MTU + VLIB_BUFFER_DATA_SIZE - 1) / VLIB_BUFFER_DATA_SIZE)
+#define NUM_BUFFERS_TO_ALLOC 32
+
+/*
+ * The size of pppoe session table
+ */
+#define PPPOE_NUM_BUCKETS (128 * 1024)
+#define PPPOE_MEMORY_SIZE (16<<20)
+
+/* *INDENT-OFF* */
+/*
+ * The PPPoE key is the mac address and session ID
+ */
+typedef struct
+{
+  union
+  {
+    struct
+    {
+      u16 session_id;
+      u8 mac[6];
+    } fields;
+    struct
+    {
+      u32 w0;
+      u32 w1;
+    } words;
+    u64 raw;
+  };
+} pppoe_entry_key_t;
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+/*
+ * The PPPoE entry results
+ */
+typedef struct
+{
+  union
+  {
+    struct
+    {
+      u32 sw_if_index;
+
+      u32 session_index;
+
+    } fields;
+    u64 raw;
+  };
+}  pppoe_entry_result_t;
+/* *INDENT-ON* */
+
+typedef struct
+{
+  /* For DP: vector of encap session instances, */
+  pppoe_session_t *sessions;
+
+  /* For CP:  vector of CP path */
+    BVT (clib_bihash) session_table;
+
+  /* Free vlib hw_if_indices */
+  u32 *free_pppoe_session_hw_if_indices;
+
+  /* Mapping from sw_if_index to session index */
+  u32 *session_index_by_sw_if_index;
+
+  /* used for pppoe cp path */
+  u32 tap_if_index;
+
+  /* API message ID base */
+  u16 msg_id_base;
+
+  /* convenience */
+  vlib_main_t *vlib_main;
+  vnet_main_t *vnet_main;
+
+} pppoe_main_t;
+
+extern pppoe_main_t pppoe_main;
+
+extern vlib_node_registration_t pppoe_input_node;
+extern vlib_node_registration_t pppoe_encap_node;
+extern vlib_node_registration_t pppoe_tap_dispatch_node;
+
+u8 *format_pppoe_encap_trace (u8 * s, va_list * args);
+
+typedef struct
+{
+  u8 is_add;
+  u8 is_ip6;
+  u16 session_id;
+  ip46_address_t client_ip;
+  u32 encap_if_index;
+  u32 decap_fib_index;
+  u8 local_mac[6];
+  u8 client_mac[6];
+} vnet_pppoe_add_del_session_args_t;
+
+int vnet_pppoe_add_del_session
+  (vnet_pppoe_add_del_session_args_t * a, u32 * sw_if_indexp);
+
+typedef struct
+{
+  u8 is_add;
+  u32 client_if_index;
+  u32 tap_if_index;
+} vnet_pppoe_add_del_tap_args_t;
+
+always_inline u64
+pppoe_make_key (u8 * mac_address, u16 session_id)
+{
+  u64 temp;
+
+  /*
+   * The mac address in memory is A:B:C:D:E:F
+   * The session_id in register is H:L
+   */
+#if CLIB_ARCH_IS_LITTLE_ENDIAN
+  /*
+   * Create the in-register key as F:E:D:C:B:A:H:L
+   * In memory the key is L:H:A:B:C:D:E:F
+   */
+  temp = *((u64 *) (mac_address)) << 16;
+  temp = (temp & ~0xffff) | (u64) (session_id);
+#else
+  /*
+   * Create the in-register key as H:L:A:B:C:D:E:F
+   * In memory the key is H:L:A:B:C:D:E:F
+   */
+  temp = *((u64 *) (mac_address)) >> 16;
+  temp = temp | (((u64) session_id) << 48);
+#endif
+
+  return temp;
+}
+
+static_always_inline void
+pppoe_lookup_1 (BVT (clib_bihash) * session_table,
+		pppoe_entry_key_t * cached_key,
+		pppoe_entry_result_t * cached_result,
+		u8 * mac0,
+		u16 session_id0,
+		pppoe_entry_key_t * key0,
+		u32 * bucket0, pppoe_entry_result_t * result0)
+{
+  /* set up key */
+  key0->raw = pppoe_make_key (mac0, session_id0);
+  *bucket0 = ~0;
+
+  if (key0->raw == cached_key->raw)
+    {
+      /* Hit in the one-entry cache */
+      result0->raw = cached_result->raw;
+    }
+  else
+    {
+      /* Do a regular session table lookup */
+      BVT (clib_bihash_kv) kv;
+
+      kv.key = key0->raw;
+      kv.value = ~0ULL;
+      BV (clib_bihash_search_inline) (session_table, &kv);
+      result0->raw = kv.value;
+
+      /* Update one-entry cache */
+      cached_key->raw = key0->raw;
+      cached_result->raw = result0->raw;
+    }
+}
+
+static_always_inline void
+pppoe_update_1 (BVT (clib_bihash) * session_table,
+		u8 * mac0,
+		u16 session_id0,
+		pppoe_entry_key_t * key0,
+		u32 * bucket0, pppoe_entry_result_t * result0)
+{
+  /* set up key */
+  key0->raw = pppoe_make_key (mac0, session_id0);
+  *bucket0 = ~0;
+
+  /* Update the entry */
+  BVT (clib_bihash_kv) kv;
+  kv.key = key0->raw;
+  kv.value = result0->raw;
+  BV (clib_bihash_add_del) (session_table, &kv, 1 /* is_add */ );
+
+}
+#endif /* _PPPOE_H */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/pppoe/pppoe_all_api_h.h b/src/plugins/pppoe/pppoe_all_api_h.h
new file mode 100644
index 00000000..393c7680
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_all_api_h.h
@@ -0,0 +1,18 @@
+/*
+ * pppoe_all_api_h.h - plug-in api #include file
+ *
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include 
diff --git a/src/plugins/pppoe/pppoe_api.c b/src/plugins/pppoe/pppoe_api.c
new file mode 100644
index 00000000..9b758460
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_api.c
@@ -0,0 +1,224 @@
+/*
+ *------------------------------------------------------------------
+ * pppoe_api.c - pppoe api
+ *
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include 
+  /* We'll want to know how many messages IDs we need... */
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+#define vl_msg_name_crc_list
+#include 
+#undef vl_msg_name_crc_list
+
+#define REPLY_MSG_ID_BASE pem->msg_id_base
+#include 
+
+static void
+setup_message_id_table (pppoe_main_t * pem, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + pem->msg_id_base);
+  foreach_vl_msg_name_crc_pppoe;
+#undef _
+}
+
+#define foreach_pppoe_plugin_api_msg                             \
+_(PPPOE_ADD_DEL_SESSION, pppoe_add_del_session)                           \
+_(PPPOE_SESSION_DUMP, pppoe_session_dump)
+
+static void vl_api_pppoe_add_del_session_t_handler
+  (vl_api_pppoe_add_del_session_t * mp)
+{
+  vl_api_pppoe_add_del_session_reply_t *rmp;
+  int rv = 0;
+  u32 decap_fib_index;
+  ip4_main_t *im = &ip4_main;
+  pppoe_main_t *pem = &pppoe_main;
+
+  uword *p = hash_get (im->fib_index_by_table_id, ntohl (mp->decap_vrf_id));
+  if (!p)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_INNER_FIB;
+      goto out;
+    }
+  decap_fib_index = p[0];
+
+  vnet_pppoe_add_del_session_args_t a = {
+    .is_add = mp->is_add,
+    .is_ip6 = mp->is_ipv6,
+    .decap_fib_index = decap_fib_index,
+    .session_id = ntohs (mp->session_id),
+    .client_ip = to_ip46 (mp->is_ipv6, mp->client_ip),
+  };
+  clib_memcpy (a.client_mac, mp->client_mac, 6);
+
+  u32 sw_if_index = ~0;
+  rv = vnet_pppoe_add_del_session (&a, &sw_if_index);
+
+out:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2(VL_API_PPPOE_ADD_DEL_SESSION_REPLY,
+  ({
+    rmp->sw_if_index = ntohl (sw_if_index);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void send_pppoe_session_details
+  (pppoe_session_t * t, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_pppoe_session_details_t *rmp;
+  ip4_main_t *im4 = &ip4_main;
+  ip6_main_t *im6 = &ip6_main;
+  u8 is_ipv6 = !ip46_address_is_ip4 (&t->client_ip);
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_PPPOE_SESSION_DETAILS);
+  if (is_ipv6)
+    {
+      memcpy (rmp->client_ip, t->client_ip.ip6.as_u8, 16);
+      rmp->decap_vrf_id = htonl (im6->fibs[t->decap_fib_index].ft_table_id);
+    }
+  else
+    {
+      memcpy (rmp->client_ip, t->client_ip.ip4.as_u8, 4);
+      rmp->decap_vrf_id = htonl (im4->fibs[t->decap_fib_index].ft_table_id);
+    }
+  rmp->session_id = htons (t->session_id);
+  rmp->encap_if_index = htonl (t->encap_if_index);
+  clib_memcpy (rmp->local_mac, t->local_mac, 6);
+  clib_memcpy (rmp->client_mac, t->client_mac, 6);
+  rmp->sw_if_index = htonl (t->sw_if_index);
+  rmp->is_ipv6 = is_ipv6;
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_pppoe_session_dump_t_handler (vl_api_pppoe_session_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  pppoe_main_t *pem = &pppoe_main;
+  pppoe_session_t *t;
+  u32 sw_if_index;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    {
+      return;
+    }
+
+  sw_if_index = ntohl (mp->sw_if_index);
+
+  if (~0 == sw_if_index)
+    {
+      /* *INDENT-OFF* */
+      pool_foreach (t, pem->sessions,
+      ({
+        send_pppoe_session_details(t, q, mp->context);
+      }));
+      /* *INDENT-ON* */
+    }
+  else
+    {
+      if ((sw_if_index >= vec_len (pem->session_index_by_sw_if_index)) ||
+	  (~0 == pem->session_index_by_sw_if_index[sw_if_index]))
+	{
+	  return;
+	}
+      t = &pem->sessions[pem->session_index_by_sw_if_index[sw_if_index]];
+      send_pppoe_session_details (t, q, mp->context);
+    }
+}
+
+
+static clib_error_t *
+pppoe_api_hookup (vlib_main_t * vm)
+{
+  pppoe_main_t *pem = &pppoe_main;
+
+  u8 *name = format (0, "pppoe_%08x%c", api_version, 0);
+  pem->msg_id_base = vl_msg_api_get_msg_ids
+    ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + pem->msg_id_base),     \
+                           #n,                  \
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_pppoe_plugin_api_msg;
+#undef _
+
+  /* Add our API messages to the global name_crc hash table */
+  setup_message_id_table (pem, &api_main);
+
+  return 0;
+}
+
+VLIB_API_INIT_FUNCTION (pppoe_api_hookup);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/pppoe/pppoe_decap.c b/src/plugins/pppoe/pppoe_decap.c
new file mode 100644
index 00000000..02c82711
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_decap.c
@@ -0,0 +1,422 @@
+/*
+ * decap.c: pppoe session decap packet processing
+ *
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+typedef struct {
+  u32 next_index;
+  u32 session_index;
+  u32 session_id;
+  u32 error;
+} pppoe_rx_trace_t;
+
+static u8 * format_pppoe_rx_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  pppoe_rx_trace_t * t = va_arg (*args, pppoe_rx_trace_t *);
+
+  if (t->session_index != ~0)
+    {
+      s = format (s, "PPPoE decap from pppoe_session%d session_id %d next %d error %d",
+                  t->session_index, t->session_id, t->next_index, t->error);
+    }
+  else
+    {
+      s = format (s, "PPPoE decap error - session for session_id %d does not exist",
+		  t->session_id);
+    }
+  return s;
+}
+
+static uword
+pppoe_input (vlib_main_t * vm,
+             vlib_node_runtime_t * node,
+             vlib_frame_t * from_frame)
+{
+  u32 n_left_from, next_index, * from, * to_next;
+  pppoe_main_t * pem = &pppoe_main;
+  vnet_main_t * vnm = pem->vnet_main;
+  vnet_interface_main_t * im = &vnm->interface_main;
+  u32 pkts_decapsulated = 0;
+  u32 thread_index = vlib_get_thread_index();
+  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
+  pppoe_entry_key_t cached_key;
+  pppoe_entry_result_t cached_result;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  /* Clear the one-entry cache in case session table was updated */
+  cached_key.raw = ~0;
+  cached_result.raw = ~0;	/* warning be gone */
+
+  next_index = node->cached_next_index;
+  stats_sw_if_index = node->runtime_data[0];
+  stats_n_packets = stats_n_bytes = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+	  u32 next0, next1;
+	  ethernet_header_t *h0, *h1;
+          pppoe_header_t * pppoe0, * pppoe1;
+          u16 ppp_proto0 = 0, ppp_proto1 = 0;
+          pppoe_session_t * t0, * t1;
+          u32 error0, error1;
+	  u32 sw_if_index0, sw_if_index1, len0, len1;
+	  pppoe_entry_key_t key0, key1;
+	  pppoe_entry_result_t result0, result1;
+	  u32 bucket0, bucket1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	    CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	  }
+
+	  bi0 = from[0];
+	  bi1 = from[1];
+	  to_next[0] = bi0;
+	  to_next[1] = bi1;
+	  from += 2;
+	  to_next += 2;
+	  n_left_to_next -= 2;
+	  n_left_from -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+          error0 = 0;
+          error1 = 0;
+
+          /* leaves current_data pointing at the pppoe header */
+          pppoe0 = vlib_buffer_get_current (b0);
+          pppoe1 = vlib_buffer_get_current (b1);
+          ppp_proto0 = clib_net_to_host_u16(pppoe0->ppp_proto);
+          ppp_proto1 = clib_net_to_host_u16(pppoe1->ppp_proto);
+
+          /* Manipulate packet 0 */
+          if ((ppp_proto0 != PPP_PROTOCOL_ip4)
+             && (ppp_proto0 != PPP_PROTOCOL_ip6))
+            {
+	      error0 = PPPOE_ERROR_CONTROL_PLANE;
+	      next0 = PPPOE_INPUT_NEXT_CP_INPUT;
+	      goto trace0;
+            }
+
+          /* get client mac */
+          vlib_buffer_reset(b0);
+          h0 = vlib_buffer_get_current (b0);
+
+	  pppoe_lookup_1 (&pem->session_table, &cached_key, &cached_result,
+			  h0->src_address, pppoe0->session_id,
+			  &key0, &bucket0, &result0);
+          if (PREDICT_FALSE (result0.fields.session_index == ~0))
+	    {
+	      error0 = PPPOE_ERROR_NO_SUCH_SESSION;
+	      next0 = PPPOE_INPUT_NEXT_DROP;
+	      goto trace0;
+	    }
+
+	  t0 = pool_elt_at_index (pem->sessions,
+				  result0.fields.session_index);
+
+	  /* Pop Eth and PPPPoE header */
+	  vlib_buffer_advance(b0, sizeof(*h0)+sizeof(*pppoe0));
+
+	  next0 = (ppp_proto0==PPP_PROTOCOL_ip4)?
+		  PPPOE_INPUT_NEXT_IP4_INPUT
+		  : PPPOE_INPUT_NEXT_IP6_INPUT;
+
+          sw_if_index0 = t0->sw_if_index;
+          len0 = vlib_buffer_length_in_chain (vm, b0);
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same pppoe session so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = sw_if_index0;
+	    }
+
+        trace0:
+          b0->error = error0 ? node->errors[error0] : 0;
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_rx_trace_t *tr
+                = vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->next_index = next0;
+              tr->error = error0;
+              tr->session_index = result0.fields.session_index;
+              tr->session_id = clib_net_to_host_u32(pppoe0->session_id);
+            }
+
+
+          /* Manipulate packet 1 */
+          if ((ppp_proto1 != PPP_PROTOCOL_ip4)
+             && (ppp_proto1 != PPP_PROTOCOL_ip6))
+            {
+	      error1 = PPPOE_ERROR_CONTROL_PLANE;
+	      next1 = PPPOE_INPUT_NEXT_CP_INPUT;
+	      goto trace1;
+            }
+
+          /* get client mac */
+          vlib_buffer_reset(b1);
+          h1 = vlib_buffer_get_current (b1);
+
+	  pppoe_lookup_1 (&pem->session_table, &cached_key, &cached_result,
+			  h1->src_address, pppoe1->session_id,
+			  &key1, &bucket1, &result1);
+          if (PREDICT_FALSE (result1.fields.session_index == ~0))
+	    {
+	      error1 = PPPOE_ERROR_NO_SUCH_SESSION;
+	      next1 = PPPOE_INPUT_NEXT_DROP;
+	      goto trace1;
+	    }
+
+	  t1 = pool_elt_at_index (pem->sessions,
+				  result1.fields.session_index);
+
+	  /* Pop Eth and PPPPoE header */
+	  vlib_buffer_advance(b1, sizeof(*h1)+sizeof(*pppoe1));
+
+	  next1 = (ppp_proto1==PPP_PROTOCOL_ip4)?
+		  PPPOE_INPUT_NEXT_IP4_INPUT
+		  : PPPOE_INPUT_NEXT_IP6_INPUT;
+
+          sw_if_index1 = t1->sw_if_index;
+          len1 = vlib_buffer_length_in_chain (vm, b1);
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len1;
+
+	  /* Batch stats increment on the same pppoe session so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (sw_if_index1 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len1;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len1;
+	      stats_sw_if_index = sw_if_index1;
+	    }
+
+        trace1:
+          b1->error = error1 ? node->errors[error1] : 0;
+
+          if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_rx_trace_t *tr
+                = vlib_add_trace (vm, node, b1, sizeof (*tr));
+              tr->next_index = next1;
+              tr->error = error1;
+              tr->session_index = result1.fields.session_index;
+              tr->session_id = clib_net_to_host_u32(pppoe1->session_id);
+            }
+
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t * b0;
+	  u32 next0;
+	  ethernet_header_t *h0;
+          pppoe_header_t * pppoe0;
+          u16 ppp_proto0 = 0;
+          pppoe_session_t * t0;
+          u32 error0;
+	  u32 sw_if_index0, len0;
+	  pppoe_entry_key_t key0;
+	  pppoe_entry_result_t result0;
+	  u32 bucket0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  error0 = 0;
+
+          /* leaves current_data pointing at the pppoe header */
+          pppoe0 = vlib_buffer_get_current (b0);
+          ppp_proto0 = clib_net_to_host_u16(pppoe0->ppp_proto);
+
+          if ((ppp_proto0 != PPP_PROTOCOL_ip4)
+             && (ppp_proto0 != PPP_PROTOCOL_ip6))
+            {
+	      error0 = PPPOE_ERROR_CONTROL_PLANE;
+	      next0 = PPPOE_INPUT_NEXT_CP_INPUT;
+	      goto trace00;
+            }
+
+          /* get client mac */
+          vlib_buffer_reset(b0);
+          h0 = vlib_buffer_get_current (b0);
+
+	  pppoe_lookup_1 (&pem->session_table, &cached_key, &cached_result,
+			  h0->src_address, pppoe0->session_id,
+			  &key0, &bucket0, &result0);
+          if (PREDICT_FALSE (result0.fields.session_index == ~0))
+	    {
+	      error0 = PPPOE_ERROR_NO_SUCH_SESSION;
+	      next0 = PPPOE_INPUT_NEXT_DROP;
+	      goto trace00;
+	    }
+
+	  t0 = pool_elt_at_index (pem->sessions,
+				  result0.fields.session_index);
+
+	  /* Pop Eth and PPPPoE header */
+	  vlib_buffer_advance(b0, sizeof(*h0)+sizeof(*pppoe0));
+
+	  next0 = (ppp_proto0==PPP_PROTOCOL_ip4)?
+		  PPPOE_INPUT_NEXT_IP4_INPUT
+		  : PPPOE_INPUT_NEXT_IP6_INPUT;
+
+	  sw_if_index0 = t0->sw_if_index;
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same pppoe session so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = sw_if_index0;
+	    }
+
+        trace00:
+          b0->error = error0 ? node->errors[error0] : 0;
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_rx_trace_t *tr
+                = vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->next_index = next0;
+              tr->error = error0;
+              tr->session_index = result0.fields.session_index;
+              tr->session_id = clib_net_to_host_u16(pppoe0->session_id);
+            }
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  /* Do we still need this now that session tx stats is kept? */
+  vlib_node_increment_counter (vm, pppoe_input_node.index,
+                               PPPOE_ERROR_DECAPSULATED,
+                               pkts_decapsulated);
+
+  /* Increment any remaining batch stats */
+  if (stats_n_packets)
+    {
+      vlib_increment_combined_counter
+	(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+	 thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
+      node->runtime_data[0] = stats_sw_if_index;
+    }
+
+  return from_frame->n_vectors;
+}
+
+static char * pppoe_error_strings[] = {
+#define pppoe_error(n,s) s,
+#include 
+#undef pppoe_error
+#undef _
+};
+
+VLIB_REGISTER_NODE (pppoe_input_node) = {
+  .function = pppoe_input,
+  .name = "pppoe-input",
+  /* Takes a vector of packets. */
+  .vector_size = sizeof (u32),
+
+  .n_errors = PPPOE_N_ERROR,
+  .error_strings = pppoe_error_strings,
+
+  .n_next_nodes = PPPOE_INPUT_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [PPPOE_INPUT_NEXT_##s] = n,
+    foreach_pppoe_input_next
+#undef _
+  },
+
+  .format_trace = format_pppoe_rx_trace,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (pppoe_input_node, pppoe_input)
+
+
diff --git a/src/plugins/pppoe/pppoe_encap.c b/src/plugins/pppoe/pppoe_encap.c
new file mode 100644
index 00000000..69bec61d
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_encap.c
@@ -0,0 +1,384 @@
+/*
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Statistics (not all errors) */
+#define foreach_pppoe_encap_error    \
+_(ENCAPSULATED, "good packets encapsulated")
+
+static char * pppoe_encap_error_strings[] = {
+#define _(sym,string) string,
+  foreach_pppoe_encap_error
+#undef _
+};
+
+typedef enum {
+#define _(sym,str) PPPOE_ENCAP_ERROR_##sym,
+    foreach_pppoe_encap_error
+#undef _
+    PPPOE_ENCAP_N_ERROR,
+} pppoe_encap_error_t;
+
+#define foreach_pppoe_encap_next       \
+_(DROP, "error-drop")                  \
+_(INTERFACE, "interface-output" )      \
+
+typedef enum
+{
+#define _(s,n) PPPOE_ENCAP_NEXT_##s,
+  foreach_pppoe_encap_next
+#undef _
+    PPPOE_ENCAP_N_NEXT,
+} pppoe_encap_next_t;
+
+typedef struct {
+  u32 session_index;
+  u32 session_id;
+} pppoe_encap_trace_t;
+
+u8 * format_pppoe_encap_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  pppoe_encap_trace_t * t
+      = va_arg (*args, pppoe_encap_trace_t *);
+
+  s = format (s, "PPPOE encap to pppoe_session%d session_id %d",
+	      t->session_index, t->session_id);
+  return s;
+}
+
+
+#define foreach_fixed_header2_offset            \
+        _(0) _(1)
+
+
+static uword
+pppoe_encap (vlib_main_t * vm,
+	      vlib_node_runtime_t * node,
+	      vlib_frame_t * from_frame)
+{
+  u32 n_left_from, next_index, * from, * to_next;
+  pppoe_main_t * pem = &pppoe_main;
+  vnet_main_t * vnm = pem->vnet_main;
+  vnet_interface_main_t * im = &vnm->interface_main;
+  u32 pkts_encapsulated = 0;
+  u32 thread_index = vlib_get_thread_index();
+  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
+  u32 sw_if_index0 = 0, sw_if_index1 = 0;
+  u32 next0 = 0, next1 = 0;
+  pppoe_session_t * t0 = NULL, * t1 = NULL;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  next_index = node->cached_next_index;
+  stats_sw_if_index = node->runtime_data[0];
+  stats_n_packets = stats_n_bytes = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+	  u32 len0, len1;
+	  ethernet_header_t * eth0, * eth1;
+          pppoe_header_t * pppoe0, * pppoe1;
+          u64 * copy_src0, * copy_dst0;
+          u64 * copy_src1, * copy_dst1;
+          u16 * copy_src_last0, * copy_dst_last0;
+          u16 * copy_src_last1, * copy_dst_last1;
+          u16 new_l0, new_l1;
+          u32 session_id0, session_id1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	    CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
+	  }
+
+	  bi0 = from[0];
+	  bi1 = from[1];
+	  to_next[0] = bi0;
+	  to_next[1] = bi1;
+	  from += 2;
+	  to_next += 2;
+	  n_left_to_next -= 2;
+	  n_left_from -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+	  /* Get next node index and if-index from session */
+	  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX];
+	  session_id0 = pem->session_index_by_sw_if_index[sw_if_index0];
+	  t0 = pool_elt_at_index(pem->sessions, session_id0);
+	  next0 = PPPOE_ENCAP_NEXT_INTERFACE;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = t0->encap_if_index;
+
+          /* Get next node index and if-index from session */
+	  sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_TX];
+	  session_id1 = pem->session_index_by_sw_if_index[sw_if_index1];
+	  t1 = pool_elt_at_index(pem->sessions, session_id1);
+	  next1 = PPPOE_ENCAP_NEXT_INTERFACE;
+          vnet_buffer(b1)->sw_if_index[VLIB_TX] = t1->encap_if_index;
+
+          /* Apply the rewrite string. $$$$ vnet_rewrite? */
+          vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite));
+          vlib_buffer_advance (b1, -(word)_vec_len(t1->rewrite));
+
+          eth0 = (ethernet_header_t *)(vlib_buffer_get_current(b0));
+          eth1 = (ethernet_header_t *)(vlib_buffer_get_current(b1));
+
+	  /* Copy the fixed header */
+	  copy_dst0 = (u64 *) eth0;
+	  copy_src0 = (u64 *) t0->rewrite;
+	  copy_dst1 = (u64 *) eth1;
+	  copy_src1 = (u64 *) t1->rewrite;
+	  /* Copy first 8-bytes at a time */
+#define _(offs) copy_dst0[offs] = copy_src0[offs];
+	  foreach_fixed_header2_offset;
+#undef _
+	  /* Last 6 octets. Hopefully gcc will be our friend */
+          copy_dst_last0 = (u16 *)(©_dst0[2]);
+          copy_src_last0 = (u16 *)(©_src0[2]);
+          copy_dst_last0[0] = copy_src_last0[0];
+          copy_dst_last0[1] = copy_src_last0[1];
+          copy_dst_last0[2] = copy_src_last0[2];
+
+#define _(offs) copy_dst1[offs] = copy_src1[offs];
+	  foreach_fixed_header2_offset;
+#undef _
+	  /* Last 6 octets. Hopefully gcc will be our friend */
+          copy_dst_last1 = (u16 *)(©_dst1[2]);
+          copy_src_last1 = (u16 *)(©_src1[2]);
+          copy_dst_last1[0] = copy_src_last1[0];
+          copy_dst_last1[1] = copy_src_last1[1];
+          copy_dst_last1[2] = copy_src_last1[2];
+
+          /* Fix PPPoE length */
+	  new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					 - sizeof (*pppoe0) - sizeof(*eth0));
+	  pppoe0 = (pppoe_header_t *)(eth0 + 1);
+	  pppoe0->length = new_l0;
+
+	  new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1)
+					 - sizeof (*pppoe1) - sizeof(*eth1));
+	  pppoe1 = (pppoe_header_t *)(eth1 + 1);
+	  pppoe1->length = new_l1;
+
+          pkts_encapsulated += 2;
+ 	  len0 = vlib_buffer_length_in_chain (vm, b0);
+ 	  len1 = vlib_buffer_length_in_chain (vm, b1);
+	  stats_n_packets += 2;
+	  stats_n_bytes += len0 + len1;
+
+	  /* Batch stats increment on the same pppoe session so counter is not
+	     incremented per packet. Note stats are still incremented for deleted
+	     and admin-down session where packets are dropped. It is not worthwhile
+	     to check for this rare case and affect normal path performance. */
+	  if (PREDICT_FALSE ((sw_if_index0 != stats_sw_if_index) ||
+			     (sw_if_index1 != stats_sw_if_index)))
+	    {
+	      stats_n_packets -= 2;
+	      stats_n_bytes -= len0 + len1;
+	      if (sw_if_index0 == sw_if_index1)
+	        {
+		  if (stats_n_packets)
+		    vlib_increment_combined_counter
+		      (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		       thread_index, stats_sw_if_index,
+		       stats_n_packets, stats_n_bytes);
+		  stats_sw_if_index = sw_if_index0;
+		  stats_n_packets = 2;
+		  stats_n_bytes = len0 + len1;
+	        }
+	      else
+	        {
+		  vlib_increment_combined_counter
+		      (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		       thread_index, sw_if_index0, 1, len0);
+		  vlib_increment_combined_counter
+		      (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		       thread_index, sw_if_index1, 1, len1);
+		}
+	    }
+
+	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_encap_trace_t *tr =
+                vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->session_index = t0 - pem->sessions;
+              tr->session_id = t0->session_id;
+           }
+
+          if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_encap_trace_t *tr =
+                vlib_add_trace (vm, node, b1, sizeof (*tr));
+              tr->session_index = t1 - pem->sessions;
+              tr->session_id = t1->session_id;
+            }
+
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t * b0;
+	  ethernet_header_t * eth0;
+          pppoe_header_t * pppoe0;
+          u64 * copy_src0, * copy_dst0;
+          u16 * copy_src_last0, * copy_dst_last0;
+          u16 new_l0;
+          u32 len0;
+          u32 session_id0;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+	  /* Get next node index and if-index from session */
+	  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX];
+	  session_id0 = pem->session_index_by_sw_if_index[sw_if_index0];
+	  t0 = pool_elt_at_index(pem->sessions, session_id0);
+	  next0 = PPPOE_ENCAP_NEXT_INTERFACE;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = t0->encap_if_index;
+
+          /* Apply the rewrite string. $$$$ vnet_rewrite? */
+          vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite));
+
+          eth0 = (ethernet_header_t *)(vlib_buffer_get_current(b0));
+	  /* Copy the fixed header */
+	  copy_dst0 = (u64 *) eth0;
+	  copy_src0 = (u64 *) t0->rewrite;
+
+	  /* Copy first 8-bytes at a time */
+#define _(offs) copy_dst0[offs] = copy_src0[offs];
+	  foreach_fixed_header2_offset;
+#undef _
+	  /* Last 6 octets. Hopefully gcc will be our friend */
+          copy_dst_last0 = (u16 *)(©_dst0[2]);
+          copy_src_last0 = (u16 *)(©_src0[2]);
+          copy_dst_last0[0] = copy_src_last0[0];
+          copy_dst_last0[1] = copy_src_last0[1];
+          copy_dst_last0[2] = copy_src_last0[2];
+
+          /* Fix PPPoE length */
+	  new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0)
+					 - sizeof (*pppoe0) - sizeof(*eth0));
+	  pppoe0 = (pppoe_header_t *)(eth0 + 1);
+	  pppoe0->length = new_l0;
+
+          pkts_encapsulated ++;
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+	  stats_n_packets += 1;
+	  stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same pppoe session so counter is not
+	     incremented per packet. Note stats are still incremented for deleted
+	     and admin-down session where packets are dropped. It is not worthwhile
+	     to check for this rare case and affect normal path performance. */
+	  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = sw_if_index0;
+	    }
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_encap_trace_t *tr =
+                vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->session_index = t0 - pem->sessions;
+              tr->session_id = t0->session_id;
+            }
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  /* Do we still need this now that session tx stats is kept? */
+  vlib_node_increment_counter (vm, node->node_index,
+                               PPPOE_ENCAP_ERROR_ENCAPSULATED,
+                               pkts_encapsulated);
+
+  /* Increment any remaining batch stats */
+  if (stats_n_packets)
+    {
+      vlib_increment_combined_counter
+	(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX,
+	 thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
+      node->runtime_data[0] = stats_sw_if_index;
+    }
+
+  return from_frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (pppoe_encap_node) = {
+  .function = pppoe_encap,
+  .name = "pppoe-encap",
+  .vector_size = sizeof (u32),
+  .format_trace = format_pppoe_encap_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(pppoe_encap_error_strings),
+  .error_strings = pppoe_encap_error_strings,
+  .n_next_nodes = PPPOE_ENCAP_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [PPPOE_ENCAP_NEXT_##s] = n,
+    foreach_pppoe_encap_next
+#undef _
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (pppoe_encap_node, pppoe_encap)
+
diff --git a/src/plugins/pppoe/pppoe_error.def b/src/plugins/pppoe/pppoe_error.def
new file mode 100644
index 00000000..a875afd0
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_error.def
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+pppoe_error (DECAPSULATED, "good packets decapsulated")
+pppoe_error (CONTROL_PLANE, "control plane packet")
+pppoe_error (NO_SUCH_SESSION, "no such sessions")
+pppoe_error (BAD_VER_TYPE, "bad version and type in pppoe header")
diff --git a/src/plugins/pppoe/pppoe_msg_enum.h b/src/plugins/pppoe/pppoe_msg_enum.h
new file mode 100644
index 00000000..7ca19189
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_msg_enum.h
@@ -0,0 +1,31 @@
+/*
+ * pppoe_msg_enum.h - vpp engine plug-in message enumeration
+ *
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_pppoe_msg_enum_h
+#define included_pppoe_msg_enum_h
+
+#include 
+
+#define vl_msg_id(n,h) n,
+typedef enum
+{
+#include 
+  /* We'll want to know how many messages IDs we need... */
+  VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_pppoe_msg_enum_h */
diff --git a/src/plugins/pppoe/pppoe_tap.c b/src/plugins/pppoe/pppoe_tap.c
new file mode 100644
index 00000000..60cdaafb
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_tap.c
@@ -0,0 +1,89 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+
+static clib_error_t *
+pppoe_add_del_tap_command_fn (vlib_main_t * vm,
+			      unformat_input_t * input,
+			      vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  pppoe_main_t *pem = &pppoe_main;
+  u8 is_add = 1;
+  u8 tap_if_index_set = 0;
+  u32 tap_if_index = 0;
+  clib_error_t *error = NULL;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "del"))
+	{
+	  is_add = 0;
+	}
+      else if (unformat (line_input, "tap-if-index %d", &tap_if_index))
+	tap_if_index_set = 1;
+      else
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (tap_if_index_set == 0)
+    {
+      error = clib_error_return (0, "tap if index not specified");
+      goto done;
+    }
+
+  if (is_add)
+    {
+      pem->tap_if_index = tap_if_index;
+    }
+  else
+    {
+      pem->tap_if_index = ~0;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (create_pppoe_tap_cmd, static) =
+{
+    .path = "create pppoe tap",
+    .short_help = "create pppoe tap if-name  [del]",
+    .function = pppoe_add_del_tap_command_fn,
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/pppoe/pppoe_tap_node.c b/src/plugins/pppoe/pppoe_tap_node.c
new file mode 100644
index 00000000..f1e0a501
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_tap_node.c
@@ -0,0 +1,297 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or pemplied.
+ * See the License for the specific language governing permissions and
+ * lpemitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include 
+#include 
+#include 
+
+vlib_node_registration_t pppoe_tap_dispatch_node;
+
+#define foreach_pppoe_tap_next        \
+_(DROP, "error-drop")                  \
+_(TUNTAP, "tuntap-tx" )                \
+_(INTERFACE, "interface-output" )      \
+
+typedef enum
+{
+#define _(s,n) PPPOE_TAP_NEXT_##s,
+  foreach_pppoe_tap_next
+#undef _
+    PPPOE_TAP_N_NEXT,
+} pppoe_tap_next_t;
+
+typedef struct {
+  u32 next_index;
+  u32 sw_if_index;
+  u32 tap_if_index;
+  u8 pppoe_code;
+  u16 ppp_proto;
+  u32 error;
+} pppoe_tap_trace_t;
+
+static u8 * format_pppoe_tap_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  pppoe_tap_trace_t * t = va_arg (*args, pppoe_tap_trace_t *);
+  pppoe_main_t * pem = &pppoe_main;
+
+  if (t->sw_if_index != pem->tap_if_index)
+    {
+      s = format (s, "PPPoE dispatch from sw_if_index %d next %d error %d \n"
+	          "  pppoe_code 0x%x  ppp_proto 0x%x",
+                  t->sw_if_index, t->next_index, t->error,
+		  t->pppoe_code, t->ppp_proto);
+    }
+  else
+    {
+      s = format (s, "PPPoE dispatch from tap_if_index %d next %d error %d \n"
+	          "  pppoe_code 0x%x  ppp_proto 0x%x",
+                  t->tap_if_index, t->next_index, t->error,
+		  t->pppoe_code, t->ppp_proto);
+    }
+  return s;
+}
+
+/**
+ * Perform learning on one packet based on the mac table lookup result.
+ * */
+static_always_inline void
+pppoe_learn_process (vlib_node_runtime_t * node,
+		     pppoe_main_t * pem,
+		     vlib_buffer_t * b0,
+		     u32 sw_if_index0,
+		     pppoe_entry_key_t * key0,
+		     pppoe_entry_key_t * cached_key,
+		     u32 * bucket0,
+		     pppoe_entry_result_t * result0)
+{
+  /* Check mac table lookup result */
+  if (PREDICT_TRUE (result0->fields.sw_if_index == sw_if_index0))
+    {
+      /*
+       * The entry was in the table, and the sw_if_index matched, the normal case
+       */
+      return;
+    }
+  else if (result0->fields.sw_if_index == ~0)
+    {
+      /* The entry was not in table, so add it  */
+      result0->fields.sw_if_index = sw_if_index0;
+      result0->fields.session_index = ~0;
+      cached_key->raw = ~0;	/* invalidate the cache */
+    }
+  else
+    {
+      /* The entry was in the table, but with the wrong sw_if_index mapping (mac move) */
+      result0->fields.sw_if_index = sw_if_index0;
+    }
+
+  /* Update the entry */
+  BVT (clib_bihash_kv) kv;
+  kv.key = key0->raw;
+  kv.value = result0->raw;
+  BV (clib_bihash_add_del) (&pem->session_table, &kv, 1 /* is_add */ );
+}
+
+static uword
+pppoe_tap_dispatch (vlib_main_t * vm,
+                    vlib_node_runtime_t * node,
+                    vlib_frame_t * from_frame)
+{
+  u32 n_left_from, next_index, * from, * to_next;
+  pppoe_main_t * pem = &pppoe_main;
+  vnet_main_t * vnm = pem->vnet_main;
+  vnet_interface_main_t * im = &vnm->interface_main;
+  u32 pkts_decapsulated = 0;
+  u32 thread_index = vlib_get_thread_index();
+  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
+  pppoe_entry_key_t cached_key;
+  pppoe_entry_result_t cached_result;
+
+  from = vlib_frame_vector_args (from_frame);
+  n_left_from = from_frame->n_vectors;
+
+  /* Clear the one-entry cache in case session table was updated */
+  cached_key.raw = ~0;
+  cached_result.raw = ~0;	/* warning be gone */
+
+  next_index = node->cached_next_index;
+  stats_sw_if_index = node->runtime_data[0];
+  stats_n_packets = stats_n_bytes = 0;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t * b0;
+	  ethernet_header_t *h0;
+	  pppoe_header_t * pppoe0;
+	  pppoe_entry_key_t key0;
+	  pppoe_entry_result_t result0;
+
+	  u32 bucket0;
+	  u32 next0;
+          u32 error0 = 0;
+	  u32 rx_sw_if_index0=~0, tx_sw_if_index0=~0, len0;
+	  vnet_hw_interface_t *hi;
+	  vnet_sw_interface_t *si;
+
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+          /* leaves current_data pointing at the pppoe header */
+          pppoe0 = vlib_buffer_get_current (b0);
+          rx_sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE (pppoe0->ver_type != PPPOE_VER_TYPE))
+	    {
+	      error0 = PPPOE_ERROR_BAD_VER_TYPE;
+	      next0 = PPPOE_INPUT_NEXT_DROP;
+	      goto trace00;
+	    }
+
+          vlib_buffer_reset(b0);
+          h0 = vlib_buffer_get_current (b0);
+
+          if(rx_sw_if_index0 == pem->tap_if_index)
+            {
+    	      pppoe_lookup_1 (&pem->session_table, &cached_key, &cached_result,
+    			      h0->dst_address, 0,
+    			      &key0, &bucket0, &result0);
+    	      tx_sw_if_index0 = result0.fields.sw_if_index;
+
+              if (PREDICT_FALSE (tx_sw_if_index0 == ~0))
+    	        {
+    	          error0 = PPPOE_ERROR_NO_SUCH_SESSION;
+    	          next0 = PPPOE_INPUT_NEXT_DROP;
+    	          goto trace00;
+    	        }
+
+              next0 = PPPOE_TAP_NEXT_INTERFACE;
+              vnet_buffer(b0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
+
+              /* set src mac address */
+              si = vnet_get_sw_interface(vnm, tx_sw_if_index0);
+              hi = vnet_get_hw_interface (vnm, si->hw_if_index);
+              clib_memcpy (vlib_buffer_get_current (b0)+6, hi->hw_address, 6);
+            }
+          else
+            {
+    	      pppoe_lookup_1 (&pem->session_table, &cached_key, &cached_result,
+    			      h0->src_address, pppoe0->session_id,
+    			      &key0, &bucket0, &result0);
+    	      tx_sw_if_index0 = result0.fields.sw_if_index;
+
+              /* learn client session */
+    	      pppoe_learn_process (node, pem, b0, rx_sw_if_index0,
+				   &key0, &cached_key,
+    			           &bucket0, &result0);
+
+              next0 = PPPOE_TAP_NEXT_TUNTAP;
+              vnet_buffer(b0)->sw_if_index[VLIB_TX] = pem->tap_if_index;
+            }
+
+	  len0 = vlib_buffer_length_in_chain (vm, b0);
+
+          pkts_decapsulated ++;
+          stats_n_packets += 1;
+          stats_n_bytes += len0;
+
+	  /* Batch stats increment on the same pppoe session so counter
+	     is not incremented per packet */
+	  if (PREDICT_FALSE (rx_sw_if_index0 != stats_sw_if_index))
+	    {
+	      stats_n_packets -= 1;
+	      stats_n_bytes -= len0;
+	      if (stats_n_packets)
+		vlib_increment_combined_counter
+		  (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+		   thread_index, stats_sw_if_index,
+		   stats_n_packets, stats_n_bytes);
+	      stats_n_packets = 1;
+	      stats_n_bytes = len0;
+	      stats_sw_if_index = rx_sw_if_index0;
+	    }
+        
+        trace00:  
+          b0->error = error0 ? node->errors[error0] : 0;
+
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              pppoe_tap_trace_t *tr
+                = vlib_add_trace (vm, node, b0, sizeof (*tr));
+              tr->next_index = next0;
+              tr->error = error0;
+              tr->sw_if_index = tx_sw_if_index0;
+              tr->tap_if_index = pem->tap_if_index;
+              tr->pppoe_code = pppoe0->code;
+              tr->ppp_proto = clib_net_to_host_u16(pppoe0->ppp_proto);
+            }
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  /* Do we still need this now that session tx stats is kept? */
+  vlib_node_increment_counter (vm, pppoe_input_node.index,
+                               PPPOE_ERROR_DECAPSULATED,
+                               pkts_decapsulated);
+
+  /* Increment any remaining batch stats */
+  if (stats_n_packets)
+    {
+      vlib_increment_combined_counter
+	(im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX,
+	 thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
+      node->runtime_data[0] = stats_sw_if_index;
+    }
+
+  return from_frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (pppoe_tap_dispatch_node) = {
+  .function = pppoe_tap_dispatch,
+  .name = "pppoe-tap-dispatch",
+  /* Takes a vector of packets. */
+  .vector_size = sizeof (u32),
+
+  .n_next_nodes = PPPOE_TAP_N_NEXT,
+  .next_nodes = {
+#define _(s,n) [PPPOE_TAP_NEXT_##s] = n,
+    foreach_pppoe_tap_next
+#undef _
+  },
+
+  .format_trace = format_pppoe_tap_trace,
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (pppoe_tap_dispatch_node, pppoe_tap_dispatch)
+
diff --git a/src/plugins/pppoe/pppoe_test.c b/src/plugins/pppoe/pppoe_test.c
new file mode 100644
index 00000000..2b67d989
--- /dev/null
+++ b/src/plugins/pppoe/pppoe_test.c
@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define __plugin_msg_base pppoe_test_main.msg_id_base
+#include 
+
+
+uword unformat_ip46_address (unformat_input_t * input, va_list * args)
+{
+  ip46_address_t *ip46 = va_arg (*args, ip46_address_t *);
+  ip46_type_t type = va_arg (*args, ip46_type_t);
+  if ((type != IP46_TYPE_IP6) &&
+      unformat(input, "%U", unformat_ip4_address, &ip46->ip4)) {
+    ip46_address_mask_ip4(ip46);
+    return 1;
+  } else if ((type != IP46_TYPE_IP4) &&
+      unformat(input, "%U", unformat_ip6_address, &ip46->ip6)) {
+    return 1;
+  }
+  return 0;
+}
+uword unformat_ip46_prefix (unformat_input_t * input, va_list * args)
+{
+  ip46_address_t *ip46 = va_arg (*args, ip46_address_t *);
+  u8 *len = va_arg (*args, u8 *);
+  ip46_type_t type = va_arg (*args, ip46_type_t);
+
+  u32 l;
+  if ((type != IP46_TYPE_IP6) && unformat(input, "%U/%u", unformat_ip4_address, &ip46->ip4, &l)) {
+    if (l > 32)
+      return 0;
+    *len = l + 96;
+    ip46->pad[0] = ip46->pad[1] = ip46->pad[2] = 0;
+  } else if ((type != IP46_TYPE_IP4) && unformat(input, "%U/%u", unformat_ip6_address, &ip46->ip6, &l)) {
+    if (l > 128)
+      return 0;
+    *len = l;
+  } else {
+    return 0;
+  }
+  return 1;
+}
+/////////////////////////
+
+#define vl_msg_id(n,h) n,
+typedef enum {
+#include 
+    /* We'll want to know how many messages IDs we need... */
+    VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+
+#define vl_endianfun             /* define message structures */
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+typedef struct {
+    /* API message ID base */
+    u16 msg_id_base;
+    vat_main_t *vat_main;
+} pppoe_test_main_t;
+
+pppoe_test_main_t pppoe_test_main;
+
+static void vl_api_pppoe_add_del_session_reply_t_handler
+  (vl_api_pppoe_add_del_session_reply_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+  i32 retval = ntohl (mp->retval);
+  if (vam->async_mode)
+    {
+      vam->async_errors += (retval < 0);
+    }
+  else
+    {
+      vam->retval = retval;
+      vam->sw_if_index = ntohl (mp->sw_if_index);
+      vam->result_ready = 1;
+    }
+}
+
+
+/*
+ * Table of message reply handlers, must include boilerplate handlers
+ * we just generated
+ */
+#define foreach_vpe_api_reply_msg                               \
+  _(PPPOE_ADD_DEL_SESSION_REPLY, pppoe_add_del_session_reply)               \
+  _(PPPOE_SESSION_DETAILS, pppoe_session_details)
+
+
+static int
+api_pppoe_add_del_session (vat_main_t * vam)
+{
+  unformat_input_t *line_input = vam->input;
+  vl_api_pppoe_add_del_session_t *mp;
+  u16 session_id = 0;
+  ip46_address_t client_ip;
+  u8 is_add = 1;
+  u8 client_ip_set = 0;
+  u8 ipv4_set = 0;
+  u8 ipv6_set = 0;
+  u32 decap_vrf_id = 0;
+  u8 client_mac[6] = { 0 };
+  u8 client_mac_set = 0;
+  int ret;
+
+  /* Can't "universally zero init" (={0}) due to GCC bug 53119 */
+  memset (&client_ip, 0, sizeof client_ip);
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "del"))
+	{
+	  is_add = 0;
+	}
+      else if (unformat (line_input, "session_id %d", &session_id))
+	;
+      else if (unformat (line_input, "client-ip %U",
+			 unformat_ip4_address, &client_ip.ip4))
+	{
+	  client_ip_set = 1;
+	  ipv4_set = 1;
+	}
+      else if (unformat (line_input, "client-ip %U",
+			 unformat_ip6_address, &client_ip.ip6))
+	{
+	  client_ip_set = 1;
+	  ipv6_set = 1;
+	}
+      else if (unformat (line_input, "decap-vrf-id %d", &decap_vrf_id))
+        ;
+      else if (unformat (line_input, "client-mac %U", unformat_ethernet_address, client_mac))
+	client_mac_set = 1;
+      else
+	{
+	  return -99;
+	}
+    }
+
+  if (client_ip_set == 0)
+    {
+      errmsg ("session client_ip address not specified");
+      return -99;
+    }
+
+  if (ipv4_set && ipv6_set)
+    {
+      errmsg ("both IPv4 and IPv6 addresses specified");
+      return -99;
+    }
+
+  if (client_mac_set == 0)
+    {
+      errmsg("session client mac not specified");
+      return -99;
+    }
+
+  M (PPPOE_ADD_DEL_SESSION, mp);
+
+  if (ipv6_set)
+    {
+      clib_memcpy (mp->client_ip, &client_ip.ip6, sizeof (client_ip.ip6));
+    }
+  else
+    {
+      clib_memcpy (mp->client_ip, &client_ip.ip4, sizeof (client_ip.ip4));
+    }
+
+  mp->decap_vrf_id = ntohl (decap_vrf_id);
+  mp->session_id = ntohl (session_id);
+  mp->is_add = is_add;
+  mp->is_ipv6 = ipv6_set;
+  memcpy (mp->client_mac, client_mac, 6);
+
+  S (mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_pppoe_session_details_t_handler
+  (vl_api_pppoe_session_details_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+  ip46_address_t client_ip = to_ip46 (mp->is_ipv6, mp->client_ip);
+
+  print (vam->ofp, "%11d%14d%24U%14d%14d%30U%30U",
+       ntohl (mp->sw_if_index), ntohl (mp->session_id),
+       format_ip46_address, &client_ip, IP46_TYPE_ANY,
+       ntohl (mp->encap_if_index), ntohl (mp->decap_vrf_id),
+       format_ethernet_address, mp->local_mac,
+       format_ethernet_address, mp->client_mac);
+}
+
+static int
+api_pppoe_session_dump (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_pppoe_session_dump_t *mp;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "sw_if_index %d", &sw_if_index))
+      sw_if_index_set = 1;
+      else
+      break;
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      sw_if_index = ~0;
+    }
+
+  if (!vam->json_output)
+    {
+      print (vam->ofp, "%11s%24s%14s%14s%14s",
+	   "sw_if_index", "client_ip", "session_id",
+	   "encap_if_index", "decap_fib_index",
+	   "local-mac", "client-mac");
+    }
+
+  /* Get list of pppoe-session interfaces */
+  M (PPPOE_SESSION_DUMP, mp);
+
+  mp->sw_if_index = htonl (sw_if_index);
+
+  S (mp);
+
+  W (ret);
+  return ret;
+}
+
+/*
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg                                            \
+_(pppoe_add_del_session,                                                 \
+  " client-addr  session-id "                            \
+  " [encap-if-index ] [decap-next [ip4|ip6|node ]]"             \
+  " local-mac  client-mac  [del]") \
+_(pppoe_session_dump, "[ | sw_if_index ]")                    \
+
+static void
+pppoe_vat_api_hookup (vat_main_t *vam)
+{
+  pppoe_test_main_t * pem = &pppoe_test_main;
+  /* Hook up handlers for replies from the data plane plug-in */
+#define _(N,n)                                                  \
+  vl_msg_api_set_handlers((VL_API_##N + pem->msg_id_base),       \
+                          #n,                                   \
+                          vl_api_##n##_t_handler,               \
+                          vl_noop_handler,                      \
+                          vl_api_##n##_t_endian,                \
+                          vl_api_##n##_t_print,                 \
+                          sizeof(vl_api_##n##_t), 1);
+  foreach_vpe_api_reply_msg;
+#undef _
+
+  /* API messages we can send */
+#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _
+
+  /* Help strings */
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t * vat_plugin_register (vat_main_t *vam)
+{
+  pppoe_test_main_t * pem = &pppoe_test_main;
+
+  u8 * name;
+
+  pem->vat_main = vam;
+
+  /* Ask the vpp engine for the first assigned message-id */
+  name = format (0, "pppoe_%08x%c", api_version, 0);
+  pem->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (pem->msg_id_base != (u16) ~0)
+    pppoe_vat_api_hookup (vam);
+
+  vec_free(name);
+
+  return 0;
+}
diff --git a/src/vpp-api/java/Makefile.am b/src/vpp-api/java/Makefile.am
index 4f1f7ed2..e0f5203d 100644
--- a/src/vpp-api/java/Makefile.am
+++ b/src/vpp-api/java/Makefile.am
@@ -148,6 +148,26 @@ jvpp-gtpu/io_fd_vpp_jvpp_gtpu_JVppGtpuImpl.h: $(jvpp_registry_ok) $(jvpp_gtpu_js
 	$(call japigen,gtpu,JVppGtpuImpl)
 endif
 
+#
+# PPPOE Plugin
+#
+if ENABLE_PPPOE_PLUGIN
+noinst_LTLIBRARIES += libjvpp_pppoe.la
+libjvpp_pppoe_la_SOURCES = jvpp-pppoe/jvpp_pppoe.c
+libjvpp_pppoe_la_CPPFLAGS = -Ijvpp-pppoe
+libjvpp_pppoe_la_LIBADD = $(JVPP_LIBS)
+libjvpp_pppoe_la_DEPENDENCIES = libjvpp_common.la
+
+BUILT_SOURCES += jvpp-pppoe/io_fd_vpp_jvpp_pppoe_JVppPppoeImpl.h
+JAR_FILES += jvpp-pppoe-$(PACKAGE_VERSION).jar
+CLEANDIRS += jvpp-pppoe/target
+
+jvpp_pppoe_json_files = @top_builddir@/plugins/pppoe/pppoe.api.json
+
+jvpp-pppoe/io_fd_vpp_jvpp_pppoe_JVppPppoeImpl.h: $(jvpp_registry_ok) $(jvpp_pppoe_json_files)
+	$(call japigen,pppoe,JVppPppoeImpl)
+endif
+
 #
 # SNAT Plugin
 #
diff --git a/src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.c b/src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.c
new file mode 100644
index 00000000..c9c30305
--- /dev/null
+++ b/src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-pppoe/io_fd_vpp_jvpp_pppoe_JVppPppoeImpl.h"
+#include "jvpp_pppoe.h"
+#include "jvpp-pppoe/jvpp_pppoe_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_pppoe_JVpppppoeImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_pppoe_JVppPppoeImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  pppoe_main_t * plugin_main = &pppoe_main;
+  clib_warning ("Java_io_fd_vpp_jvpp_pppoe_JVppPppoeImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = (unix_shared_memory_queue_t *)queue_address;
+
+  plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+  plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+  // verify API has not changed since jar generation
+  #define _(N)             \
+      get_message_id(env, #N);  \
+      foreach_supported_api_message;
+  #undef _
+
+  #define _(N,n)                                  \
+      vl_msg_api_set_handlers(get_message_id(env, #N), #n,     \
+              vl_api_##n##_t_handler,             \
+              vl_noop_handler,                    \
+              vl_noop_handler,                    \
+              vl_noop_handler,                    \
+              sizeof(vl_api_##n##_t), 1);
+      foreach_api_reply_handler;
+  #undef _
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_pppoe_JVppPppoeImpl_close0
+(JNIEnv *env, jclass clazz) {
+  pppoe_main_t * plugin_main = &pppoe_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP ACL */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP ACL */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.h b/src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.h
new file mode 100644
index 00000000..4523ba9c
--- /dev/null
+++ b/src/vpp-api/java/jvpp-pppoe/jvpp_pppoe.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_pppoe_h__
+#define __included_jvpp_pppoe_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-pppoe */
+typedef struct {
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} pppoe_main_t;
+
+pppoe_main_t pppoe_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_pppoe_h__ */
diff --git a/test/test_pppoe.py b/test/test_pppoe.py
new file mode 100644
index 00000000..0baf4546
--- /dev/null
+++ b/test/test_pppoe.py
@@ -0,0 +1,605 @@
+#!/usr/bin/env python
+
+import unittest
+from logging import *
+
+from framework import VppTestCase, VppTestRunner
+from vpp_ip_route import VppIpRoute, VppRoutePath
+from vpp_pppoe_interface import VppPppoeInterface, VppPppoe6Interface
+from vpp_papi_provider import L2_VTR_OP
+
+from scapy.packet import Raw
+from scapy.layers.l2 import Ether
+from scapy.layers.ppp import PPPoE, PPPoED, PPP
+from scapy.layers.inet import IP, UDP
+from scapy.layers.inet6 import IPv6
+from scapy.volatile import RandMAC, RandIP
+
+from util import ppp, ppc, mactobinary
+import socket
+
+
+class TestPPPoE(VppTestCase):
+    """ PPPoE Test Case """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestPPPoE, cls).setUpClass()
+
+        cls.session_id = 1
+        cls.dst_ip = "100.1.1.100"
+        cls.dst_ipn = socket.inet_pton(socket.AF_INET, cls.dst_ip)
+
+    def setUp(self):
+        super(TestPPPoE, self).setUp()
+
+        # create 2 pg interfaces
+        self.create_pg_interfaces(range(3))
+
+        for i in self.pg_interfaces:
+            i.admin_up()
+            i.config_ip4()
+            i.resolve_arp()
+
+    def tearDown(self):
+        super(TestPPPoE, self).tearDown()
+
+        self.logger.info(self.vapi.cli("show int"))
+        self.logger.info(self.vapi.cli("show pppoe fib"))
+        self.logger.info(self.vapi.cli("show pppoe session"))
+        self.logger.info(self.vapi.cli("show ip fib"))
+        self.logger.info(self.vapi.cli("show trace"))
+
+        for i in self.pg_interfaces:
+            i.unconfig_ip4()
+            i.admin_down()
+
+    def create_stream_pppoe_discovery(self, src_if, dst_if,
+                                      client_mac, count=1):
+        packets = []
+        for i in range(count):
+            # create packet info stored in the test case instance
+            info = self.create_packet_info(src_if, dst_if)
+            # convert the info into packet payload
+            payload = self.info_to_payload(info)
+            # create the packet itself
+            p = (Ether(dst=src_if.local_mac, src=client_mac) /
+                 PPPoED(sessionid=0) /
+                 Raw(payload))
+            # store a copy of the packet in the packet info
+            info.data = p.copy()
+            # append the packet to the list
+            packets.append(p)
+
+        # return the created packet list
+        return packets
+
+    def create_stream_pppoe_lcp(self, src_if, dst_if,
+                                client_mac, session_id, count=1):
+        packets = []
+        for i in range(count):
+            # create packet info stored in the test case instance
+            info = self.create_packet_info(src_if, dst_if)
+            # convert the info into packet payload
+            payload = self.info_to_payload(info)
+            # create the packet itself
+            p = (Ether(dst=src_if.local_mac, src=client_mac) /
+                 PPPoE(sessionid=session_id) /
+                 PPP(proto=0xc021) /
+                 Raw(payload))
+            # store a copy of the packet in the packet info
+            info.data = p.copy()
+            # append the packet to the list
+            packets.append(p)
+
+        # return the created packet list
+        return packets
+
+    def create_stream_pppoe_ip4(self, src_if, dst_if,
+                                client_mac, session_id, client_ip, count=1):
+        packets = []
+        for i in range(count):
+            # create packet info stored in the test case instance
+            info = self.create_packet_info(src_if, dst_if)
+            # convert the info into packet payload
+            payload = self.info_to_payload(info)
+            # create the packet itself
+            p = (Ether(dst=src_if.local_mac, src=client_mac) /
+                 PPPoE(sessionid=session_id) /
+                 PPP(proto=0x0021) /
+                 IP(src=client_ip, dst=self.dst_ip) /
+                 Raw(payload))
+            # store a copy of the packet in the packet info
+            info.data = p.copy()
+            # append the packet to the list
+            packets.append(p)
+
+        # return the created packet list
+        return packets
+
+    def create_stream_ip4(self, src_if, dst_if, client_ip, dst_ip, count=1):
+        pkts = []
+        for i in range(count):
+            # create packet info stored in the test case instance
+            info = self.create_packet_info(src_if, dst_if)
+            # convert the info into packet payload
+            payload = self.info_to_payload(info)
+            # create the packet itself
+            p = (Ether(dst=src_if.local_mac, src=src_if.remote_mac) /
+                 IP(src=dst_ip, dst=client_ip) /
+                 Raw(payload))
+            # store a copy of the packet in the packet info
+            info.data = p.copy()
+            # append the packet to the list
+            pkts.append(p)
+
+        # return the created packet list
+        return pkts
+
+    def verify_decapped_pppoe(self, src_if, capture, sent):
+        self.assertEqual(len(capture), len(sent))
+
+        for i in range(len(capture)):
+            try:
+                tx = sent[i]
+                rx = capture[i]
+
+                tx_ip = tx[IP]
+                rx_ip = rx[IP]
+
+                self.assertEqual(rx_ip.src, tx_ip.src)
+                self.assertEqual(rx_ip.dst, tx_ip.dst)
+
+            except:
+                self.logger.error(ppp("Rx:", rx))
+                self.logger.error(ppp("Tx:", tx))
+                raise
+
+    def verify_encaped_pppoe(self, src_if, capture, sent, session_id):
+
+        self.assertEqual(len(capture), len(sent))
+
+        for i in range(len(capture)):
+            try:
+                tx = sent[i]
+                rx = capture[i]
+
+                tx_ip = tx[IP]
+                rx_ip = rx[IP]
+
+                self.assertEqual(rx_ip.src, tx_ip.src)
+                self.assertEqual(rx_ip.dst, tx_ip.dst)
+
+                rx_pppoe = rx[PPPoE]
+
+                self.assertEqual(rx_pppoe.sessionid, session_id)
+
+            except:
+                self.logger.error(ppp("Rx:", rx))
+                self.logger.error(ppp("Tx:", tx))
+                raise
+
+    def test_PPPoE_Decap(self):
+        """ PPPoE Decap Test """
+
+        self.vapi.cli("clear trace")
+
+        #
+        # Add a route that resolves the server's destination
+        #
+        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
+                                     [VppRoutePath(self.pg1.remote_ip4,
+                                                   self.pg1.sw_if_index)])
+        route_sever_dst.add_vpp_config()
+
+        # Send PPPoE Discovery
+        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
+                                                 self.pg0.remote_mac)
+        self.pg0.add_stream(tx0)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP
+        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id)
+        self.pg0.add_stream(tx1)
+        self.pg_start()
+
+        # Create PPPoE session
+        pppoe_if = VppPppoeInterface(self,
+                                     self.pg0.remote_ip4,
+                                     self.pg0.remote_mac,
+                                     self.session_id)
+        pppoe_if.add_vpp_config()
+
+        #
+        # Send tunneled packets that match the created tunnel and
+        # are decapped and forwarded
+        #
+        tx2 = self.create_stream_pppoe_ip4(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id,
+                                           self.pg0.remote_ip4)
+        self.pg0.add_stream(tx2)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx2 = self.pg1.get_capture(len(tx2))
+        self.verify_decapped_pppoe(self.pg0, rx2, tx2)
+
+        self.logger.info(self.vapi.cli("show pppoe fib"))
+        self.logger.info(self.vapi.cli("show pppoe session"))
+        self.logger.info(self.vapi.cli("show ip fib"))
+
+        #
+        # test case cleanup
+        #
+
+        # Delete PPPoE session
+        pppoe_if.remove_vpp_config()
+
+        # Delete a route that resolves the server's destination
+        route_sever_dst.remove_vpp_config()
+
+    def test_PPPoE_Encap(self):
+        """ PPPoE Encap Test """
+
+        self.vapi.cli("clear trace")
+
+        #
+        # Add a route that resolves the server's destination
+        #
+        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
+                                     [VppRoutePath(self.pg1.remote_ip4,
+                                                   self.pg1.sw_if_index)])
+        route_sever_dst.add_vpp_config()
+
+        # Send PPPoE Discovery
+        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
+                                                 self.pg0.remote_mac)
+        self.pg0.add_stream(tx0)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP
+        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id)
+        self.pg0.add_stream(tx1)
+        self.pg_start()
+
+        # Create PPPoE session
+        pppoe_if = VppPppoeInterface(self,
+                                     self.pg0.remote_ip4,
+                                     self.pg0.remote_mac,
+                                     self.session_id)
+        pppoe_if.add_vpp_config()
+
+        #
+        # Send a packet stream that is routed into the session
+        #  - packets are PPPoE encapped
+        #
+        self.vapi.cli("clear trace")
+        tx2 = self.create_stream_ip4(self.pg1, self.pg0,
+                                     self.pg0.remote_ip4, self.dst_ip)
+        self.pg1.add_stream(tx2)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx2 = self.pg0.get_capture(len(tx2))
+        self.verify_encaped_pppoe(self.pg1, rx2, tx2, self.session_id)
+
+        self.logger.info(self.vapi.cli("show pppoe fib"))
+        self.logger.info(self.vapi.cli("show pppoe session"))
+        self.logger.info(self.vapi.cli("show ip fib"))
+
+        #
+        # test case cleanup
+        #
+
+        # Delete PPPoE session
+        pppoe_if.remove_vpp_config()
+
+        # Delete a route that resolves the server's destination
+        route_sever_dst.remove_vpp_config()
+
+    def test_PPPoE_Add_Twice(self):
+        """ PPPoE Add Same Session Twice Test """
+
+        self.vapi.cli("clear trace")
+
+        #
+        # Add a route that resolves the server's destination
+        #
+        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
+                                     [VppRoutePath(self.pg1.remote_ip4,
+                                                   self.pg1.sw_if_index)])
+        route_sever_dst.add_vpp_config()
+
+        # Send PPPoE Discovery
+        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
+                                                 self.pg0.remote_mac)
+        self.pg0.add_stream(tx0)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP
+        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id)
+        self.pg0.add_stream(tx1)
+        self.pg_start()
+
+        # Create PPPoE session
+        pppoe_if = VppPppoeInterface(self,
+                                     self.pg0.remote_ip4,
+                                     self.pg0.remote_mac,
+                                     self.session_id)
+        pppoe_if.add_vpp_config()
+
+        #
+        # The double create (create the same session twice) should fail,
+        # and we should still be able to use the original
+        #
+        try:
+            gre_if.add_vpp_config()
+        except Exception:
+            pass
+        else:
+            self.fail("Double GRE tunnel add does not fail")
+
+        #
+        # test case cleanup
+        #
+
+        # Delete PPPoE session
+        pppoe_if.remove_vpp_config()
+
+        # Delete a route that resolves the server's destination
+        route_sever_dst.remove_vpp_config()
+
+    def test_PPPoE_Del_Twice(self):
+        """ PPPoE Delete Same Session Twice Test """
+
+        self.vapi.cli("clear trace")
+
+        #
+        # Add a route that resolves the server's destination
+        #
+        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
+                                     [VppRoutePath(self.pg1.remote_ip4,
+                                                   self.pg1.sw_if_index)])
+        route_sever_dst.add_vpp_config()
+
+        # Send PPPoE Discovery
+        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
+                                                 self.pg0.remote_mac)
+        self.pg0.add_stream(tx0)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP
+        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id)
+        self.pg0.add_stream(tx1)
+        self.pg_start()
+
+        # Create PPPoE session
+        pppoe_if = VppPppoeInterface(self,
+                                     self.pg0.remote_ip4,
+                                     self.pg0.remote_mac,
+                                     self.session_id)
+        pppoe_if.add_vpp_config()
+
+        # Delete PPPoE session
+        pppoe_if.remove_vpp_config()
+
+        #
+        # The double del (del the same session twice) should fail,
+        # and we should still be able to use the original
+        #
+        try:
+            gre_if.remove_vpp_config()
+        except Exception:
+            pass
+        else:
+            self.fail("Double GRE tunnel del does not fail")
+
+        #
+        # test case cleanup
+        #
+
+        # Delete a route that resolves the server's destination
+        route_sever_dst.remove_vpp_config()
+
+    def test_PPPoE_Decap_Multiple(self):
+        """ PPPoE Decap Multiple Sessions Test """
+
+        self.vapi.cli("clear trace")
+
+        #
+        # Add a route that resolves the server's destination
+        #
+        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
+                                     [VppRoutePath(self.pg1.remote_ip4,
+                                                   self.pg1.sw_if_index)])
+        route_sever_dst.add_vpp_config()
+
+        # Send PPPoE Discovery 1
+        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
+                                                 self.pg0.remote_mac)
+        self.pg0.add_stream(tx0)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP 1
+        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id)
+        self.pg0.add_stream(tx1)
+        self.pg_start()
+
+        # Create PPPoE session 1
+        pppoe_if1 = VppPppoeInterface(self,
+                                      self.pg0.remote_ip4,
+                                      self.pg0.remote_mac,
+                                      self.session_id)
+        pppoe_if1.add_vpp_config()
+
+        # Send PPPoE Discovery 2
+        tx3 = self.create_stream_pppoe_discovery(self.pg2, self.pg1,
+                                                 self.pg2.remote_mac)
+        self.pg2.add_stream(tx3)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP 2
+        tx4 = self.create_stream_pppoe_lcp(self.pg2, self.pg1,
+                                           self.pg2.remote_mac,
+                                           self.session_id + 1)
+        self.pg2.add_stream(tx4)
+        self.pg_start()
+
+        # Create PPPoE session 2
+        pppoe_if2 = VppPppoeInterface(self,
+                                      self.pg2.remote_ip4,
+                                      self.pg2.remote_mac,
+                                      self.session_id + 1)
+        pppoe_if2.add_vpp_config()
+
+        #
+        # Send tunneled packets that match the created tunnel and
+        # are decapped and forwarded
+        #
+        tx2 = self.create_stream_pppoe_ip4(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id,
+                                           self.pg0.remote_ip4)
+        self.pg0.add_stream(tx2)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx2 = self.pg1.get_capture(len(tx2))
+        self.verify_decapped_pppoe(self.pg0, rx2, tx2)
+
+        tx5 = self.create_stream_pppoe_ip4(self.pg2, self.pg1,
+                                           self.pg2.remote_mac,
+                                           self.session_id + 1,
+                                           self.pg2.remote_ip4)
+        self.pg2.add_stream(tx5)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx5 = self.pg1.get_capture(len(tx5))
+        self.verify_decapped_pppoe(self.pg2, rx5, tx5)
+
+        self.logger.info(self.vapi.cli("show pppoe fib"))
+        self.logger.info(self.vapi.cli("show pppoe session"))
+        self.logger.info(self.vapi.cli("show ip fib"))
+
+        #
+        # test case cleanup
+        #
+
+        # Delete PPPoE session
+        pppoe_if1.remove_vpp_config()
+        pppoe_if2.remove_vpp_config()
+
+        # Delete a route that resolves the server's destination
+        route_sever_dst.remove_vpp_config()
+
+    def test_PPPoE_Encap_Multiple(self):
+        """ PPPoE Encap Multiple Sessions Test """
+
+        self.vapi.cli("clear trace")
+
+        #
+        # Add a route that resolves the server's destination
+        #
+        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
+                                     [VppRoutePath(self.pg1.remote_ip4,
+                                                   self.pg1.sw_if_index)])
+        route_sever_dst.add_vpp_config()
+
+        # Send PPPoE Discovery 1
+        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
+                                                 self.pg0.remote_mac)
+        self.pg0.add_stream(tx0)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP 1
+        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
+                                           self.pg0.remote_mac,
+                                           self.session_id)
+        self.pg0.add_stream(tx1)
+        self.pg_start()
+
+        # Create PPPoE session 1
+        pppoe_if1 = VppPppoeInterface(self,
+                                      self.pg0.remote_ip4,
+                                      self.pg0.remote_mac,
+                                      self.session_id)
+        pppoe_if1.add_vpp_config()
+
+        # Send PPPoE Discovery 2
+        tx3 = self.create_stream_pppoe_discovery(self.pg2, self.pg1,
+                                                 self.pg2.remote_mac)
+        self.pg2.add_stream(tx3)
+        self.pg_start()
+
+        # Send PPPoE PPP LCP 2
+        tx4 = self.create_stream_pppoe_lcp(self.pg2, self.pg1,
+                                           self.pg2.remote_mac,
+                                           self.session_id + 1)
+        self.pg2.add_stream(tx4)
+        self.pg_start()
+
+        # Create PPPoE session 2
+        pppoe_if2 = VppPppoeInterface(self,
+                                      self.pg2.remote_ip4,
+                                      self.pg2.remote_mac,
+                                      self.session_id + 1)
+        pppoe_if2.add_vpp_config()
+
+        #
+        # Send a packet stream that is routed into the session
+        #  - packets are PPPoE encapped
+        #
+        self.vapi.cli("clear trace")
+        tx2 = self.create_stream_ip4(self.pg1, self.pg0,
+                                     self.pg0.remote_ip4, self.dst_ip)
+        self.pg1.add_stream(tx2)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx2 = self.pg0.get_capture(len(tx2))
+        self.verify_encaped_pppoe(self.pg1, rx2, tx2, self.session_id)
+
+        tx5 = self.create_stream_ip4(self.pg1, self.pg2,
+                                     self.pg2.remote_ip4, self.dst_ip)
+        self.pg1.add_stream(tx5)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx5 = self.pg2.get_capture(len(tx5))
+        self.verify_encaped_pppoe(self.pg1, rx5, tx5, self.session_id + 1)
+
+        self.logger.info(self.vapi.cli("show pppoe fib"))
+        self.logger.info(self.vapi.cli("show pppoe session"))
+        self.logger.info(self.vapi.cli("show ip fib"))
+
+        #
+        # test case cleanup
+        #
+
+        # Delete PPPoE session
+        pppoe_if1.remove_vpp_config()
+        pppoe_if2.remove_vpp_config()
+
+        # Delete a route that resolves the server's destination
+        route_sever_dst.remove_vpp_config()
+
+if __name__ == '__main__':
+    unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 3ba2ad4a..4d017c1f 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -2087,3 +2087,29 @@ class VppPapiProvider(object):
                          'decap_vrf_id': decap_vrf_id,
                          'protocol': protocol,
                          'vni': vni})
+
+    def pppoe_add_del_session(
+            self,
+            client_ip,
+            client_mac,
+            session_id=0,
+            is_add=1,
+            is_ipv6=0,
+            decap_vrf_id=0):
+        """
+
+        :param is_add:  (Default value = 1)
+        :param is_ipv6:  (Default value = 0)
+        :param client_ip:
+        :param session_id:  (Default value = 0)
+        :param client_mac:
+        :param decap_vrf_id:  (Default value = 0)
+
+        """
+        return self.api(self.papi.pppoe_add_del_session,
+                        {'is_add': is_add,
+                         'is_ipv6': is_ipv6,
+                         'session_id': session_id,
+                         'client_ip': client_ip,
+                         'decap_vrf_id': decap_vrf_id,
+                         'client_mac': client_mac})
diff --git a/test/vpp_pppoe_interface.py b/test/vpp_pppoe_interface.py
new file mode 100644
index 00000000..9a8b8699
--- /dev/null
+++ b/test/vpp_pppoe_interface.py
@@ -0,0 +1,79 @@
+
+from vpp_interface import VppInterface
+import socket
+from util import ppp, ppc, mactobinary
+
+
+class VppPppoeInterface(VppInterface):
+    """
+    VPP Pppoe interface
+    """
+
+    def __init__(self, test, client_ip, client_mac,
+                 session_id, decap_vrf_id=0):
+        """ Create VPP PPPoE4 interface """
+        self._sw_if_index = 0
+        super(VppPppoeInterface, self).__init__(test)
+        self._test = test
+        self.client_ip = client_ip
+        self.client_mac = client_mac
+        self.session_id = session_id
+        self.decap_vrf_id = decap_vrf_id
+
+    def add_vpp_config(self):
+        cip = socket.inet_pton(socket.AF_INET, self.client_ip)
+        cmac = mactobinary(self.client_mac)
+        r = self.test.vapi.pppoe_add_del_session(
+                cip, cmac,
+                session_id=self.session_id,
+                decap_vrf_id=self.decap_vrf_id)
+        self._sw_if_index = r.sw_if_index
+        self.generate_remote_hosts()
+
+    def remove_vpp_config(self):
+        cip = socket.inet_pton(socket.AF_INET, self.client_ip)
+        cmac = mactobinary(self.client_mac)
+        self.unconfig()
+        r = self.test.vapi.pppoe_add_del_session(
+                cip, cmac,
+                session_id=self.session_id,
+                decap_vrf_id=self.decap_vrf_id,
+                is_add=0)
+
+
+class VppPppoe6Interface(VppInterface):
+    """
+    VPP Pppoe IPv6 interface
+    """
+
+    def __init__(self, test, src_ip, dst_ip, outer_fib_id=0, is_teb=0):
+        """ Create VPP PPPoE6 interface """
+        self._sw_if_index = 0
+        super(VppPppoe6Interface, self).__init__(test)
+        self._test = test
+        self.client_ip = client_ip
+        self.client_mac = client_mac
+        self.session_id = session_id
+        self.decap_vrf_id = decap_vrf_id
+
+    def add_vpp_config(self):
+        cip = socket.inet_pton(socket.AF_INET6, self.client_ip)
+        cmac = mactobinary(self.client_mac)
+        r = self.test.vapi.pppoe_add_del_session(
+                cip, cmac,
+                session_id=self.session_id,
+                decap_vrf_id=self.decap_vrf_id,
+                is_ip6=1)
+        self._sw_if_index = r.sw_if_index
+        self.generate_remote_hosts()
+
+    def remove_vpp_config(self):
+        cip = socket.inet_pton(socket.AF_INET6, self.client_ip)
+        cmac = mactobinary(self.client_mac)
+        self.unconfig()
+        r = self.test.vapi.pppoe_add_del_session(
+                cip, cmac,
+                session_id=self.session_id,
+                decap_vrf_id=self.decap_vrf_id,
+                is_add=0,
+                is_ip6=1)
-- 
cgit 1.2.3-korg


From 2ba92e32e0197f676dd905e5edcb4ff3e1bec241 Mon Sep 17 00:00:00 2001
From: Matus Fabian 
Date: Mon, 21 Aug 2017 07:05:03 -0700
Subject: NAT: Rename snat plugin to nat (VPP-955)

Change-Id: I30a7e3da7a4efc6038a91e27b48045d4b07e2764
Signed-off-by: Matus Fabian 
---
 src/configure.ac                                   |    2 +-
 src/examples/sample-plugin/sample_plugin_doc.md    |    2 +-
 src/plugins/Makefile.am                            |    4 +-
 src/plugins/nat.am                                 |   41 +
 src/plugins/nat/in2out.c                           | 3454 ++++++++++++++++++
 src/plugins/nat/nat.api                            | 1513 ++++++++
 src/plugins/nat/nat.c                              | 2842 +++++++++++++++
 src/plugins/nat/nat.h                              |  541 +++
 src/plugins/nat/nat64.c                            |  858 +++++
 src/plugins/nat/nat64.h                            |  322 ++
 src/plugins/nat/nat64_cli.c                        |  983 ++++++
 src/plugins/nat/nat64_db.c                         |  603 ++++
 src/plugins/nat/nat64_db.h                         |  307 ++
 src/plugins/nat/nat64_doc.md                       |   73 +
 src/plugins/nat/nat64_in2out.c                     | 1118 ++++++
 src/plugins/nat/nat64_out2in.c                     |  494 +++
 src/plugins/nat/nat_all_api_h.h                    |   19 +
 src/plugins/nat/nat_api.c                          | 3255 +++++++++++++++++
 src/plugins/nat/nat_det.c                          |  158 +
 src/plugins/nat/nat_det.h                          |  196 ++
 src/plugins/nat/nat_ipfix_logging.c                |  835 +++++
 src/plugins/nat/nat_ipfix_logging.h                |   79 +
 src/plugins/nat/nat_msg_enum.h                     |   31 +
 src/plugins/nat/nat_test.c                         | 1167 ++++++
 src/plugins/nat/out2in.c                           | 2294 ++++++++++++
 src/plugins/snat.am                                |   41 -
 src/plugins/snat/in2out.c                          | 3454 ------------------
 src/plugins/snat/nat64.c                           |  858 -----
 src/plugins/snat/nat64.h                           |  322 --
 src/plugins/snat/nat64_cli.c                       |  983 ------
 src/plugins/snat/nat64_db.c                        |  603 ----
 src/plugins/snat/nat64_db.h                        |  307 --
 src/plugins/snat/nat64_doc.md                      |   73 -
 src/plugins/snat/nat64_in2out.c                    | 1118 ------
 src/plugins/snat/nat64_out2in.c                    |  494 ---
 src/plugins/snat/out2in.c                          | 2294 ------------
 src/plugins/snat/snat.api                          |  897 -----
 src/plugins/snat/snat.c                            | 2842 ---------------
 src/plugins/snat/snat.h                            |  541 ---
 src/plugins/snat/snat_all_api_h.h                  |   19 -
 src/plugins/snat/snat_api.c                        | 1970 -----------
 src/plugins/snat/snat_det.c                        |  158 -
 src/plugins/snat/snat_det.h                        |  196 --
 src/plugins/snat/snat_ipfix_logging.c              |  835 -----
 src/plugins/snat/snat_ipfix_logging.h              |   79 -
 src/plugins/snat/snat_msg_enum.h                   |   31 -
 src/plugins/snat/snat_test.c                       | 1167 ------
 src/scripts/vnet/nat44                             |   41 +
 src/scripts/vnet/nat44_det                         |  108 +
 src/scripts/vnet/nat44_static                      |   44 +
 src/scripts/vnet/nat44_static_with_port            |   44 +
 src/scripts/vnet/snat                              |   41 -
 src/scripts/vnet/snat_det                          |  108 -
 src/scripts/vnet/snat_static                       |   44 -
 src/scripts/vnet/snat_static_with_port             |   44 -
 src/vpp-api/java/Makefile.am                       |   26 +-
 .../vpp/jvpp/nat/examples/CallbackApiExample.java  |   68 +
 .../io/fd/vpp/jvpp/nat/examples/Readme.txt         |    1 +
 src/vpp-api/java/jvpp-nat/jvpp_nat.c               |  107 +
 src/vpp-api/java/jvpp-nat/jvpp_nat.h               |   42 +
 .../vpp/jvpp/snat/examples/CallbackApiExample.java |   68 -
 .../io/fd/vpp/jvpp/snat/examples/Readme.txt        |    1 -
 src/vpp-api/java/jvpp-snat/jvpp_snat.c             |  107 -
 src/vpp-api/java/jvpp-snat/jvpp_snat.h             |   42 -
 test/test_nat.py                                   | 3720 ++++++++++++++++++++
 test/test_snat.py                                  | 3718 -------------------
 test/vpp_papi_provider.py                          |  187 +-
 67 files changed, 25462 insertions(+), 23572 deletions(-)
 create mode 100644 src/plugins/nat.am
 create mode 100644 src/plugins/nat/in2out.c
 create mode 100644 src/plugins/nat/nat.api
 create mode 100644 src/plugins/nat/nat.c
 create mode 100644 src/plugins/nat/nat.h
 create mode 100644 src/plugins/nat/nat64.c
 create mode 100644 src/plugins/nat/nat64.h
 create mode 100644 src/plugins/nat/nat64_cli.c
 create mode 100644 src/plugins/nat/nat64_db.c
 create mode 100644 src/plugins/nat/nat64_db.h
 create mode 100644 src/plugins/nat/nat64_doc.md
 create mode 100644 src/plugins/nat/nat64_in2out.c
 create mode 100644 src/plugins/nat/nat64_out2in.c
 create mode 100644 src/plugins/nat/nat_all_api_h.h
 create mode 100644 src/plugins/nat/nat_api.c
 create mode 100644 src/plugins/nat/nat_det.c
 create mode 100644 src/plugins/nat/nat_det.h
 create mode 100644 src/plugins/nat/nat_ipfix_logging.c
 create mode 100644 src/plugins/nat/nat_ipfix_logging.h
 create mode 100644 src/plugins/nat/nat_msg_enum.h
 create mode 100644 src/plugins/nat/nat_test.c
 create mode 100644 src/plugins/nat/out2in.c
 delete mode 100644 src/plugins/snat.am
 delete mode 100644 src/plugins/snat/in2out.c
 delete mode 100644 src/plugins/snat/nat64.c
 delete mode 100644 src/plugins/snat/nat64.h
 delete mode 100644 src/plugins/snat/nat64_cli.c
 delete mode 100644 src/plugins/snat/nat64_db.c
 delete mode 100644 src/plugins/snat/nat64_db.h
 delete mode 100644 src/plugins/snat/nat64_doc.md
 delete mode 100644 src/plugins/snat/nat64_in2out.c
 delete mode 100644 src/plugins/snat/nat64_out2in.c
 delete mode 100644 src/plugins/snat/out2in.c
 delete mode 100644 src/plugins/snat/snat.api
 delete mode 100644 src/plugins/snat/snat.c
 delete mode 100644 src/plugins/snat/snat.h
 delete mode 100644 src/plugins/snat/snat_all_api_h.h
 delete mode 100644 src/plugins/snat/snat_api.c
 delete mode 100644 src/plugins/snat/snat_det.c
 delete mode 100644 src/plugins/snat/snat_det.h
 delete mode 100644 src/plugins/snat/snat_ipfix_logging.c
 delete mode 100644 src/plugins/snat/snat_ipfix_logging.h
 delete mode 100644 src/plugins/snat/snat_msg_enum.h
 delete mode 100644 src/plugins/snat/snat_test.c
 create mode 100644 src/scripts/vnet/nat44
 create mode 100644 src/scripts/vnet/nat44_det
 create mode 100644 src/scripts/vnet/nat44_static
 create mode 100644 src/scripts/vnet/nat44_static_with_port
 delete mode 100644 src/scripts/vnet/snat
 delete mode 100644 src/scripts/vnet/snat_det
 delete mode 100644 src/scripts/vnet/snat_static
 delete mode 100644 src/scripts/vnet/snat_static_with_port
 create mode 100644 src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/CallbackApiExample.java
 create mode 100644 src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/Readme.txt
 create mode 100644 src/vpp-api/java/jvpp-nat/jvpp_nat.c
 create mode 100644 src/vpp-api/java/jvpp-nat/jvpp_nat.h
 delete mode 100644 src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/CallbackApiExample.java
 delete mode 100644 src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/Readme.txt
 delete mode 100644 src/vpp-api/java/jvpp-snat/jvpp_snat.c
 delete mode 100644 src/vpp-api/java/jvpp-snat/jvpp_snat.h
 create mode 100644 test/test_nat.py
 delete mode 100644 test/test_snat.py

(limited to 'src/plugins/Makefile.am')

diff --git a/src/configure.ac b/src/configure.ac
index 7a038c2e..4c2d3b47 100644
--- a/src/configure.ac
+++ b/src/configure.ac
@@ -169,7 +169,7 @@ PLUGIN_ENABLED(lb)
 PLUGIN_ENABLED(memif)
 PLUGIN_ENABLED(pppoe)
 PLUGIN_ENABLED(sixrd)
-PLUGIN_ENABLED(snat)
+PLUGIN_ENABLED(nat)
 
 ###############################################################################
 # Dependency checks
diff --git a/src/examples/sample-plugin/sample_plugin_doc.md b/src/examples/sample-plugin/sample_plugin_doc.md
index 9348094c..501a8dca 100644
--- a/src/examples/sample-plugin/sample_plugin_doc.md
+++ b/src/examples/sample-plugin/sample_plugin_doc.md
@@ -29,7 +29,7 @@ Now run VPP and make sure the plugin is loaded.
 	...
 	load_one_plugin:184: Loaded plugin: memif_plugin.so (Packet Memory Interface (experimetal))
 	load_one_plugin:184: Loaded plugin: sample_plugin.so (Sample of VPP Plugin)
-	load_one_plugin:184: Loaded plugin: snat_plugin.so (Network Address Translation)
+	load_one_plugin:184: Loaded plugin: nat_plugin.so (Network Address Translation)
 	...
 	DBGvpp#
 
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 8c7b3fac..205bfe6d 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -74,8 +74,8 @@ if ENABLE_SIXRD_PLUGIN
 include sixrd.am
 endif
 
-if ENABLE_SNAT_PLUGIN
-include snat.am
+if ENABLE_NAT_PLUGIN
+include nat.am
 endif
 
 include ../suffix-rules.mk
diff --git a/src/plugins/nat.am b/src/plugins/nat.am
new file mode 100644
index 00000000..b967a716
--- /dev/null
+++ b/src/plugins/nat.am
@@ -0,0 +1,41 @@
+
+# Copyright (c)  
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+vppapitestplugins_LTLIBRARIES += nat_test_plugin.la
+vppplugins_LTLIBRARIES += nat_plugin.la
+
+nat_plugin_la_SOURCES = nat/nat.c		\
+        nat/nat_api.c				\
+        nat/in2out.c				\
+        nat/out2in.c				\
+	nat/nat_plugin.api.h			\
+        nat/nat_ipfix_logging.c	        	\
+        nat/nat_det.c               		\
+        nat/nat64.c                  		\
+        nat/nat64_cli.c              		\
+        nat/nat64_in2out.c           		\
+        nat/nat64_out2in.c           		\
+        nat/nat64_db.c
+
+API_FILES += nat/nat.api
+
+nobase_apiinclude_HEADERS +=		\
+  nat/nat_all_api_h.h			\
+  nat/nat_msg_enum.h			\
+  nat/nat.api.h
+
+nat_test_plugin_la_SOURCES = \
+  nat/nat_test.c nat/nat_plugin.api.h
+
+# vi:syntax=automake
diff --git a/src/plugins/nat/in2out.c b/src/plugins/nat/in2out.c
new file mode 100644
index 00000000..62a11170
--- /dev/null
+++ b/src/plugins/nat/in2out.c
@@ -0,0 +1,3454 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+typedef struct {
+  u32 sw_if_index;
+  u32 next_index;
+  u32 session_index;
+  u32 is_slow_path;
+} snat_in2out_trace_t;
+
+typedef struct {
+  u32 next_worker_index;
+  u8 do_handoff;
+} snat_in2out_worker_handoff_trace_t;
+
+/* packet trace format function */
+static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
+  char * tag;
+
+  tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
+
+  s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
+              t->sw_if_index, t->next_index, t->session_index);
+
+  return s;
+}
+
+static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
+
+  s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
+              t->sw_if_index, t->next_index);
+
+  return s;
+}
+
+static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  snat_in2out_worker_handoff_trace_t * t =
+    va_arg (*args, snat_in2out_worker_handoff_trace_t *);
+  char * m;
+
+  m = t->do_handoff ? "next worker" : "same worker";
+  s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
+
+  return s;
+}
+
+vlib_node_registration_t snat_in2out_node;
+vlib_node_registration_t snat_in2out_slowpath_node;
+vlib_node_registration_t snat_in2out_fast_node;
+vlib_node_registration_t snat_in2out_worker_handoff_node;
+vlib_node_registration_t snat_det_in2out_node;
+vlib_node_registration_t snat_in2out_output_node;
+vlib_node_registration_t snat_in2out_output_slowpath_node;
+vlib_node_registration_t snat_in2out_output_worker_handoff_node;
+vlib_node_registration_t snat_hairpin_dst_node;
+vlib_node_registration_t snat_hairpin_src_node;
+
+
+#define foreach_snat_in2out_error                       \
+_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
+_(IN2OUT_PACKETS, "Good in2out packets processed")      \
+_(OUT_OF_PORTS, "Out of ports")                         \
+_(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
+_(BAD_ICMP_TYPE, "unsupported ICMP type")               \
+_(NO_TRANSLATION, "No translation")
+
+typedef enum {
+#define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
+  foreach_snat_in2out_error
+#undef _
+  SNAT_IN2OUT_N_ERROR,
+} snat_in2out_error_t;
+
+static char * snat_in2out_error_strings[] = {
+#define _(sym,string) string,
+  foreach_snat_in2out_error
+#undef _
+};
+
+typedef enum {
+  SNAT_IN2OUT_NEXT_LOOKUP,
+  SNAT_IN2OUT_NEXT_DROP,
+  SNAT_IN2OUT_NEXT_ICMP_ERROR,
+  SNAT_IN2OUT_NEXT_SLOW_PATH,
+  SNAT_IN2OUT_N_NEXT,
+} snat_in2out_next_t;
+
+typedef enum {
+  SNAT_HAIRPIN_SRC_NEXT_DROP,
+  SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
+  SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
+  SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
+  SNAT_HAIRPIN_SRC_N_NEXT,
+} snat_hairpin_next_t;
+
+/**
+ * @brief Check if packet should be translated
+ *
+ * Packets aimed at outside interface and external addresss with active session
+ * should be translated.
+ *
+ * @param sm            NAT main
+ * @param rt            NAT runtime data
+ * @param sw_if_index0  index of the inside interface
+ * @param ip0           IPv4 header
+ * @param proto0        NAT protocol
+ * @param rx_fib_index0 RX FIB index
+ *
+ * @returns 0 if packet should be translated otherwise 1
+ */
+static inline int
+snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
+                         u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
+                         u32 rx_fib_index0)
+{
+  fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP4,
+    .fp_len = 32,
+    .fp_addr = {
+        .ip4.as_u32 = ip0->dst_address.as_u32,
+    },
+  };
+
+  /* Don't NAT packet aimed at the intfc address */
+  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
+                                      ip0->dst_address.as_u32)))
+    return 1;
+
+  fei = fib_table_lookup (rx_fib_index0, &pfx);
+  if (FIB_NODE_INDEX_INVALID != fei)
+    {
+      u32 sw_if_index = fib_entry_get_resolving_interface (fei);
+      if (sw_if_index == ~0)
+        {
+          fei = fib_table_lookup (sm->outside_fib_index, &pfx);
+          if (FIB_NODE_INDEX_INVALID != fei)
+            sw_if_index = fib_entry_get_resolving_interface (fei);
+        }
+      snat_interface_t *i;
+      pool_foreach (i, sm->interfaces,
+      ({
+        /* NAT packet aimed at outside interface */
+        if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
+          return 0;
+      }));
+    }
+
+  return 1;
+}
+
+static inline int
+snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
+                    u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
+                    u32 rx_fib_index0)
+{
+  udp_header_t * udp0 = ip4_next_header (ip0);
+  snat_session_key_t key0, sm0;
+  clib_bihash_kv_8_8_t kv0, value0;
+
+  key0.addr = ip0->dst_address;
+  key0.port = udp0->dst_port;
+  key0.protocol = proto0;
+  key0.fib_index = sm->outside_fib_index;
+  kv0.key = key0.as_u64;
+
+  /* NAT packet aimed at external address if */
+  /* has active sessions */
+  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
+    {
+      /* or is static mappings */
+      if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
+        return 0;
+    }
+  else
+    return 0;
+
+  return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
+                                 rx_fib_index0);
+}
+
+static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
+                      ip4_header_t * ip0,
+                      u32 rx_fib_index0,
+                      snat_session_key_t * key0,
+                      snat_session_t ** sessionp,
+                      vlib_node_runtime_t * node,
+                      u32 next0,
+                      u32 thread_index)
+{
+  snat_user_t *u;
+  snat_user_key_t user_key;
+  snat_session_t *s;
+  clib_bihash_kv_8_8_t kv0, value0;
+  u32 oldest_per_user_translation_list_index;
+  dlist_elt_t * oldest_per_user_translation_list_elt;
+  dlist_elt_t * per_user_translation_list_elt;
+  dlist_elt_t * per_user_list_head_elt;
+  u32 session_index;
+  snat_session_key_t key1;
+  u32 address_index = ~0;
+  u32 outside_fib_index;
+  uword * p;
+  snat_worker_key_t worker_by_out_key;
+
+  p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
+  if (! p)
+    {
+      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
+      return SNAT_IN2OUT_NEXT_DROP;
+    }
+  outside_fib_index = p[0];
+
+  key1.protocol = key0->protocol;
+  user_key.addr = ip0->src_address;
+  user_key.fib_index = rx_fib_index0;
+  kv0.key = user_key.as_u64;
+
+  /* Ever heard of the "user" = src ip4 address before? */
+  if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
+    {
+      /* no, make a new one */
+      pool_get (sm->per_thread_data[thread_index].users, u);
+      memset (u, 0, sizeof (*u));
+      u->addr = ip0->src_address;
+      u->fib_index = rx_fib_index0;
+
+      pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
+
+      u->sessions_per_user_list_head_index = per_user_list_head_elt -
+        sm->per_thread_data[thread_index].list_pool;
+
+      clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
+                       u->sessions_per_user_list_head_index);
+
+      kv0.value = u - sm->per_thread_data[thread_index].users;
+
+      /* add user */
+      clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
+    }
+  else
+    {
+      u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
+                             value0.value);
+    }
+
+  /* Over quota? Recycle the least recently used dynamic translation */
+  if (u->nsessions >= sm->max_translations_per_user)
+    {
+      /* Remove the oldest dynamic translation */
+      do {
+          oldest_per_user_translation_list_index =
+            clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
+                                    u->sessions_per_user_list_head_index);
+
+          ASSERT (oldest_per_user_translation_list_index != ~0);
+
+          /* add it back to the end of the LRU list */
+          clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                              u->sessions_per_user_list_head_index,
+                              oldest_per_user_translation_list_index);
+          /* Get the list element */
+          oldest_per_user_translation_list_elt =
+            pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
+                               oldest_per_user_translation_list_index);
+
+          /* Get the session index from the list element */
+          session_index = oldest_per_user_translation_list_elt->value;
+
+          /* Get the session */
+          s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                 session_index);
+      } while (snat_is_session_static (s));
+
+      if (snat_is_unk_proto_session (s))
+        {
+          clib_bihash_kv_16_8_t up_kv;
+          snat_unk_proto_ses_key_t key;
+
+          /* Remove from lookup tables */
+          key.l_addr = s->in2out.addr;
+          key.r_addr = s->ext_host_addr;
+          key.fib_index = s->in2out.fib_index;
+          key.proto = s->in2out.port;
+          up_kv.key[0] = key.as_u64[0];
+          up_kv.key[1] = key.as_u64[1];
+          if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &up_kv, 0))
+            clib_warning ("in2out key del failed");
+
+          key.l_addr = s->out2in.addr;
+          key.fib_index = s->out2in.fib_index;
+          up_kv.key[0] = key.as_u64[0];
+          up_kv.key[1] = key.as_u64[1];
+          if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &up_kv, 0))
+            clib_warning ("out2in key del failed");
+        }
+      else
+        {
+          /* Remove in2out, out2in keys */
+          kv0.key = s->in2out.as_u64;
+          if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
+              clib_warning ("in2out key delete failed");
+          kv0.key = s->out2in.as_u64;
+          if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
+              clib_warning ("out2in key delete failed");
+
+          /* log NAT event */
+          snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
+                                              s->out2in.addr.as_u32,
+                                              s->in2out.protocol,
+                                              s->in2out.port,
+                                              s->out2in.port,
+                                              s->in2out.fib_index);
+
+          snat_free_outside_address_and_port
+            (sm, &s->out2in, s->outside_address_index);
+        }
+      s->outside_address_index = ~0;
+
+      if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, thread_index,
+                                               &key1, &address_index))
+        {
+          ASSERT(0);
+
+          b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
+          return SNAT_IN2OUT_NEXT_DROP;
+        }
+      s->outside_address_index = address_index;
+    }
+  else
+    {
+      u8 static_mapping = 1;
+
+      /* First try to match static mapping by local address and port */
+      if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
+        {
+          static_mapping = 0;
+          /* Try to create dynamic translation */
+          if (snat_alloc_outside_address_and_port (sm, rx_fib_index0,
+                                                   thread_index, &key1,
+                                                   &address_index))
+            {
+              b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
+              return SNAT_IN2OUT_NEXT_DROP;
+            }
+        }
+
+      /* Create a new session */
+      pool_get (sm->per_thread_data[thread_index].sessions, s);
+      memset (s, 0, sizeof (*s));
+
+      s->outside_address_index = address_index;
+
+      if (static_mapping)
+        {
+          u->nstaticsessions++;
+          s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+        }
+      else
+        {
+          u->nsessions++;
+        }
+
+      /* Create list elts */
+      pool_get (sm->per_thread_data[thread_index].list_pool,
+                per_user_translation_list_elt);
+      clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
+                       per_user_translation_list_elt -
+                       sm->per_thread_data[thread_index].list_pool);
+
+      per_user_translation_list_elt->value =
+        s - sm->per_thread_data[thread_index].sessions;
+      s->per_user_index = per_user_translation_list_elt -
+                          sm->per_thread_data[thread_index].list_pool;
+      s->per_user_list_head_index = u->sessions_per_user_list_head_index;
+
+      clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                          s->per_user_list_head_index,
+                          per_user_translation_list_elt -
+                          sm->per_thread_data[thread_index].list_pool);
+   }
+
+  s->in2out = *key0;
+  s->out2in = key1;
+  s->out2in.protocol = key0->protocol;
+  s->out2in.fib_index = outside_fib_index;
+  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
+  *sessionp = s;
+
+  /* Add to translation hashes */
+  kv0.key = s->in2out.as_u64;
+  kv0.value = s - sm->per_thread_data[thread_index].sessions;
+  if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
+      clib_warning ("in2out key add failed");
+
+  kv0.key = s->out2in.as_u64;
+  kv0.value = s - sm->per_thread_data[thread_index].sessions;
+
+  if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
+      clib_warning ("out2in key add failed");
+
+  /* Add to translated packets worker lookup */
+  worker_by_out_key.addr = s->out2in.addr;
+  worker_by_out_key.port = s->out2in.port;
+  worker_by_out_key.fib_index = s->out2in.fib_index;
+  kv0.key = worker_by_out_key.as_u64;
+  kv0.value = thread_index;
+  clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
+
+  /* log NAT event */
+  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
+                                      s->out2in.addr.as_u32,
+                                      s->in2out.protocol,
+                                      s->in2out.port,
+                                      s->out2in.port,
+                                      s->in2out.fib_index);
+  return next0;
+}
+
+static_always_inline
+snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
+                                 snat_session_key_t *p_key0)
+{
+  icmp46_header_t *icmp0;
+  snat_session_key_t key0;
+  icmp_echo_header_t *echo0, *inner_echo0 = 0;
+  ip4_header_t *inner_ip0 = 0;
+  void *l4_header = 0;
+  icmp46_header_t *inner_icmp0;
+
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  echo0 = (icmp_echo_header_t *)(icmp0+1);
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      key0.protocol = SNAT_PROTOCOL_ICMP;
+      key0.addr = ip0->src_address;
+      key0.port = echo0->identifier;
+    }
+  else
+    {
+      inner_ip0 = (ip4_header_t *)(echo0+1);
+      l4_header = ip4_next_header (inner_ip0);
+      key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
+      key0.addr = inner_ip0->dst_address;
+      switch (key0.protocol)
+        {
+        case SNAT_PROTOCOL_ICMP:
+          inner_icmp0 = (icmp46_header_t*)l4_header;
+          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
+          key0.port = inner_echo0->identifier;
+          break;
+        case SNAT_PROTOCOL_UDP:
+        case SNAT_PROTOCOL_TCP:
+          key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
+          break;
+        default:
+          return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
+        }
+    }
+  *p_key0 = key0;
+  return -1; /* success */
+}
+
+/**
+ * Get address and port values to be used for ICMP packet translation
+ * and create session if needed
+ *
+ * @param[in,out] sm             NAT main
+ * @param[in,out] node           NAT node runtime
+ * @param[in] thread_index       thread index
+ * @param[in,out] b0             buffer containing packet to be translated
+ * @param[out] p_proto           protocol used for matching
+ * @param[out] p_value           address and port after NAT translation
+ * @param[out] p_dont_translate  if packet should not be translated
+ * @param d                      optional parameter
+ * @param e                      optional parameter
+ */
+u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e)
+{
+  ip4_header_t *ip0;
+  icmp46_header_t *icmp0;
+  u32 sw_if_index0;
+  u32 rx_fib_index0;
+  snat_session_key_t key0;
+  snat_session_t *s0 = 0;
+  u8 dont_translate = 0;
+  clib_bihash_kv_8_8_t kv0, value0;
+  u32 next0 = ~0;
+  int err;
+  u32 iph_offset0 = 0;
+
+  if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
+    {
+      iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
+    }
+  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+
+  err = icmp_get_key (ip0, &key0);
+  if (err != -1)
+    {
+      b0->error = node->errors[err];
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      goto out;
+    }
+  key0.fib_index = rx_fib_index0;
+
+  kv0.key = key0.as_u64;
+
+  if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
+    {
+      if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
+          IP_PROTOCOL_ICMP, rx_fib_index0) &&
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+
+      if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
+        {
+          b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          goto out;
+        }
+
+      next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
+                         &s0, node, next0, thread_index);
+
+      if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
+        goto out;
+    }
+  else
+    {
+      if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
+                        icmp0->type != ICMP4_echo_reply &&
+                        !icmp_is_error_message (icmp0)))
+        {
+          b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          goto out;
+        }
+
+      s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                              value0.value);
+    }
+
+out:
+  *p_proto = key0.protocol;
+  if (s0)
+    *p_value = s0->out2in;
+  *p_dont_translate = dont_translate;
+  if (d)
+    *(snat_session_t**)d = s0;
+  return next0;
+}
+
+/**
+ * Get address and port values to be used for ICMP packet translation
+ *
+ * @param[in] sm                 NAT main
+ * @param[in,out] node           NAT node runtime
+ * @param[in] thread_index       thread index
+ * @param[in,out] b0             buffer containing packet to be translated
+ * @param[out] p_proto           protocol used for matching
+ * @param[out] p_value           address and port after NAT translation
+ * @param[out] p_dont_translate  if packet should not be translated
+ * @param d                      optional parameter
+ * @param e                      optional parameter
+ */
+u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e)
+{
+  ip4_header_t *ip0;
+  icmp46_header_t *icmp0;
+  u32 sw_if_index0;
+  u32 rx_fib_index0;
+  snat_session_key_t key0;
+  snat_session_key_t sm0;
+  u8 dont_translate = 0;
+  u8 is_addr_only;
+  u32 next0 = ~0;
+  int err;
+
+  ip0 = vlib_buffer_get_current (b0);
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+
+  err = icmp_get_key (ip0, &key0);
+  if (err != -1)
+    {
+      b0->error = node->errors[err];
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      goto out2;
+    }
+  key0.fib_index = rx_fib_index0;
+
+  if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
+    {
+      if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
+          IP_PROTOCOL_ICMP, rx_fib_index0)))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+
+      if (icmp_is_error_message (icmp0))
+        {
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          goto out;
+        }
+
+      b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      goto out;
+    }
+
+  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
+                    (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
+                    !icmp_is_error_message (icmp0)))
+    {
+      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      goto out;
+    }
+
+out:
+  *p_value = sm0;
+out2:
+  *p_proto = key0.protocol;
+  *p_dont_translate = dont_translate;
+  return next0;
+}
+
+static inline u32 icmp_in2out (snat_main_t *sm,
+                               vlib_buffer_t * b0,
+                               ip4_header_t * ip0,
+                               icmp46_header_t * icmp0,
+                               u32 sw_if_index0,
+                               u32 rx_fib_index0,
+                               vlib_node_runtime_t * node,
+                               u32 next0,
+                               u32 thread_index,
+                               void *d,
+                               void *e)
+{
+  snat_session_key_t sm0;
+  u8 protocol;
+  icmp_echo_header_t *echo0, *inner_echo0 = 0;
+  ip4_header_t *inner_ip0;
+  void *l4_header = 0;
+  icmp46_header_t *inner_icmp0;
+  u8 dont_translate;
+  u32 new_addr0, old_addr0;
+  u16 old_id0, new_id0;
+  ip_csum_t sum0;
+  u16 checksum0;
+  u32 next0_tmp;
+
+  echo0 = (icmp_echo_header_t *)(icmp0+1);
+
+  next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
+                                       &protocol, &sm0, &dont_translate, d, e);
+  if (next0_tmp != ~0)
+    next0 = next0_tmp;
+  if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
+    goto out;
+
+  sum0 = ip_incremental_checksum (0, icmp0,
+                                  ntohs(ip0->length) - ip4_header_bytes (ip0));
+  checksum0 = ~ip_csum_fold (sum0);
+  if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
+    {
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      goto out;
+    }
+
+  old_addr0 = ip0->src_address.as_u32;
+  new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
+  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
+    vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+
+  sum0 = ip0->checksum;
+  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
+                         src_address /* changed member */);
+  ip0->checksum = ip_csum_fold (sum0);
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      new_id0 = sm0.port;
+      if (PREDICT_FALSE(new_id0 != echo0->identifier))
+        {
+          old_id0 = echo0->identifier;
+          new_id0 = sm0.port;
+          echo0->identifier = new_id0;
+
+          sum0 = icmp0->checksum;
+          sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
+                                 identifier);
+          icmp0->checksum = ip_csum_fold (sum0);
+        }
+    }
+  else
+    {
+      inner_ip0 = (ip4_header_t *)(echo0+1);
+      l4_header = ip4_next_header (inner_ip0);
+
+      if (!ip4_header_checksum_is_valid (inner_ip0))
+        {
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          goto out;
+        }
+
+      old_addr0 = inner_ip0->dst_address.as_u32;
+      inner_ip0->dst_address = sm0.addr;
+      new_addr0 = inner_ip0->dst_address.as_u32;
+
+      sum0 = icmp0->checksum;
+      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
+                             dst_address /* changed member */);
+      icmp0->checksum = ip_csum_fold (sum0);
+
+      switch (protocol)
+        {
+          case SNAT_PROTOCOL_ICMP:
+            inner_icmp0 = (icmp46_header_t*)l4_header;
+            inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
+
+            old_id0 = inner_echo0->identifier;
+            new_id0 = sm0.port;
+            inner_echo0->identifier = new_id0;
+
+            sum0 = icmp0->checksum;
+            sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
+                                   identifier);
+            icmp0->checksum = ip_csum_fold (sum0);
+            break;
+          case SNAT_PROTOCOL_UDP:
+          case SNAT_PROTOCOL_TCP:
+            old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
+            new_id0 = sm0.port;
+            ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
+
+            sum0 = icmp0->checksum;
+            sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
+                                   dst_port);
+            icmp0->checksum = ip_csum_fold (sum0);
+            break;
+          default:
+            ASSERT(0);
+        }
+    }
+
+out:
+  return next0;
+}
+
+/**
+ * @brief Hairpinning
+ *
+ * Hairpinning allows two endpoints on the internal side of the NAT to
+ * communicate even if they only use each other's external IP addresses
+ * and ports.
+ *
+ * @param sm     NAT main.
+ * @param b0     Vlib buffer.
+ * @param ip0    IP header.
+ * @param udp0   UDP header.
+ * @param tcp0   TCP header.
+ * @param proto0 NAT protocol.
+ */
+static inline void
+snat_hairpinning (snat_main_t *sm,
+                  vlib_buffer_t * b0,
+                  ip4_header_t * ip0,
+                  udp_header_t * udp0,
+                  tcp_header_t * tcp0,
+                  u32 proto0)
+{
+  snat_session_key_t key0, sm0;
+  snat_worker_key_t k0;
+  snat_session_t * s0;
+  clib_bihash_kv_8_8_t kv0, value0;
+  ip_csum_t sum0;
+  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
+  u16 new_dst_port0, old_dst_port0;
+
+  key0.addr = ip0->dst_address;
+  key0.port = udp0->dst_port;
+  key0.protocol = proto0;
+  key0.fib_index = sm->outside_fib_index;
+  kv0.key = key0.as_u64;
+
+  /* Check if destination is in active sessions */
+  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
+    {
+      /* or static mappings */
+      if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
+        {
+          new_dst_addr0 = sm0.addr.as_u32;
+          new_dst_port0 = sm0.port;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+        }
+    }
+  else
+    {
+      si = value0.value;
+      if (sm->num_workers > 1)
+        {
+          k0.addr = ip0->dst_address;
+          k0.port = udp0->dst_port;
+          k0.fib_index = sm->outside_fib_index;
+          kv0.key = k0.as_u64;
+          if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
+            ASSERT(0);
+          else
+            ti = value0.value;
+        }
+      else
+        ti = sm->num_workers;
+
+      s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
+      new_dst_addr0 = s0->in2out.addr.as_u32;
+      new_dst_port0 = s0->in2out.port;
+      vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
+    }
+
+  /* Destination is behind the same NAT, use internal address and port */
+  if (new_dst_addr0)
+    {
+      old_dst_addr0 = ip0->dst_address.as_u32;
+      ip0->dst_address.as_u32 = new_dst_addr0;
+      sum0 = ip0->checksum;
+      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
+                             ip4_header_t, dst_address);
+      ip0->checksum = ip_csum_fold (sum0);
+
+      old_dst_port0 = tcp0->dst;
+      if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
+        {
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              tcp0->dst = new_dst_port0;
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
+                                     ip4_header_t, dst_address);
+              sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
+                                     ip4_header_t /* cheat */, length);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              udp0->dst_port = new_dst_port0;
+              udp0->checksum = 0;
+            }
+        }
+      else
+        {
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
+                                     ip4_header_t, dst_address);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+        }
+    }
+}
+
+static inline void
+snat_icmp_hairpinning (snat_main_t *sm,
+                       vlib_buffer_t * b0,
+                       ip4_header_t * ip0,
+                       icmp46_header_t * icmp0)
+{
+  snat_session_key_t key0, sm0;
+  clib_bihash_kv_8_8_t kv0, value0;
+  snat_worker_key_t k0;
+  u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
+  ip_csum_t sum0;
+  snat_session_t *s0;
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
+      u16 icmp_id0 = echo0->identifier;
+      key0.addr = ip0->dst_address;
+      key0.port = icmp_id0;
+      key0.protocol = SNAT_PROTOCOL_ICMP;
+      key0.fib_index = sm->outside_fib_index;
+      kv0.key = key0.as_u64;
+
+      /* Check if destination is in active sessions */
+      if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
+        {
+          /* or static mappings */
+          if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
+            {
+              new_dst_addr0 = sm0.addr.as_u32;
+              vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+            }
+        }
+      else
+        {
+          si = value0.value;
+          if (sm->num_workers > 1)
+            {
+              k0.addr = ip0->dst_address;
+              k0.port = icmp_id0;
+              k0.fib_index = sm->outside_fib_index;
+              kv0.key = k0.as_u64;
+              if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
+                ASSERT(0);
+              else
+                ti = value0.value;
+            }
+          else
+            ti = sm->num_workers;
+
+          s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
+          new_dst_addr0 = s0->in2out.addr.as_u32;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
+          echo0->identifier = s0->in2out.port;
+          sum0 = icmp0->checksum;
+          sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
+                                 icmp_echo_header_t, identifier);
+          icmp0->checksum = ip_csum_fold (sum0);
+        }
+
+      /* Destination is behind the same NAT, use internal address and port */
+      if (new_dst_addr0)
+        {
+          old_dst_addr0 = ip0->dst_address.as_u32;
+          ip0->dst_address.as_u32 = new_dst_addr0;
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
+                                 ip4_header_t, dst_address);
+          ip0->checksum = ip_csum_fold (sum0);
+        }
+    }
+
+}
+
+static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
+                                         vlib_buffer_t * b0,
+                                         ip4_header_t * ip0,
+                                         icmp46_header_t * icmp0,
+                                         u32 sw_if_index0,
+                                         u32 rx_fib_index0,
+                                         vlib_node_runtime_t * node,
+                                         u32 next0,
+                                         f64 now,
+                                         u32 thread_index,
+                                         snat_session_t ** p_s0)
+{
+  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                      next0, thread_index, p_s0, 0);
+  snat_session_t * s0 = *p_s0;
+  if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
+    {
+      /* Hairpinning */
+      if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
+        snat_icmp_hairpinning(sm, b0, ip0, icmp0);
+      /* Accounting */
+      s0->last_heard = now;
+      s0->total_pkts++;
+      s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
+      /* Per-user LRU list maintenance for dynamic translations */
+      if (!snat_is_session_static (s0))
+        {
+          clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                             s0->per_user_index);
+          clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                              s0->per_user_list_head_index,
+                              s0->per_user_index);
+        }
+    }
+  return next0;
+}
+static inline void
+snat_hairpinning_unknown_proto (snat_main_t *sm,
+                                vlib_buffer_t * b,
+                                ip4_header_t * ip)
+{
+  u32 old_addr, new_addr = 0, ti = 0;
+  clib_bihash_kv_8_8_t kv, value;
+  clib_bihash_kv_16_8_t s_kv, s_value;
+  snat_unk_proto_ses_key_t key;
+  snat_session_key_t m_key;
+  snat_worker_key_t w_key;
+  snat_static_mapping_t *m;
+  ip_csum_t sum;
+  snat_session_t *s;
+
+  old_addr = ip->dst_address.as_u32;
+  key.l_addr.as_u32 = ip->dst_address.as_u32;
+  key.r_addr.as_u32 = ip->src_address.as_u32;
+  key.fib_index = sm->outside_fib_index;
+  key.proto = ip->protocol;
+  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
+  s_kv.key[0] = key.as_u64[0];
+  s_kv.key[1] = key.as_u64[1];
+  if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+    {
+      m_key.addr = ip->dst_address;
+      m_key.fib_index = sm->outside_fib_index;
+      m_key.port = 0;
+      m_key.protocol = 0;
+      kv.key = m_key.as_u64;
+      if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+        return;
+
+      m = pool_elt_at_index (sm->static_mappings, value.value);
+      if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
+        vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
+      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
+    }
+  else
+    {
+      if (sm->num_workers > 1)
+        {
+          w_key.addr = ip->dst_address;
+          w_key.port = 0;
+          w_key.fib_index = sm->outside_fib_index;
+          kv.key = w_key.as_u64;
+          if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv, &value))
+            return;
+          else
+            ti = value.value;
+        }
+      else
+        ti = sm->num_workers;
+
+      s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
+      if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
+        vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
+      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
+    }
+  sum = ip->checksum;
+  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
+  ip->checksum = ip_csum_fold (sum);
+}
+
+static void
+snat_in2out_unknown_proto (snat_main_t *sm,
+                           vlib_buffer_t * b,
+                           ip4_header_t * ip,
+                           u32 rx_fib_index,
+                           u32 thread_index,
+                           f64 now,
+                           vlib_main_t * vm)
+{
+  clib_bihash_kv_8_8_t kv, value;
+  clib_bihash_kv_16_8_t s_kv, s_value;
+  snat_static_mapping_t *m;
+  snat_session_key_t m_key;
+  u32 old_addr, new_addr = 0;
+  ip_csum_t sum;
+  snat_user_key_t u_key;
+  snat_user_t *u;
+  dlist_elt_t *head, *elt, *oldest;
+  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
+  u32 elt_index, head_index, ses_index, oldest_index;
+  snat_session_t * s;
+  snat_unk_proto_ses_key_t key;
+  u32 address_index = ~0;
+  int i;
+  u8 is_sm = 0;
+
+  old_addr = ip->src_address.as_u32;
+
+  key.l_addr = ip->src_address;
+  key.r_addr = ip->dst_address;
+  key.fib_index = rx_fib_index;
+  key.proto = ip->protocol;
+  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
+  s_kv.key[0] = key.as_u64[0];
+  s_kv.key[1] = key.as_u64[1];
+
+  if (!clib_bihash_search_16_8 (&sm->in2out_unk_proto, &s_kv, &s_value))
+    {
+      s = pool_elt_at_index (tsm->sessions, s_value.value);
+      new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
+    }
+  else
+    {
+      u_key.addr = ip->src_address;
+      u_key.fib_index = rx_fib_index;
+      kv.key = u_key.as_u64;
+
+      /* Ever heard of the "user" = src ip4 address before? */
+      if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+        {
+          /* no, make a new one */
+          pool_get (tsm->users, u);
+          memset (u, 0, sizeof (*u));
+          u->addr = ip->src_address;
+          u->fib_index = rx_fib_index;
+
+          pool_get (tsm->list_pool, head);
+          u->sessions_per_user_list_head_index = head - tsm->list_pool;
+
+          clib_dlist_init (tsm->list_pool,
+                           u->sessions_per_user_list_head_index);
+
+          kv.value = u - tsm->users;
+
+          /* add user */
+          clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
+        }
+      else
+        {
+          u = pool_elt_at_index (tsm->users, value.value);
+        }
+
+      m_key.addr = ip->src_address;
+      m_key.port = 0;
+      m_key.protocol = 0;
+      m_key.fib_index = rx_fib_index;
+      kv.key = m_key.as_u64;
+
+      /* Try to find static mapping first */
+      if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
+        {
+          m = pool_elt_at_index (sm->static_mappings, value.value);
+          new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
+          is_sm = 1;
+          goto create_ses;
+        }
+      /* Fallback to 3-tuple key */
+      else
+        {
+          /* Choose same out address as for TCP/UDP session to same destination */
+          if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+            {
+              head_index = u->sessions_per_user_list_head_index;
+              head = pool_elt_at_index (tsm->list_pool, head_index);
+              elt_index = head->next;
+              elt = pool_elt_at_index (tsm->list_pool, elt_index);
+              ses_index = elt->value;
+              while (ses_index != ~0)
+                {
+                  s =  pool_elt_at_index (tsm->sessions, ses_index);
+                  elt_index = elt->next;
+                  elt = pool_elt_at_index (tsm->list_pool, elt_index);
+                  ses_index = elt->value;
+
+                  if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
+                    {
+                      new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
+                      address_index = s->outside_address_index;
+
+                      key.fib_index = sm->outside_fib_index;
+                      key.l_addr.as_u32 = new_addr;
+                      s_kv.key[0] = key.as_u64[0];
+                      s_kv.key[1] = key.as_u64[1];
+                      if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+                        break;
+
+                      goto create_ses;
+                    }
+                }
+            }
+          key.fib_index = sm->outside_fib_index;
+          for (i = 0; i < vec_len (sm->addresses); i++)
+            {
+              key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
+              s_kv.key[0] = key.as_u64[0];
+              s_kv.key[1] = key.as_u64[1];
+              if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+                {
+                  new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
+                  address_index = i;
+                  goto create_ses;
+                }
+            }
+          return;
+        }
+
+create_ses:
+      /* Over quota? Recycle the least recently used dynamic translation */
+      if (u->nsessions >= sm->max_translations_per_user && !is_sm)
+        {
+          /* Remove the oldest dynamic translation */
+          do {
+              oldest_index = clib_dlist_remove_head (
+                tsm->list_pool, u->sessions_per_user_list_head_index);
+
+              ASSERT (oldest_index != ~0);
+
+              /* add it back to the end of the LRU list */
+              clib_dlist_addtail (tsm->list_pool,
+                                  u->sessions_per_user_list_head_index,
+                                  oldest_index);
+              /* Get the list element */
+              oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
+
+              /* Get the session index from the list element */
+              ses_index = oldest->value;
+
+              /* Get the session */
+              s = pool_elt_at_index (tsm->sessions, ses_index);
+          } while (snat_is_session_static (s));
+
+          if (snat_is_unk_proto_session (s))
+            {
+              /* Remove from lookup tables */
+              key.l_addr = s->in2out.addr;
+              key.r_addr = s->ext_host_addr;
+              key.fib_index = s->in2out.fib_index;
+              key.proto = s->in2out.port;
+              s_kv.key[0] = key.as_u64[0];
+              s_kv.key[1] = key.as_u64[1];
+              if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 0))
+                clib_warning ("in2out key del failed");
+
+              key.l_addr = s->out2in.addr;
+              key.fib_index = s->out2in.fib_index;
+              s_kv.key[0] = key.as_u64[0];
+              s_kv.key[1] = key.as_u64[1];
+              if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 0))
+                clib_warning ("out2in key del failed");
+            }
+          else
+            {
+              /* log NAT event */
+              snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
+                                                  s->out2in.addr.as_u32,
+                                                  s->in2out.protocol,
+                                                  s->in2out.port,
+                                                  s->out2in.port,
+                                                  s->in2out.fib_index);
+
+              snat_free_outside_address_and_port (sm, &s->out2in,
+                                                  s->outside_address_index);
+
+              /* Remove in2out, out2in keys */
+              kv.key = s->in2out.as_u64;
+              if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
+                clib_warning ("in2out key del failed");
+              kv.key = s->out2in.as_u64;
+              if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
+                clib_warning ("out2in key del failed");
+            }
+        }
+      else
+        {
+          /* Create a new session */
+          pool_get (tsm->sessions, s);
+          memset (s, 0, sizeof (*s));
+
+          /* Create list elts */
+          pool_get (tsm->list_pool, elt);
+          clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
+          elt->value = s - tsm->sessions;
+          s->per_user_index = elt - tsm->list_pool;
+          s->per_user_list_head_index = u->sessions_per_user_list_head_index;
+          clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                              s->per_user_index);
+        }
+
+      s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
+      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
+      s->outside_address_index = address_index;
+      s->out2in.addr.as_u32 = new_addr;
+      s->out2in.fib_index = sm->outside_fib_index;
+      s->in2out.addr.as_u32 = old_addr;
+      s->in2out.fib_index = rx_fib_index;
+      s->in2out.port = s->out2in.port = ip->protocol;
+      if (is_sm)
+        {
+          u->nstaticsessions++;
+          s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+        }
+      else
+        {
+          u->nsessions++;
+        }
+
+      /* Add to lookup tables */
+      key.l_addr.as_u32 = old_addr;
+      key.r_addr = ip->dst_address;
+      key.proto = ip->protocol;
+      key.fib_index = rx_fib_index;
+      s_kv.key[0] = key.as_u64[0];
+      s_kv.key[1] = key.as_u64[1];
+      s_kv.value = s - tsm->sessions;
+      if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
+        clib_warning ("in2out key add failed");
+
+      key.l_addr.as_u32 = new_addr;
+      key.fib_index = sm->outside_fib_index;
+      s_kv.key[0] = key.as_u64[0];
+      s_kv.key[1] = key.as_u64[1];
+      if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
+        clib_warning ("out2in key add failed");
+  }
+
+  /* Update IP checksum */
+  sum = ip->checksum;
+  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
+  ip->checksum = ip_csum_fold (sum);
+
+  /* Accounting */
+  s->last_heard = now;
+  s->total_pkts++;
+  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
+  /* Per-user LRU list maintenance */
+  clib_dlist_remove (tsm->list_pool, s->per_user_index);
+  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                      s->per_user_index);
+
+  /* Hairpinning */
+  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
+    snat_hairpinning_unknown_proto(sm, b, ip);
+
+  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
+    vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+}
+
+static inline uword
+snat_in2out_node_fn_inline (vlib_main_t * vm,
+                            vlib_node_runtime_t * node,
+                            vlib_frame_t * frame, int is_slow_path,
+                            int is_output_feature)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_in2out_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+  f64 now = vlib_time_now (vm);
+  u32 stats_node_index;
+  u32 thread_index = vlib_get_thread_index ();
+
+  stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
+    snat_in2out_node.index;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+          u32 next0, next1;
+          u32 sw_if_index0, sw_if_index1;
+          ip4_header_t * ip0, * ip1;
+          ip_csum_t sum0, sum1;
+          u32 new_addr0, old_addr0, new_addr1, old_addr1;
+          u16 old_port0, new_port0, old_port1, new_port1;
+          udp_header_t * udp0, * udp1;
+          tcp_header_t * tcp0, * tcp1;
+          icmp46_header_t * icmp0, * icmp1;
+          snat_session_key_t key0, key1;
+          u32 rx_fib_index0, rx_fib_index1;
+          u32 proto0, proto1;
+          snat_session_t * s0 = 0, * s1 = 0;
+          clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
+          u32 iph_offset0 = 0, iph_offset1 = 0;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+          /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          if (is_output_feature)
+            iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
+
+          ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
+                 iph_offset0);
+
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+          icmp0 = (icmp46_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
+                                   sw_if_index0);
+
+          next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace00;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          /* Next configured feature, probably ip4-lookup */
+          if (is_slow_path)
+            {
+              if (PREDICT_FALSE (proto0 == ~0))
+                {
+                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
+                                             thread_index, now, vm);
+                  goto trace00;
+                }
+
+              if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+                {
+                  next0 = icmp_in2out_slow_path
+                    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
+                     node, next0, now, thread_index, &s0);
+                  goto trace00;
+                }
+            }
+          else
+            {
+              if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
+                {
+                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
+                  goto trace00;
+                }
+            }
+
+          key0.addr = ip0->src_address;
+          key0.port = udp0->src_port;
+          key0.protocol = proto0;
+          key0.fib_index = rx_fib_index0;
+
+          kv0.key = key0.as_u64;
+
+          if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
+            {
+              if (is_slow_path)
+                {
+                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
+                      ip0, proto0, rx_fib_index0)) && !is_output_feature)
+                    goto trace00;
+
+                  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
+                                     &s0, node, next0, thread_index);
+                  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
+                    goto trace00;
+                }
+              else
+                {
+                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
+                  goto trace00;
+                }
+            }
+          else
+            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                    value0.value);
+
+          old_addr0 = ip0->src_address.as_u32;
+          ip0->src_address = s0->out2in.addr;
+          new_addr0 = ip0->src_address.as_u32;
+          if (!is_output_feature)
+            vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              old_port0 = tcp0->src_port;
+              tcp0->src_port = s0->out2in.port;
+              new_port0 = tcp0->src_port;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              old_port0 = udp0->src_port;
+              udp0->src_port = s0->out2in.port;
+              udp0->checksum = 0;
+            }
+
+          /* Hairpinning */
+          if (!is_output_feature)
+            snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
+
+          /* Accounting */
+          s0->last_heard = now;
+          s0->total_pkts++;
+          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
+          /* Per-user LRU list maintenance for dynamic translation */
+          if (!snat_is_session_static (s0))
+            {
+              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                                 s0->per_user_index);
+              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                                  s0->per_user_list_head_index,
+                                  s0->per_user_index);
+            }
+        trace00:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->is_slow_path = is_slow_path;
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+                  t->session_index = ~0;
+              if (s0)
+                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          if (is_output_feature)
+            iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
+
+          ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
+                 iph_offset1);
+
+          udp1 = ip4_next_header (ip1);
+          tcp1 = (tcp_header_t *) udp1;
+          icmp1 = (icmp46_header_t *) udp1;
+
+          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
+	  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
+                                   sw_if_index1);
+
+          if (PREDICT_FALSE(ip1->ttl == 1))
+            {
+              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace01;
+            }
+
+          proto1 = ip_proto_to_snat_proto (ip1->protocol);
+
+          /* Next configured feature, probably ip4-lookup */
+          if (is_slow_path)
+            {
+              if (PREDICT_FALSE (proto1 == ~0))
+                {
+                  snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
+                                             thread_index, now, vm);
+                  goto trace01;
+                }
+
+              if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
+                {
+                  next1 = icmp_in2out_slow_path
+                    (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
+                     next1, now, thread_index, &s1);
+                  goto trace01;
+                }
+            }
+          else
+            {
+              if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
+                {
+                  next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
+                  goto trace01;
+                }
+            }
+
+          key1.addr = ip1->src_address;
+          key1.port = udp1->src_port;
+          key1.protocol = proto1;
+          key1.fib_index = rx_fib_index1;
+
+          kv1.key = key1.as_u64;
+
+            if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
+            {
+              if (is_slow_path)
+                {
+                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
+                      ip1, proto1, rx_fib_index1)) && !is_output_feature)
+                    goto trace01;
+
+                  next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
+                                     &s1, node, next1, thread_index);
+                  if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
+                    goto trace01;
+                }
+              else
+                {
+                  next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
+                  goto trace01;
+                }
+            }
+          else
+            s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                    value1.value);
+
+          old_addr1 = ip1->src_address.as_u32;
+          ip1->src_address = s1->out2in.addr;
+          new_addr1 = ip1->src_address.as_u32;
+          if (!is_output_feature)
+            vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
+
+          sum1 = ip1->checksum;
+          sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip1->checksum = ip_csum_fold (sum1);
+
+          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
+            {
+              old_port1 = tcp1->src_port;
+              tcp1->src_port = s1->out2in.port;
+              new_port1 = tcp1->src_port;
+
+              sum1 = tcp1->checksum;
+              sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+              sum1 = ip_csum_update (sum1, old_port1, new_port1,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp1->checksum = ip_csum_fold(sum1);
+            }
+          else
+            {
+              old_port1 = udp1->src_port;
+              udp1->src_port = s1->out2in.port;
+              udp1->checksum = 0;
+            }
+
+          /* Hairpinning */
+          if (!is_output_feature)
+            snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
+
+          /* Accounting */
+          s1->last_heard = now;
+          s1->total_pkts++;
+          s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
+          /* Per-user LRU list maintenance for dynamic translation */
+          if (!snat_is_session_static (s1))
+            {
+              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                                 s1->per_user_index);
+              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                                  s1->per_user_list_head_index,
+                                  s1->per_user_index);
+            }
+        trace01:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b1->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b1, sizeof (*t));
+              t->sw_if_index = sw_if_index1;
+              t->next_index = next1;
+              t->session_index = ~0;
+              if (s1)
+                t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
+            }
+
+          pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueues, maybe switch current next frame */
+          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+                                           to_next, n_left_to_next,
+                                           bi0, bi1, next0, next1);
+        }
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0;
+          u32 sw_if_index0;
+          ip4_header_t * ip0;
+          ip_csum_t sum0;
+          u32 new_addr0, old_addr0;
+          u16 old_port0, new_port0;
+          udp_header_t * udp0;
+          tcp_header_t * tcp0;
+          icmp46_header_t * icmp0;
+          snat_session_key_t key0;
+          u32 rx_fib_index0;
+          u32 proto0;
+          snat_session_t * s0 = 0;
+          clib_bihash_kv_8_8_t kv0, value0;
+          u32 iph_offset0 = 0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
+
+          if (is_output_feature)
+            iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
+
+          ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
+                 iph_offset0);
+
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+          icmp0 = (icmp46_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
+                                   sw_if_index0);
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace0;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          /* Next configured feature, probably ip4-lookup */
+          if (is_slow_path)
+            {
+              if (PREDICT_FALSE (proto0 == ~0))
+                {
+                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
+                                             thread_index, now, vm);
+                  goto trace0;
+                }
+
+              if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+                {
+                  next0 = icmp_in2out_slow_path
+                    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                     next0, now, thread_index, &s0);
+                  goto trace0;
+                }
+            }
+          else
+            {
+              if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
+                {
+                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
+                  goto trace0;
+                }
+            }
+
+          key0.addr = ip0->src_address;
+          key0.port = udp0->src_port;
+          key0.protocol = proto0;
+          key0.fib_index = rx_fib_index0;
+
+          kv0.key = key0.as_u64;
+
+          if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
+            {
+              if (is_slow_path)
+                {
+                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
+                      ip0, proto0, rx_fib_index0)) && !is_output_feature)
+                    goto trace0;
+
+                  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
+                                     &s0, node, next0, thread_index);
+
+                  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
+                    goto trace0;
+                }
+              else
+                {
+                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
+                  goto trace0;
+                }
+            }
+          else
+            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                    value0.value);
+
+          old_addr0 = ip0->src_address.as_u32;
+          ip0->src_address = s0->out2in.addr;
+          new_addr0 = ip0->src_address.as_u32;
+          if (!is_output_feature)
+            vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              old_port0 = tcp0->src_port;
+              tcp0->src_port = s0->out2in.port;
+              new_port0 = tcp0->src_port;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              old_port0 = udp0->src_port;
+              udp0->src_port = s0->out2in.port;
+              udp0->checksum = 0;
+            }
+
+          /* Hairpinning */
+          if (!is_output_feature)
+            snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
+
+          /* Accounting */
+          s0->last_heard = now;
+          s0->total_pkts++;
+          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
+          /* Per-user LRU list maintenance for dynamic translation */
+          if (!snat_is_session_static (s0))
+            {
+              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                                 s0->per_user_index);
+              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                                  s0->per_user_list_head_index,
+                                  s0->per_user_index);
+            }
+
+        trace0:
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->is_slow_path = is_slow_path;
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+                  t->session_index = ~0;
+              if (s0)
+                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, stats_node_index,
+                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+static uword
+snat_in2out_fast_path_fn (vlib_main_t * vm,
+                          vlib_node_runtime_t * node,
+                          vlib_frame_t * frame)
+{
+  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_node) = {
+  .function = snat_in2out_fast_path_fn,
+  .name = "nat44-in2out",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
+
+static uword
+snat_in2out_output_fast_path_fn (vlib_main_t * vm,
+                                 vlib_node_runtime_t * node,
+                                 vlib_frame_t * frame)
+{
+  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_output_node) = {
+  .function = snat_in2out_output_fast_path_fn,
+  .name = "nat44-in2out-output",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
+                              snat_in2out_output_fast_path_fn);
+
+static uword
+snat_in2out_slow_path_fn (vlib_main_t * vm,
+                          vlib_node_runtime_t * node,
+                          vlib_frame_t * frame)
+{
+  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
+  .function = snat_in2out_slow_path_fn,
+  .name = "nat44-in2out-slowpath",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
+                              snat_in2out_slow_path_fn);
+
+static uword
+snat_in2out_output_slow_path_fn (vlib_main_t * vm,
+                                 vlib_node_runtime_t * node,
+                                 vlib_frame_t * frame)
+{
+  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
+  .function = snat_in2out_output_slow_path_fn,
+  .name = "nat44-in2out-output-slowpath",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
+                              snat_in2out_output_slow_path_fn);
+
+/**************************/
+/*** deterministic mode ***/
+/**************************/
+static uword
+snat_det_in2out_node_fn (vlib_main_t * vm,
+                         vlib_node_runtime_t * node,
+                         vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_in2out_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+  u32 now = (u32) vlib_time_now (vm);
+  u32 thread_index = vlib_get_thread_index ();
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+        {
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+          u32 next0, next1;
+          u32 sw_if_index0, sw_if_index1;
+          ip4_header_t * ip0, * ip1;
+          ip_csum_t sum0, sum1;
+          ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
+          u16 old_port0, new_port0, lo_port0, i0;
+          u16 old_port1, new_port1, lo_port1, i1;
+          udp_header_t * udp0, * udp1;
+          tcp_header_t * tcp0, * tcp1;
+          u32 proto0, proto1;
+          snat_det_out_key_t key0, key1;
+          snat_det_map_t * dm0, * dm1;
+          snat_det_session_t * ses0 = 0, * ses1 = 0;
+          u32 rx_fib_index0, rx_fib_index1;
+          icmp46_header_t * icmp0, * icmp1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+          /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
+          next1 = SNAT_IN2OUT_NEXT_LOOKUP;
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace0;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+              icmp0 = (icmp46_header_t *) udp0;
+
+              next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
+                                  rx_fib_index0, node, next0, thread_index,
+                                  &ses0, &dm0);
+              goto trace0;
+            }
+
+          dm0 = snat_det_map_by_user(sm, &ip0->src_address);
+          if (PREDICT_FALSE(!dm0))
+            {
+              clib_warning("no match for internal host %U",
+                           format_ip4_address, &ip0->src_address);
+              next0 = SNAT_IN2OUT_NEXT_DROP;
+              b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
+              goto trace0;
+            }
+
+          snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
+
+          key0.ext_host_addr = ip0->dst_address;
+          key0.ext_host_port = tcp0->dst;
+
+          ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
+          if (PREDICT_FALSE(!ses0))
+            {
+              for (i0 = 0; i0 < dm0->ports_per_host; i0++)
+                {
+                  key0.out_port = clib_host_to_net_u16 (lo_port0 +
+                    ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
+
+                  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
+                    continue;
+
+                  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
+                  break;
+                }
+              if (PREDICT_FALSE(!ses0))
+                {
+                  /* too many sessions for user, send ICMP error packet */
+
+                  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+                  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
+                                               ICMP4_destination_unreachable_destination_unreachable_host,
+                                               0);
+                  next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+                  goto trace0;
+                }
+            }
+
+          new_port0 = ses0->out.out_port;
+
+          old_addr0.as_u32 = ip0->src_address.as_u32;
+          ip0->src_address.as_u32 = new_addr0.as_u32;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              if (tcp0->flags & TCP_FLAG_SYN)
+                ses0->state = SNAT_SESSION_TCP_SYN_SENT;
+              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
+                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
+              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
+                ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
+              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
+                snat_det_ses_close(dm0, ses0);
+              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
+                ses0->state = SNAT_SESSION_TCP_LAST_ACK;
+              else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
+                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
+
+              old_port0 = tcp0->src;
+              tcp0->src = new_port0;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              ses0->state = SNAT_SESSION_UDP_ACTIVE;
+              old_port0 = udp0->src_port;
+              udp0->src_port = new_port0;
+              udp0->checksum = 0;
+            }
+
+          switch(ses0->state)
+            {
+            case SNAT_SESSION_UDP_ACTIVE:
+                ses0->expire = now + sm->udp_timeout;
+                break;
+            case SNAT_SESSION_TCP_SYN_SENT:
+            case SNAT_SESSION_TCP_FIN_WAIT:
+            case SNAT_SESSION_TCP_CLOSE_WAIT:
+            case SNAT_SESSION_TCP_LAST_ACK:
+                ses0->expire = now + sm->tcp_transitory_timeout;
+                break;
+            case SNAT_SESSION_TCP_ESTABLISHED:
+                ses0->expire = now + sm->tcp_established_timeout;
+                break;
+            }
+
+        trace0:
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->is_slow_path = 0;
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+              t->session_index = ~0;
+              if (ses0)
+                t->session_index = ses0 - dm0->sessions;
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          ip1 = vlib_buffer_get_current (b1);
+          udp1 = ip4_next_header (ip1);
+          tcp1 = (tcp_header_t *) udp1;
+
+          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE(ip1->ttl == 1))
+            {
+              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace1;
+            }
+
+          proto1 = ip_proto_to_snat_proto (ip1->protocol);
+
+          if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
+            {
+              rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
+              icmp1 = (icmp46_header_t *) udp1;
+
+              next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
+                                  rx_fib_index1, node, next1, thread_index,
+                                  &ses1, &dm1);
+              goto trace1;
+            }
+
+          dm1 = snat_det_map_by_user(sm, &ip1->src_address);
+          if (PREDICT_FALSE(!dm1))
+            {
+              clib_warning("no match for internal host %U",
+                           format_ip4_address, &ip0->src_address);
+              next1 = SNAT_IN2OUT_NEXT_DROP;
+              b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
+              goto trace1;
+            }
+
+          snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
+
+          key1.ext_host_addr = ip1->dst_address;
+          key1.ext_host_port = tcp1->dst;
+
+          ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
+          if (PREDICT_FALSE(!ses1))
+            {
+              for (i1 = 0; i1 < dm1->ports_per_host; i1++)
+                {
+                  key1.out_port = clib_host_to_net_u16 (lo_port1 +
+                    ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
+
+                  if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
+                    continue;
+
+                  ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
+                  break;
+                }
+              if (PREDICT_FALSE(!ses1))
+                {
+                  /* too many sessions for user, send ICMP error packet */
+
+                  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+                  icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
+                                               ICMP4_destination_unreachable_destination_unreachable_host,
+                                               0);
+                  next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+                  goto trace1;
+                }
+            }
+
+          new_port1 = ses1->out.out_port;
+
+          old_addr1.as_u32 = ip1->src_address.as_u32;
+          ip1->src_address.as_u32 = new_addr1.as_u32;
+          vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+
+          sum1 = ip1->checksum;
+          sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip1->checksum = ip_csum_fold (sum1);
+
+          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
+            {
+              if (tcp1->flags & TCP_FLAG_SYN)
+                ses1->state = SNAT_SESSION_TCP_SYN_SENT;
+              else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
+                ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
+              else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
+                ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
+              else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
+                snat_det_ses_close(dm1, ses1);
+              else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
+                ses1->state = SNAT_SESSION_TCP_LAST_ACK;
+              else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
+                ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
+
+              old_port1 = tcp1->src;
+              tcp1->src = new_port1;
+
+              sum1 = tcp1->checksum;
+              sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+              sum1 = ip_csum_update (sum1, old_port1, new_port1,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp1->checksum = ip_csum_fold(sum1);
+            }
+          else
+            {
+              ses1->state = SNAT_SESSION_UDP_ACTIVE;
+              old_port1 = udp1->src_port;
+              udp1->src_port = new_port1;
+              udp1->checksum = 0;
+            }
+
+          switch(ses1->state)
+            {
+            case SNAT_SESSION_UDP_ACTIVE:
+                ses1->expire = now + sm->udp_timeout;
+                break;
+            case SNAT_SESSION_TCP_SYN_SENT:
+            case SNAT_SESSION_TCP_FIN_WAIT:
+            case SNAT_SESSION_TCP_CLOSE_WAIT:
+            case SNAT_SESSION_TCP_LAST_ACK:
+                ses1->expire = now + sm->tcp_transitory_timeout;
+                break;
+            case SNAT_SESSION_TCP_ESTABLISHED:
+                ses1->expire = now + sm->tcp_established_timeout;
+                break;
+            }
+
+        trace1:
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b1->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b1, sizeof (*t));
+              t->is_slow_path = 0;
+              t->sw_if_index = sw_if_index1;
+              t->next_index = next1;
+              t->session_index = ~0;
+              if (ses1)
+                t->session_index = ses1 - dm1->sessions;
+            }
+
+          pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueues, maybe switch current next frame */
+          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+                                           to_next, n_left_to_next,
+                                           bi0, bi1, next0, next1);
+         }
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0;
+          u32 sw_if_index0;
+          ip4_header_t * ip0;
+          ip_csum_t sum0;
+          ip4_address_t new_addr0, old_addr0;
+          u16 old_port0, new_port0, lo_port0, i0;
+          udp_header_t * udp0;
+          tcp_header_t * tcp0;
+          u32 proto0;
+          snat_det_out_key_t key0;
+          snat_det_map_t * dm0;
+          snat_det_session_t * ses0 = 0;
+          u32 rx_fib_index0;
+          icmp46_header_t * icmp0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace00;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+              icmp0 = (icmp46_header_t *) udp0;
+
+              next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
+                                  rx_fib_index0, node, next0, thread_index,
+                                  &ses0, &dm0);
+              goto trace00;
+            }
+
+          dm0 = snat_det_map_by_user(sm, &ip0->src_address);
+          if (PREDICT_FALSE(!dm0))
+            {
+              clib_warning("no match for internal host %U",
+                           format_ip4_address, &ip0->src_address);
+              next0 = SNAT_IN2OUT_NEXT_DROP;
+              b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
+              goto trace00;
+            }
+
+          snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
+
+          key0.ext_host_addr = ip0->dst_address;
+          key0.ext_host_port = tcp0->dst;
+
+          ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
+          if (PREDICT_FALSE(!ses0))
+            {
+              for (i0 = 0; i0 < dm0->ports_per_host; i0++)
+                {
+                  key0.out_port = clib_host_to_net_u16 (lo_port0 +
+                    ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
+
+                  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
+                    continue;
+
+                  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
+                  break;
+                }
+              if (PREDICT_FALSE(!ses0))
+                {
+                  /* too many sessions for user, send ICMP error packet */
+
+                  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+                  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
+                                               ICMP4_destination_unreachable_destination_unreachable_host,
+                                               0);
+                  next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+                  goto trace00;
+                }
+            }
+
+          new_port0 = ses0->out.out_port;
+
+          old_addr0.as_u32 = ip0->src_address.as_u32;
+          ip0->src_address.as_u32 = new_addr0.as_u32;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              if (tcp0->flags & TCP_FLAG_SYN)
+                ses0->state = SNAT_SESSION_TCP_SYN_SENT;
+              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
+                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
+              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
+                ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
+              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
+                snat_det_ses_close(dm0, ses0);
+              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
+                ses0->state = SNAT_SESSION_TCP_LAST_ACK;
+              else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
+                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
+
+              old_port0 = tcp0->src;
+              tcp0->src = new_port0;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              ses0->state = SNAT_SESSION_UDP_ACTIVE;
+              old_port0 = udp0->src_port;
+              udp0->src_port = new_port0;
+              udp0->checksum = 0;
+            }
+
+          switch(ses0->state)
+            {
+            case SNAT_SESSION_UDP_ACTIVE:
+                ses0->expire = now + sm->udp_timeout;
+                break;
+            case SNAT_SESSION_TCP_SYN_SENT:
+            case SNAT_SESSION_TCP_FIN_WAIT:
+            case SNAT_SESSION_TCP_CLOSE_WAIT:
+            case SNAT_SESSION_TCP_LAST_ACK:
+                ses0->expire = now + sm->tcp_transitory_timeout;
+                break;
+            case SNAT_SESSION_TCP_ESTABLISHED:
+                ses0->expire = now + sm->tcp_established_timeout;
+                break;
+            }
+
+        trace00:
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->is_slow_path = 0;
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+              t->session_index = ~0;
+              if (ses0)
+                t->session_index = ses0 - dm0->sessions;
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, snat_det_in2out_node.index,
+                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_det_in2out_node) = {
+  .function = snat_det_in2out_node_fn,
+  .name = "nat44-det-in2out",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = 3,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
+
+/**
+ * Get address and port values to be used for ICMP packet translation
+ * and create session if needed
+ *
+ * @param[in,out] sm             NAT main
+ * @param[in,out] node           NAT node runtime
+ * @param[in] thread_index       thread index
+ * @param[in,out] b0             buffer containing packet to be translated
+ * @param[out] p_proto           protocol used for matching
+ * @param[out] p_value           address and port after NAT translation
+ * @param[out] p_dont_translate  if packet should not be translated
+ * @param d                      optional parameter
+ * @param e                      optional parameter
+ */
+u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
+                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                          snat_session_key_t *p_value,
+                          u8 *p_dont_translate, void *d, void *e)
+{
+  ip4_header_t *ip0;
+  icmp46_header_t *icmp0;
+  u32 sw_if_index0;
+  u32 rx_fib_index0;
+  u8 protocol;
+  snat_det_out_key_t key0;
+  u8 dont_translate = 0;
+  u32 next0 = ~0;
+  icmp_echo_header_t *echo0, *inner_echo0 = 0;
+  ip4_header_t *inner_ip0;
+  void *l4_header = 0;
+  icmp46_header_t *inner_icmp0;
+  snat_det_map_t * dm0 = 0;
+  ip4_address_t new_addr0;
+  u16 lo_port0, i0;
+  snat_det_session_t * ses0 = 0;
+  ip4_address_t in_addr;
+  u16 in_port;
+
+  ip0 = vlib_buffer_get_current (b0);
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  echo0 = (icmp_echo_header_t *)(icmp0+1);
+  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      protocol = SNAT_PROTOCOL_ICMP;
+      in_addr = ip0->src_address;
+      in_port = echo0->identifier;
+    }
+  else
+    {
+      inner_ip0 = (ip4_header_t *)(echo0+1);
+      l4_header = ip4_next_header (inner_ip0);
+      protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
+      in_addr = inner_ip0->dst_address;
+      switch (protocol)
+        {
+        case SNAT_PROTOCOL_ICMP:
+          inner_icmp0 = (icmp46_header_t*)l4_header;
+          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
+          in_port = inner_echo0->identifier;
+          break;
+        case SNAT_PROTOCOL_UDP:
+        case SNAT_PROTOCOL_TCP:
+          in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
+          break;
+        default:
+          b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          goto out;
+        }
+    }
+
+  dm0 = snat_det_map_by_user(sm, &in_addr);
+  if (PREDICT_FALSE(!dm0))
+    {
+      clib_warning("no match for internal host %U",
+                   format_ip4_address, &in_addr);
+      if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
+          IP_PROTOCOL_ICMP, rx_fib_index0)))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
+      goto out;
+    }
+
+  snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
+
+  key0.ext_host_addr = ip0->dst_address;
+  key0.ext_host_port = 0;
+
+  ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
+  if (PREDICT_FALSE(!ses0))
+    {
+      if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
+          IP_PROTOCOL_ICMP, rx_fib_index0)))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+      if (icmp0->type != ICMP4_echo_request)
+        {
+          b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          goto out;
+        }
+      for (i0 = 0; i0 < dm0->ports_per_host; i0++)
+        {
+          key0.out_port = clib_host_to_net_u16 (lo_port0 +
+            ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
+
+          if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
+            continue;
+
+          ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
+          break;
+        }
+      if (PREDICT_FALSE(!ses0))
+        {
+          next0 = SNAT_IN2OUT_NEXT_DROP;
+          b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
+          goto out;
+        }
+    }
+
+  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
+                    !icmp_is_error_message (icmp0)))
+    {
+      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
+      next0 = SNAT_IN2OUT_NEXT_DROP;
+      goto out;
+    }
+
+  u32 now = (u32) vlib_time_now (sm->vlib_main);
+
+  ses0->state = SNAT_SESSION_ICMP_ACTIVE;
+  ses0->expire = now + sm->icmp_timeout;
+
+out:
+  *p_proto = protocol;
+  if (ses0)
+    {
+      p_value->addr = new_addr0;
+      p_value->fib_index = sm->outside_fib_index;
+      p_value->port = ses0->out.out_port;
+    }
+  *p_dont_translate = dont_translate;
+  if (d)
+    *(snat_det_session_t**)d = ses0;
+  if (e)
+    *(snat_det_map_t**)e = dm0;
+  return next0;
+}
+
+/**********************/
+/*** worker handoff ***/
+/**********************/
+static inline uword
+snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
+                                      vlib_node_runtime_t * node,
+                                      vlib_frame_t * frame,
+                                      u8 is_output)
+{
+  snat_main_t *sm = &snat_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  u32 n_left_from, *from, *to_next = 0;
+  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
+  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
+    = 0;
+  vlib_frame_queue_elt_t *hf = 0;
+  vlib_frame_t *f = 0;
+  int i;
+  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
+  u32 next_worker_index = 0;
+  u32 current_worker_index = ~0;
+  u32 thread_index = vlib_get_thread_index ();
+  u32 fq_index;
+  u32 to_node_index;
+
+  ASSERT (vec_len (sm->workers));
+
+  if (is_output)
+    {
+      fq_index = sm->fq_in2out_output_index;
+      to_node_index = sm->in2out_output_node_index;
+    }
+  else
+    {
+      fq_index = sm->fq_in2out_index;
+      to_node_index = sm->in2out_node_index;
+    }
+
+  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
+    {
+      vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
+
+      vec_validate_init_empty (congested_handoff_queue_by_worker_index,
+			       sm->first_worker_index + sm->num_workers - 1,
+			       (vlib_frame_queue_t *) (~0));
+    }
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+
+  while (n_left_from > 0)
+    {
+      u32 bi0;
+      vlib_buffer_t *b0;
+      u32 sw_if_index0;
+      u32 rx_fib_index0;
+      ip4_header_t * ip0;
+      u8 do_handoff;
+
+      bi0 = from[0];
+      from += 1;
+      n_left_from -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
+      rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+
+      ip0 = vlib_buffer_get_current (b0);
+
+      next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
+
+      if (PREDICT_FALSE (next_worker_index != thread_index))
+        {
+          do_handoff = 1;
+
+          if (next_worker_index != current_worker_index)
+            {
+              if (hf)
+                hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
+
+              hf = vlib_get_worker_handoff_queue_elt (fq_index,
+                                                      next_worker_index,
+                                                      handoff_queue_elt_by_worker_index);
+
+              n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
+              to_next_worker = &hf->buffer_index[hf->n_vectors];
+              current_worker_index = next_worker_index;
+            }
+
+          /* enqueue to correct worker thread */
+          to_next_worker[0] = bi0;
+          to_next_worker++;
+          n_left_to_next_worker--;
+
+          if (n_left_to_next_worker == 0)
+            {
+              hf->n_vectors = VLIB_FRAME_SIZE;
+              vlib_put_frame_queue_elt (hf);
+              current_worker_index = ~0;
+              handoff_queue_elt_by_worker_index[next_worker_index] = 0;
+              hf = 0;
+            }
+        }
+      else
+        {
+          do_handoff = 0;
+          /* if this is 1st frame */
+          if (!f)
+            {
+              f = vlib_get_frame_to_node (vm, to_node_index);
+              to_next = vlib_frame_vector_args (f);
+            }
+
+          to_next[0] = bi0;
+          to_next += 1;
+          f->n_vectors++;
+        }
+
+      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	{
+          snat_in2out_worker_handoff_trace_t *t =
+            vlib_add_trace (vm, node, b0, sizeof (*t));
+          t->next_worker_index = next_worker_index;
+          t->do_handoff = do_handoff;
+        }
+    }
+
+  if (f)
+    vlib_put_frame_to_node (vm, to_node_index, f);
+
+  if (hf)
+    hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
+
+  /* Ship frames to the worker nodes */
+  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
+    {
+      if (handoff_queue_elt_by_worker_index[i])
+	{
+	  hf = handoff_queue_elt_by_worker_index[i];
+	  /*
+	   * It works better to let the handoff node
+	   * rate-adapt, always ship the handoff queue element.
+	   */
+	  if (1 || hf->n_vectors == hf->last_n_vectors)
+	    {
+	      vlib_put_frame_queue_elt (hf);
+	      handoff_queue_elt_by_worker_index[i] = 0;
+	    }
+	  else
+	    hf->last_n_vectors = hf->n_vectors;
+	}
+      congested_handoff_queue_by_worker_index[i] =
+	(vlib_frame_queue_t *) (~0);
+    }
+  hf = 0;
+  current_worker_index = ~0;
+  return frame->n_vectors;
+}
+
+static uword
+snat_in2out_worker_handoff_fn (vlib_main_t * vm,
+                               vlib_node_runtime_t * node,
+                               vlib_frame_t * frame)
+{
+  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
+  .function = snat_in2out_worker_handoff_fn,
+  .name = "nat44-in2out-worker-handoff",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_worker_handoff_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
+                              snat_in2out_worker_handoff_fn);
+
+static uword
+snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
+                                      vlib_node_runtime_t * node,
+                                      vlib_frame_t * frame)
+{
+  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
+}
+
+VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
+  .function = snat_in2out_output_worker_handoff_fn,
+  .name = "nat44-in2out-output-worker-handoff",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_worker_handoff_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
+                              snat_in2out_output_worker_handoff_fn);
+
+static_always_inline int
+is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
+{
+  snat_address_t * ap;
+  clib_bihash_kv_8_8_t kv, value;
+  snat_session_key_t m_key;
+
+  vec_foreach (ap, sm->addresses)
+    {
+      if (ap->addr.as_u32 == dst_addr->as_u32)
+        return 1;
+    }
+
+  m_key.addr.as_u32 = dst_addr->as_u32;
+  m_key.fib_index = sm->outside_fib_index;
+  m_key.port = 0;
+  m_key.protocol = 0;
+  kv.key = m_key.as_u64;
+  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+    return 1;
+
+  return 0;
+}
+
+static uword
+snat_hairpin_dst_fn (vlib_main_t * vm,
+                     vlib_node_runtime_t * node,
+                     vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_in2out_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0;
+          ip4_header_t * ip0;
+          u32 proto0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
+          ip0 = vlib_buffer_get_current (b0);
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          vnet_buffer (b0)->snat.flags = 0;
+          if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
+            {
+              if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
+                {
+                  udp_header_t * udp0 = ip4_next_header (ip0);
+                  tcp_header_t * tcp0 = (tcp_header_t *) udp0;
+
+                  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
+                }
+              else if (proto0 == SNAT_PROTOCOL_ICMP)
+                {
+                  icmp46_header_t * icmp0 = ip4_next_header (ip0);
+
+                  snat_icmp_hairpinning (sm, b0, ip0, icmp0);
+                }
+              else
+                {
+                  snat_hairpinning_unknown_proto (sm, b0, ip0);
+                }
+
+              vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
+              clib_warning("is hairpinning");
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+         }
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
+                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
+  .function = snat_hairpin_dst_fn,
+  .name = "nat44-hairpin-dst",
+  .vector_size = sizeof (u32),
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+  .n_next_nodes = 2,
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
+                              snat_hairpin_dst_fn);
+
+static uword
+snat_hairpin_src_fn (vlib_main_t * vm,
+                     vlib_node_runtime_t * node,
+                     vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_in2out_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t *sm = &snat_main;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+          next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
+
+          if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) & SNAT_FLAG_HAIRPINNING))
+            {
+              if (PREDICT_TRUE (sm->num_workers > 1))
+                next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
+              else
+                next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+         }
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
+                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
+  .function = snat_hairpin_src_fn,
+  .name = "nat44-hairpin-src",
+  .vector_size = sizeof (u32),
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
+  .next_nodes = {
+     [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
+     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
+     [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
+     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
+                              snat_hairpin_src_fn);
+
+static uword
+snat_in2out_fast_static_map_fn (vlib_main_t * vm,
+                                vlib_node_runtime_t * node,
+                                vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_in2out_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+  u32 stats_node_index;
+
+  stats_node_index = snat_in2out_fast_node.index;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0;
+          u32 sw_if_index0;
+          ip4_header_t * ip0;
+          ip_csum_t sum0;
+          u32 new_addr0, old_addr0;
+          u16 old_port0, new_port0;
+          udp_header_t * udp0;
+          tcp_header_t * tcp0;
+          icmp46_header_t * icmp0;
+          snat_session_key_t key0, sm0;
+          u32 proto0;
+          u32 rx_fib_index0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+          icmp0 = (icmp46_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+	  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
+              goto trace0;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE (proto0 == ~0))
+              goto trace0;
+
+          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
+                                  rx_fib_index0, node, next0, ~0, 0, 0);
+              goto trace0;
+            }
+
+          key0.addr = ip0->src_address;
+          key0.port = udp0->src_port;
+          key0.fib_index = rx_fib_index0;
+
+          if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
+            {
+              b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
+              next0= SNAT_IN2OUT_NEXT_DROP;
+              goto trace0;
+            }
+
+          new_addr0 = sm0.addr.as_u32;
+          new_port0 = sm0.port;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+          old_addr0 = ip0->src_address.as_u32;
+          ip0->src_address.as_u32 = new_addr0;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                 ip4_header_t,
+                                 src_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_FALSE(new_port0 != udp0->dst_port))
+            {
+              if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+                {
+                  old_port0 = tcp0->src_port;
+                  tcp0->src_port = new_port0;
+
+                  sum0 = tcp0->checksum;
+                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                         ip4_header_t,
+                                         dst_address /* changed member */);
+                  sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                         ip4_header_t /* cheat */,
+                                         length /* changed member */);
+                  tcp0->checksum = ip_csum_fold(sum0);
+                }
+              else
+                {
+                  old_port0 = udp0->src_port;
+                  udp0->src_port = new_port0;
+                  udp0->checksum = 0;
+                }
+            }
+          else
+            {
+              if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+                {
+                  sum0 = tcp0->checksum;
+                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                         ip4_header_t,
+                                         dst_address /* changed member */);
+                  tcp0->checksum = ip_csum_fold(sum0);
+                }
+            }
+
+          /* Hairpinning */
+          snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
+
+        trace0:
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_in2out_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+            }
+
+          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, stats_node_index,
+                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+
+VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
+  .function = snat_in2out_fast_static_map_fn,
+  .name = "nat44-in2out-fast",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_in2out_fast_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
+  .error_strings = snat_in2out_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
+    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
+    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);
diff --git a/src/plugins/nat/nat.api b/src/plugins/nat/nat.api
new file mode 100644
index 00000000..7245cb07
--- /dev/null
+++ b/src/plugins/nat/nat.api
@@ -0,0 +1,1513 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file nat.api
+ * @brief VPP control-plane API messages.
+ *
+ * This file defines VPP control-plane API messages which are generally
+ * called through a shared memory interface.
+ */
+
+/*
+ * Old "snat" APIs, will be deprecated after 17.10
+ */
+
+/** \brief Add/del NAT44 address range
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param first_ip_address - first IP address
+    @param last_ip_address - last IP address
+    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
+    @param is_add - 1 if add, 0 if delete
+*/
+autoreply define snat_add_address_range {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u8 first_ip_address[16];
+  u8 last_ip_address[16];
+  u32 vrf_id;
+  u8 is_add;
+};
+
+/** \brief Dump NAT44 addresses
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_address_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 address details response
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param ip_address - IP address
+    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
+*/
+define snat_address_details {
+  u32 context;
+  u8 is_ip4;
+  u8 ip_address[16];
+  u32 vrf_id;
+};
+
+/** \brief Enable/disable NAT44 feature on the interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+autoreply define snat_interface_add_del_feature {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump interfaces with NAT44 feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_interface_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 interface details response
+    @param context - sender context, to match reply w/ request
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+define snat_interface_details {
+  u32 context;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Enable/disbale NAT44 as an interface output feature (postrouting
+           in2out translation)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+autoreply define snat_interface_add_del_output_feature {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump interfaces with NAT44 output feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_interface_output_feature_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 interface with output feature details response
+    @param context - sender context, to match reply w/ request
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+define snat_interface_output_feature_details {
+  u32 context;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Add/delete NAT44 static mapping
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_ip4 - 1 if address type is IPv4
+    @param addr_only - 1 if address only mapping
+    @param local_ip_address - local IP address
+    @param external_ip_address - external IP address
+    @param protocol - IP protocol
+    @param local_port - local port number
+    @param external_port - external port number
+    @param external_sw_if_index - external interface (if set
+                                  external_ip_address is ignored, ~0 means not
+                                  used)
+    @param vfr_id - VRF ID
+*/
+autoreply define snat_add_static_mapping {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_ip4;
+  u8 addr_only;
+  u8 local_ip_address[16];
+  u8 external_ip_address[16];
+  u8 protocol;
+  u16 local_port;
+  u16 external_port;
+  u32 external_sw_if_index;
+  u32 vrf_id;
+};
+
+/** \brief Dump NAT44 static mappings
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_static_mapping_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 static mapping details response
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param addr_only - 1 if address only mapping
+    @param local_ip_address - local IP address
+    @param external_ip_address - external IP address
+    @param protocol - IP protocol
+    @param local_port - local port number
+    @param external_port - external port number
+    @param external_sw_if_index - external interface
+    @param vfr_id - VRF ID
+*/
+define snat_static_mapping_details {
+  u32 context;
+  u8 is_ip4;
+  u8 addr_only;
+  u8 local_ip_address[16];
+  u8 external_ip_address[16];
+  u8 protocol;
+  u16 local_port;
+  u16 external_port;
+  u32 external_sw_if_index;
+  u32 vrf_id;
+};
+
+/** \brief Control ping from client to api server request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_control_ping
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Control ping from the client to the server response
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param vpe_pid - the pid of the vpe, returned by the server
+*/
+define snat_control_ping_reply
+{
+  u32 context;
+  i32 retval;
+  u32 client_index;
+  u32 vpe_pid;
+};
+
+/** \brief Show NAT plugin startup config
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_show_config
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Show NAT plugin startup config reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param static_mapping_only - if 1 dynamic translations disabled
+    @param static_mapping_connection_tracking - if 1 create session data
+    @param deterministic - if 1 deterministic mapping
+    @param translation_buckets - number of translation hash buckets
+    @param translation_memory_size - translation hash memory size
+    @param user_buckets - number of user hash buckets
+    @param user_memory_size - user hash memory size
+    @param max_translations_per_user - maximum number of translations per user
+    @param outside_vrf_id - outside VRF id
+    @param inside_vrf_id - default inside VRF id
+*/
+define snat_show_config_reply
+{
+  u32 context;
+  i32 retval;
+  u8 static_mapping_only;
+  u8 static_mapping_connection_tracking;
+  u8 deterministic;
+  u32 translation_buckets;
+  u32 translation_memory_size;
+  u32 user_buckets;
+  u32 user_memory_size;
+  u32 max_translations_per_user;
+  u32 outside_vrf_id;
+  u32 inside_vrf_id;
+};
+
+/** \brief Set NAT workers
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param worker_mask - NAT workers mask
+*/
+autoreply define snat_set_workers {
+  u32 client_index;
+  u32 context;
+  u64 worker_mask;
+};
+
+/** \brief Dump NAT workers
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_worker_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT workers details response
+    @param context - sender context, to match reply w/ request
+    @param worker_index - worker index
+    @param lcore_id - lcore ID
+    @param name - worker name
+*/
+define snat_worker_details {
+  u32 context;
+  u32 worker_index;
+  u32 lcore_id;
+  u8 name[64];
+};
+
+/** \brief Add/delete NAT44 pool address from specific interfce
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param sw_if_index - software index of the interface
+*/
+autoreply define snat_add_del_interface_addr {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump NAT44 pool addresses interfaces
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_interface_addr_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 pool addresses interfaces details response
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface
+*/
+define snat_interface_addr_details {
+  u32 context;
+  u32 sw_if_index;
+};
+
+/** \brief Enable/disable NAT IPFIX logging
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param domain_id - observation domain ID
+    @param src_port - source port number
+    @param enable - 1 if enable, 0 if disable
+*/
+autoreply define snat_ipfix_enable_disable {
+  u32 client_index;
+  u32 context;
+  u32 domain_id;
+  u16 src_port;
+  u8 enable;
+};
+
+/** \brief Dump NAT44 users
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_user_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 users response
+    @param context - sender context, to match reply w/ request
+    @vrf_id - VRF ID
+    @param is_ip4 - 1 if address type is IPv4
+    @param ip_adress - IP address
+    @param nsessions - number of dynamic sessions
+    @param nstaticsessions - number of static sessions
+*/
+define snat_user_details {
+  u32 context;
+  u32 vrf_id;
+  u8 is_ip4;
+  u8 ip_address[16];
+  u32 nsessions;
+  u32 nstaticsessions;
+};
+
+/** \brief NAT44 user's sessions
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param user_ip - IP address of the user to dump
+    @param vrf_id - VRF_ID
+*/
+define snat_user_session_dump {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u8 ip_address[16];
+  u32 vrf_id;
+};
+
+/** \brief NAT44 user's sessions response
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param outside_ip_address - outside IP address
+    @param outside_port - outside port
+    @param inside_ip_address - inside IP address
+    @param inside_port - inside port
+    @param protocol - protocol
+    @param is_static - 1 if session is static
+    @param last_heard - last heard timer
+    @param total_bytes - count of bytes sent through session
+    @param total_pkts - count of pakets sent through session
+*/
+define snat_user_session_details {
+  u32 context;
+  u8 is_ip4;
+  u8 outside_ip_address[16];
+  u16 outside_port;
+  u8 inside_ip_address[16];
+  u16 inside_port;
+  u16 protocol;
+  u8 is_static;
+  u64 last_heard;
+  u64 total_bytes;
+  u32 total_pkts;
+};
+
+/** \brief Add/delete NAT deterministic mapping
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_ip4 - 1 if address type is IPv4
+    @param in_addr - inside IP address
+    @param in_plen - inside IP address prefix length
+    @param out_addr - outside IP address
+    @param out_addr - outside IP address prefix length
+*/
+autoreply define snat_add_det_map {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_ip4;
+  u8 addr_only;
+  u8 in_addr[16];
+  u8 in_plen;
+  u8 out_addr[16];
+  u8 out_plen;
+};
+
+/** \brief Get outside address and port range from inside address
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param in_addr - inside IP address
+*/
+define snat_det_forward {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u8 in_addr[16];
+};
+
+/** \brief Get outside address and port range from inside address
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param out_port_lo - outside port range start
+    @param out_port_hi - outside port range end
+    @param is_ip4 - 1 if address type is IPv4
+    @param out_addr - outside IP address
+*/
+define snat_det_forward_reply {
+  u32 context;
+  i32 retval;
+  u16 out_port_lo;
+  u16 out_port_hi;
+  u8 is_ip4;
+  u8 out_addr[16];
+};
+
+/** \brief Get inside address from outside address and port
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param out_port - outside port
+    @param is_ip4 - 1 if address type is IPv4
+    @param out_addr - outside IP address
+*/
+define snat_det_reverse {
+  u32 client_index;
+  u32 context;
+  u16 out_port;
+  u8 is_ip4;
+  u8 out_addr[16];
+};
+
+/** \brief Get inside address from outside address and port reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param is_ip4 - 1 if address type is IPv4
+    @param in_addr - inside IP address
+*/
+define snat_det_reverse_reply {
+  u32 context;
+  i32 retval;
+  u8 is_ip4;
+  u8 in_addr[16];
+};
+
+/** \brief Dump NAT deterministic mappings
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_det_map_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT users response
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param in_addr - inside IP address
+    @param in_plen - inside IP address prefix length
+    @param out_addr - outside IP address
+    @param out_plen - outside IP address prefix length
+    @param sharing_ratio - outside to inside address sharing ratio
+    @param ports_per_host - number of ports available to a host
+    @param ses_num - number of sessions belonging to this mapping
+*/
+define snat_det_map_details {
+  u32 context;
+  u8 is_ip4;
+  u8 in_addr[16];
+  u8 in_plen;
+  u8 out_addr[16];
+  u8 out_plen;
+  u32 sharing_ratio;
+  u16 ports_per_host;
+  u32 ses_num;
+};
+
+/** \brief Set values of timeouts for deterministic NAT (seconds, 0 = default)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param udp - UDP timeout (default 300sec)
+    @param tcp_established - TCP established timeout (default 7440sec)
+    @param tcp_transitory - TCP transitory timeout (default 240sec)
+    @param icmp - ICMP timeout (default 60sec)
+*/
+autoreply define snat_det_set_timeouts {
+  u32 client_index;
+  u32 context;
+  u32 udp;
+  u32 tcp_established;
+  u32 tcp_transitory;
+  u32 icmp;
+};
+
+/** \brief Get values of timeouts for deterministic NAT (seconds)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define snat_det_get_timeouts {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Get values of timeouts for deterministic NAT reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param udp - UDP timeout (default 300sec)
+    @param tcp_established - TCP established timeout (default 7440sec)
+    @param tcp_transitory - TCP transitory timeout (default 240sec)
+    @param icmp - ICMP timeout (default 60sec)
+*/
+define snat_det_get_timeouts_reply {
+  u32 context;
+  i32 retval;
+  u32 udp;
+  u32 tcp_established;
+  u32 tcp_transitory;
+  u32 icmp;
+};
+
+/** \brief Close deterministic NAT session by outside address and port
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param out_addr - outside IP address
+    @param out_port - outside port
+    @param ext_addr - external host address
+    @param ext_port - external host port
+*/
+autoreply define snat_det_close_session_out {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u8 out_addr[16];
+  u16 out_port;
+  u8 ext_addr[16];
+  u16 ext_port;
+};
+
+/** \brief Close deterministic NAT session by inside address and port
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param in_addr - inside IP address
+    @param in_port - inside port
+    @param ext_addr - external host address
+    @param ext_port - external host port
+*/
+autoreply define snat_det_close_session_in {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u8 in_addr[16];
+  u16 in_port;
+  u8 ext_addr[16];
+  u16 ext_port;
+};
+
+/** \brief Dump determinstic NAT sessions
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param user_addr - address of an inside user whose sessions to dump
+*/
+define snat_det_session_dump {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u8 user_addr[16];
+};
+
+/** \brief Deterministic NAT sessions reply
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param in_port - inside port
+    @param ext_addr - external host address
+    @param ext_port - external host port
+    @param out_port - outside NAT port
+    @param state - session state
+    @param expire - session expiration timestamp
+*/
+define snat_det_session_details {
+  u32 client_index;
+  u32 context;
+  u8 is_ip4;
+  u16 in_port;
+  u8 ext_addr[16];
+  u16 ext_port;
+  u16 out_port;
+  u8 state;
+  u32 expire;
+};
+
+/*
+ * Common NAT plugin APIs
+ */
+
+/** \brief Control ping from client to api server request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat_control_ping
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Control ping from the client to the server response
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param vpe_pid - the pid of the vpe, returned by the server
+*/
+define nat_control_ping_reply
+{
+  u32 context;
+  i32 retval;
+  u32 client_index;
+  u32 vpe_pid;
+};
+
+/** \brief Show NAT plugin startup config
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat_show_config
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Show NAT plugin startup config reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param static_mapping_only - if 1 dynamic translations disabled
+    @param static_mapping_connection_tracking - if 1 create session data
+    @param deterministic - if 1 deterministic mapping
+    @param translation_buckets - number of translation hash buckets
+    @param translation_memory_size - translation hash memory size
+    @param user_buckets - number of user hash buckets
+    @param user_memory_size - user hash memory size
+    @param max_translations_per_user - maximum number of translations per user
+    @param outside_vrf_id - outside VRF id
+    @param inside_vrf_id - default inside VRF id
+*/
+define nat_show_config_reply
+{
+  u32 context;
+  i32 retval;
+  u8 static_mapping_only;
+  u8 static_mapping_connection_tracking;
+  u8 deterministic;
+  u32 translation_buckets;
+  u32 translation_memory_size;
+  u32 user_buckets;
+  u32 user_memory_size;
+  u32 max_translations_per_user;
+  u32 outside_vrf_id;
+  u32 inside_vrf_id;
+};
+
+/** \brief Set NAT workers
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param worker_mask - NAT workers mask
+*/
+autoreply define nat_set_workers {
+  u32 client_index;
+  u32 context;
+  u64 worker_mask;
+};
+
+/** \brief Dump NAT workers
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat_worker_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT workers details response
+    @param context - sender context, to match reply w/ request
+    @param worker_index - worker index
+    @param lcore_id - lcore ID
+    @param name - worker name
+*/
+define nat_worker_details {
+  u32 context;
+  u32 worker_index;
+  u32 lcore_id;
+  u8 name[64];
+};
+
+/** \brief Enable/disable NAT IPFIX logging
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param domain_id - observation domain ID
+    @param src_port - source port number
+    @param enable - 1 if enable, 0 if disable
+*/
+autoreply define nat_ipfix_enable_disable {
+  u32 client_index;
+  u32 context;
+  u32 domain_id;
+  u16 src_port;
+  u8 enable;
+};
+
+/*
+ * NAT44 APIs
+ */
+
+/** \brief Add/del NAT44 address range
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param first_ip_address - first IPv4 address
+    @param last_ip_address - last IPv4 address
+    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
+    @param is_add - 1 if add, 0 if delete
+*/
+autoreply define nat44_add_del_address_range {
+  u32 client_index;
+  u32 context;
+  u8 first_ip_address[4];
+  u8 last_ip_address[4];
+  u32 vrf_id;
+  u8 is_add;
+};
+
+/** \brief Dump NAT44 addresses
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat44_address_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 address details response
+    @param context - sender context, to match reply w/ request
+    @param ip_address - IPv4 address
+    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
+*/
+define nat44_address_details {
+  u32 context;
+  u8 ip_address[4];
+  u32 vrf_id;
+};
+
+/** \brief Enable/disable NAT44 feature on the interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+autoreply define nat44_interface_add_del_feature {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump interfaces with NAT44 feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat44_interface_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 interface details response
+    @param context - sender context, to match reply w/ request
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+define nat44_interface_details {
+  u32 context;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Enable/disbale NAT44 as an interface output feature (postrouting
+           in2out translation)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+autoreply define nat44_interface_add_del_output_feature {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump interfaces with NAT44 output feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat44_interface_output_feature_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 interface with output feature details response
+    @param context - sender context, to match reply w/ request
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - software index of the interface
+*/
+define nat44_interface_output_feature_details {
+  u32 context;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Add/delete NAT44 static mapping
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param addr_only - 1 if address only mapping
+    @param local_ip_address - local IPv4 address
+    @param external_ip_address - external IPv4 address
+    @param protocol - IP protocol
+    @param local_port - local port number
+    @param external_port - external port number
+    @param external_sw_if_index - external interface (if set
+                                  external_ip_address is ignored, ~0 means not
+                                  used)
+    @param vfr_id - VRF ID
+*/
+autoreply define nat44_add_del_static_mapping {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 addr_only;
+  u8 local_ip_address[4];
+  u8 external_ip_address[4];
+  u8 protocol;
+  u16 local_port;
+  u16 external_port;
+  u32 external_sw_if_index;
+  u32 vrf_id;
+};
+
+/** \brief Dump NAT44 static mappings
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat44_static_mapping_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 static mapping details response
+    @param context - sender context, to match reply w/ request
+    @param addr_only - 1 if address only mapping
+    @param local_ip_address - local IPv4 address
+    @param external_ip_address - external IPv4 address
+    @param protocol - IP protocol
+    @param local_port - local port number
+    @param external_port - external port number
+    @param external_sw_if_index - external interface
+    @param vfr_id - VRF ID
+*/
+define nat44_static_mapping_details {
+  u32 context;
+  u8 addr_only;
+  u8 local_ip_address[4];
+  u8 external_ip_address[4];
+  u8 protocol;
+  u16 local_port;
+  u16 external_port;
+  u32 external_sw_if_index;
+  u32 vrf_id;
+};
+
+/** \brief Add/delete NAT44 pool address from specific interfce
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param sw_if_index - software index of the interface
+*/
+autoreply define nat44_add_del_interface_addr {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Dump NAT44 pool addresses interfaces
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat44_interface_addr_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 pool addresses interfaces details response
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the interface
+*/
+define nat44_interface_addr_details {
+  u32 context;
+  u32 sw_if_index;
+};
+
+/** \brief Dump NAT44 users
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat44_user_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT44 users response
+    @param context - sender context, to match reply w/ request
+    @vrf_id - VRF ID
+    @param ip_adress - IPv4 address
+    @param nsessions - number of dynamic sessions
+    @param nstaticsessions - number of static sessions
+*/
+define nat44_user_details {
+  u32 context;
+  u32 vrf_id;
+  u8 ip_address[4];
+  u32 nsessions;
+  u32 nstaticsessions;
+};
+
+/** \brief NAT44 user's sessions
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_ip4 - 1 if address type is IPv4
+    @param user_ip - IP address of the user to dump
+    @param vrf_id - VRF_ID
+*/
+define nat44_user_session_dump {
+  u32 client_index;
+  u32 context;
+  u8 ip_address[4];
+  u32 vrf_id;
+};
+
+/** \brief NAT44 user's sessions response
+    @param context - sender context, to match reply w/ request
+    @param outside_ip_address - outside IPv4 address
+    @param outside_port - outside port
+    @param inside_ip_address - inside IPv4 address
+    @param inside_port - inside port
+    @param protocol - protocol
+    @param is_static - 1 if session is static
+    @param last_heard - last heard timer
+    @param total_bytes - count of bytes sent through session
+    @param total_pkts - count of pakets sent through session
+*/
+define nat44_user_session_details {
+  u32 context;
+  u8 outside_ip_address[4];
+  u16 outside_port;
+  u8 inside_ip_address[4];
+  u16 inside_port;
+  u16 protocol;
+  u8 is_static;
+  u64 last_heard;
+  u64 total_bytes;
+  u32 total_pkts;
+};
+
+/*
+ * Deterministic NAT (CGN) APIs
+ */
+
+/** \brief Add/delete NAT deterministic mapping
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param is_nat44 - 1 if NAT44
+    @param in_addr - inside IP address
+    @param in_plen - inside IP address prefix length
+    @param out_addr - outside IPv4 address
+    @param out_addr - outside IPv4 address prefix length
+*/
+autoreply define nat_det_add_del_map {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 is_nat44;
+  u8 addr_only;
+  u8 in_addr[16];
+  u8 in_plen;
+  u8 out_addr[4];
+  u8 out_plen;
+};
+
+/** \brief Get outside address and port range from inside address
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_nat44 - 1 if NAT44
+    @param in_addr - inside IP address
+*/
+define nat_det_forward {
+  u32 client_index;
+  u32 context;
+  u8 is_nat44;
+  u8 in_addr[16];
+};
+
+/** \brief Get outside address and port range from inside address
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param out_port_lo - outside port range start
+    @param out_port_hi - outside port range end
+    @param out_addr - outside IPv4 address
+*/
+define nat_det_forward_reply {
+  u32 context;
+  i32 retval;
+  u16 out_port_lo;
+  u16 out_port_hi;
+  u8 out_addr[4];
+};
+
+/** \brief Get inside address from outside address and port
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param out_port - outside port
+    @param out_addr - outside IPv4 address
+*/
+define nat_det_reverse {
+  u32 client_index;
+  u32 context;
+  u16 out_port;
+  u8 out_addr[4];
+};
+
+/** \brief Get inside address from outside address and port reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param is_nat44 - 1 if NAT44
+    @param in_addr - inside IP address
+*/
+define nat_det_reverse_reply {
+  u32 context;
+  i32 retval;
+  u8 is_nat44;
+  u8 in_addr[16];
+};
+
+/** \brief Dump NAT deterministic mappings
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat_det_map_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT users response
+    @param context - sender context, to match reply w/ request
+    @param is_nat44 - 1 if NAT44
+    @param in_addr - inside IP address
+    @param in_plen - inside IP address prefix length
+    @param out_addr - outside IPv4 address
+    @param out_plen - outside IPv4 address prefix length
+    @param sharing_ratio - outside to inside address sharing ratio
+    @param ports_per_host - number of ports available to a host
+    @param ses_num - number of sessions belonging to this mapping
+*/
+define nat_det_map_details {
+  u32 context;
+  u8 is_nat44;
+  u8 in_addr[16];
+  u8 in_plen;
+  u8 out_addr[4];
+  u8 out_plen;
+  u32 sharing_ratio;
+  u16 ports_per_host;
+  u32 ses_num;
+};
+
+/** \brief Set values of timeouts for deterministic NAT (seconds, 0 = default)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param udp - UDP timeout (default 300sec)
+    @param tcp_established - TCP established timeout (default 7440sec)
+    @param tcp_transitory - TCP transitory timeout (default 240sec)
+    @param icmp - ICMP timeout (default 60sec)
+*/
+autoreply define nat_det_set_timeouts {
+  u32 client_index;
+  u32 context;
+  u32 udp;
+  u32 tcp_established;
+  u32 tcp_transitory;
+  u32 icmp;
+};
+
+/** \brief Get values of timeouts for deterministic NAT (seconds)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat_det_get_timeouts {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Get values of timeouts for deterministic NAT reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param udp - UDP timeout (default 300sec)
+    @param tcp_established - TCP established timeout (default 7440sec)
+    @param tcp_transitory - TCP transitory timeout (default 240sec)
+    @param icmp - ICMP timeout (default 60sec)
+*/
+define nat_det_get_timeouts_reply {
+  u32 context;
+  i32 retval;
+  u32 udp;
+  u32 tcp_established;
+  u32 tcp_transitory;
+  u32 icmp;
+};
+
+/** \brief Close deterministic NAT session by outside address and port
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param out_addr - outside IPv4 address
+    @param out_port - outside port
+    @param ext_addr - external host IPv4 address
+    @param ext_port - external host port
+*/
+autoreply define nat_det_close_session_out {
+  u32 client_index;
+  u32 context;
+  u8 out_addr[4];
+  u16 out_port;
+  u8 ext_addr[4];
+  u16 ext_port;
+};
+
+/** \brief Close deterministic NAT session by inside address and port
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_nat44 - 1 if NAT44
+    @param in_addr - inside IP address
+    @param in_port - inside port
+    @param ext_addr - external host IP address
+    @param ext_port - external host port
+*/
+autoreply define nat_det_close_session_in {
+  u32 client_index;
+  u32 context;
+  u8 is_nat44;
+  u8 in_addr[16];
+  u16 in_port;
+  u8 ext_addr[16];
+  u16 ext_port;
+};
+
+/** \brief Dump determinstic NAT sessions
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_nat44 - 1 if NAT44
+    @param user_addr - address of an inside user whose sessions to dump
+*/
+define nat_det_session_dump {
+  u32 client_index;
+  u32 context;
+  u8 is_nat44;
+  u8 user_addr[16];
+};
+
+/** \brief Deterministic NAT sessions reply
+    @param context - sender context, to match reply w/ request
+    @param in_port - inside port
+    @param ext_addr - external host address
+    @param ext_port - external host port
+    @param out_port - outside NAT port
+    @param state - session state
+    @param expire - session expiration timestamp
+*/
+define nat_det_session_details {
+  u32 client_index;
+  u32 context;
+  u16 in_port;
+  u8 ext_addr[4];
+  u16 ext_port;
+  u16 out_port;
+  u8 state;
+  u32 expire;
+};
+
+/*
+ * NAT64 APIs
+ */
+
+/** \brief Add/delete address range to NAT64 pool
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param start_addr - start address of the range
+    @param end_addr - end address of the range
+    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
+    @param is_add - 1 if add, 0 if delete
+*/
+autoreply define nat64_add_del_pool_addr_range {
+  u32 client_index;
+  u32 context;
+  u8 start_addr[4];
+  u8 end_addr[4];
+  u32 vrf_id;
+  u8 is_add;
+};
+
+/** \brief Dump NAT64 pool addresses
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat64_pool_addr_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT64 pool address details response
+    @param context - sender context, to match reply w/ request
+    @param address - IPv4 address
+    @param vfr_id - VRF id of tenant, ~0 means independent of VRF
+*/
+define nat64_pool_addr_details {
+  u32 context;
+  u8 address[4];
+  u32 vrf_id;
+};
+
+/** \brief Enable/disable NAT64 feature on the interface
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - index of the interface
+    @param is_inside - 1 if inside, 0 if outside
+    @param is_add - 1 if add, 0 if delete
+*/
+autoreply define nat64_add_del_interface {
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+  u8 is_inside;
+  u8 is_add;
+};
+
+/** \brief Dump interfaces with NAT64 feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat64_interface_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief NAT64 interface details response
+    @param context - sender context, to match reply w/ request
+    @param is_inside - 1 if inside, 0 if outside
+    @param sw_if_index - index of the interface
+*/
+define nat64_interface_details {
+  u32 context;
+  u8 is_inside;
+  u32 sw_if_index;
+};
+
+/** \brief Add/delete NAT64 static BIB entry
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param i_addr - inside IPv6 address
+    @param o_addr - outside IPv4 address
+    @param i_port - inside port number
+    @param o_port - outside port number
+    @param vrf_id - VRF id of tenant
+    @param proto - protocol number
+    @param is_add - 1 if add, 0 if delete
+*/
+ autoreply define nat64_add_del_static_bib {
+  u32 client_index;
+  u32 context;
+  u8 i_addr[16];
+  u8 o_addr[4];
+  u16 i_port;
+  u16 o_port;
+  u32 vrf_id;
+  u8 proto;
+  u8 is_add;
+};
+
+/** \brief Dump NAT64 BIB
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param proto - protocol of the BIB: 255 - all BIBs
+                                        6 - TCP BIB
+                                        17 - UDP BIB
+                                        1/58 - ICMP BIB
+                                        otherwise - "unknown" protocol BIB
+*/
+define nat64_bib_dump {
+  u32 client_index;
+  u32 context;
+  u8 proto;
+};
+
+/** \brief NAT64 BIB details response
+    @param context - sender context, to match reply w/ request
+    @param i_addr - inside IPv6 address
+    @param o_addr - outside IPv4 address
+    @param i_port - inside port number
+    @param o_port - outside port number
+    @param vrf_id - VRF id of tenant
+    @param proto - protocol number
+    @param is_static - 1 if static BIB entry, 0 if dynamic
+    @param ses_num - number of sessions associated with the BIB entry
+*/
+define nat64_bib_details {
+  u32 context;
+  u8 i_addr[16];
+  u8 o_addr[4];
+  u16 i_port;
+  u16 o_port;
+  u32 vrf_id;
+  u8 proto;
+  u8 is_static;
+  u32 ses_num;
+};
+
+/** \brief Set values of timeouts for NAT64 (seconds, 0 = default)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param udp - UDP timeout (default 300sec)
+    @param icmp - ICMP timeout (default 60sec)
+    @param tcp_trans - TCP transitory timeout (default 240sec)
+    @param tcp_est - TCP established timeout (default 7440sec)
+    @param tcp_incoming_syn - TCP incoming SYN timeout (default 6sec)
+*/
+autoreply define nat64_set_timeouts {
+  u32 client_index;
+  u32 context;
+  u32 udp;
+  u32 icmp;
+  u32 tcp_trans;
+  u32 tcp_est;
+  u32 tcp_incoming_syn;
+};
+
+/** \brief Get values of timeouts for NAT64 (seconds)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat64_get_timeouts {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Get values of timeouts for NAT64 reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code
+    @param udp - UDP timeout
+    @param icmp - ICMP timeout
+    @param tcp_trans - TCP transitory timeout
+    @param tcp_est - TCP established timeout
+    @param tcp_incoming_syn - TCP incoming SYN timeout
+*/
+define nat64_get_timeouts_reply {
+  u32 context;
+  i32 retval;
+  u32 udp;
+  u32 icmp;
+  u32 tcp_trans;
+  u32 tcp_est;
+  u32 tcp_incoming_syn;
+};
+
+/** \brief Dump NAT64 session table
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param proto - protocol of the session table: 255 - all STs
+                                                  6 - TCP ST
+                                                  17 - UDP ST
+                                                  1/58 - ICMP ST
+                                                  otherwise - "unknown" proto ST
+*/
+define nat64_st_dump {
+  u32 client_index;
+  u32 context;
+  u8 proto;
+};
+
+/** \brief NAT64 session table details response
+    @param context - sender context, to match reply w/ request
+    @param il_addr - inside IPv6 address of the local host
+    @param ol_addr - outside IPv4 address of the local host
+    @param il_port - inside port number id of the local host/inside ICMP id
+    @param ol_port - outside port number of the local host/outside ICMP id
+    @param il_addr - inside IPv6 address of the remote host
+    @param ol_addr - outside IPv4 address of the remote host
+    @param l_port - port number of the remote host (not used for ICMP)
+    @param vrf_id - VRF id of tenant
+    @param proto - protocol number
+*/
+define nat64_st_details {
+  u32 context;
+  u8 il_addr[16];
+  u8 ol_addr[4];
+  u16 il_port;
+  u16 ol_port;
+  u8 ir_addr[16];
+  u8 or_addr[4];
+  u16 r_port;
+  u32 vrf_id;
+  u8 proto;
+};
+
+/** \brief Add/del NAT64 prefix
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param prefix - NAT64 prefix
+    @param prefix - NAT64 prefix length
+    @param vrf_id - VRF id of tenant
+    @param is_add - 1 if add, 0 if delete
+*/
+autoreply define nat64_add_del_prefix {
+  u32 client_index;
+  u32 context;
+  u8 prefix[16];
+  u8 prefix_len;
+  u32 vrf_id;
+  u8 is_add;
+};
+
+/** \brief Dump NAT64 prefix
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat64_prefix_dump {
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Dump NAT64 prefix details response
+    @param context - sender context, to match reply w/ request
+    @param prefix - NAT64 prefix
+    @param prefix - NAT64 prefix length
+    @param vrf_id - VRF id of tenant
+*/
+define nat64_prefix_details {
+  u32 context;
+  u8 prefix[16];
+  u8 prefix_len;
+  u32 vrf_id;
+};
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
new file mode 100644
index 00000000..ac39be95
--- /dev/null
+++ b/src/plugins/nat/nat.c
@@ -0,0 +1,2842 @@
+/*
+ * snat.c - simple nat plugin
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+snat_main_t snat_main;
+
+
+/* Hook up input features */
+VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-in2out",
+  .runs_before = VNET_FEATURES ("nat44-out2in"),
+};
+VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-out2in",
+  .runs_before = VNET_FEATURES ("ip4-lookup"),
+};
+VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-det-in2out",
+  .runs_before = VNET_FEATURES ("nat44-det-out2in"),
+};
+VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-det-out2in",
+  .runs_before = VNET_FEATURES ("ip4-lookup"),
+};
+VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-in2out-worker-handoff",
+  .runs_before = VNET_FEATURES ("nat44-out2in-worker-handoff"),
+};
+VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-out2in-worker-handoff",
+  .runs_before = VNET_FEATURES ("ip4-lookup"),
+};
+VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-in2out-fast",
+  .runs_before = VNET_FEATURES ("nat44-out2in-fast"),
+};
+VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-out2in-fast",
+  .runs_before = VNET_FEATURES ("ip4-lookup"),
+};
+VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-hairpin-dst",
+  .runs_before = VNET_FEATURES ("ip4-lookup"),
+};
+
+/* Hook up output features */
+VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
+  .arc_name = "ip4-output",
+  .node_name = "nat44-in2out-output",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
+  .arc_name = "ip4-output",
+  .node_name = "nat44-in2out-output-worker-handoff",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
+  .arc_name = "ip4-output",
+  .node_name = "nat44-hairpin-src",
+  .runs_before = VNET_FEATURES ("interface-output"),
+};
+
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .description = "Network Address Translation",
+};
+/* *INDENT-ON* */
+
+/**
+ * @brief Add/del NAT address to FIB.
+ *
+ * Add the external NAT address to the FIB as receive entries. This ensures
+ * that VPP will reply to ARP for this address and we don't need to enable
+ * proxy ARP on the outside interface.
+ *
+ * @param addr IPv4 address.
+ * @param plen address prefix length
+ * @param sw_if_index Interface.
+ * @param is_add If 0 delete, otherwise add.
+ */
+void
+snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
+                          int is_add)
+{
+  fib_prefix_t prefix = {
+    .fp_len = p_len,
+    .fp_proto = FIB_PROTOCOL_IP4,
+    .fp_addr = {
+        .ip4.as_u32 = addr->as_u32,
+    },
+  };
+  u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
+
+  if (is_add)
+    fib_table_entry_update_one_path(fib_index,
+                                    &prefix,
+                                    FIB_SOURCE_PLUGIN_HI,
+                                    (FIB_ENTRY_FLAG_CONNECTED |
+                                     FIB_ENTRY_FLAG_LOCAL |
+                                     FIB_ENTRY_FLAG_EXCLUSIVE),
+                                    DPO_PROTO_IP4,
+                                    NULL,
+                                    sw_if_index,
+                                    ~0,
+                                    1,
+                                    NULL,
+                                    FIB_ROUTE_PATH_FLAG_NONE);
+  else
+    fib_table_entry_delete(fib_index,
+                           &prefix,
+                           FIB_SOURCE_PLUGIN_HI);
+}
+
+void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id)
+{
+  snat_address_t * ap;
+  snat_interface_t *i;
+
+  if (vrf_id != ~0)
+    sm->vrf_mode = 1;
+
+  /* Check if address already exists */
+  vec_foreach (ap, sm->addresses)
+    {
+      if (ap->addr.as_u32 == addr->as_u32)
+        return;
+    }
+
+  vec_add2 (sm->addresses, ap, 1);
+  ap->addr = *addr;
+  if (vrf_id != ~0)
+    ap->fib_index =
+      fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id);
+  else
+    ap->fib_index = ~0;
+#define _(N, i, n, s) \
+  clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535);
+  foreach_snat_protocol
+#undef _
+
+  /* Add external address to FIB */
+  pool_foreach (i, sm->interfaces,
+  ({
+    if (i->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
+    break;
+  }));
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    if (i->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
+    break;
+  }));
+}
+
+static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
+                                                   ip4_address_t addr)
+{
+  snat_static_mapping_t *m;
+  pool_foreach (m, sm->static_mappings,
+  ({
+      if (m->external_addr.as_u32 == addr.as_u32)
+        return 1;
+  }));
+
+  return 0;
+}
+
+void increment_v4_address (ip4_address_t * a)
+{
+  u32 v;
+
+  v = clib_net_to_host_u32(a->as_u32) + 1;
+  a->as_u32 = clib_host_to_net_u32(v);
+}
+
+static void
+snat_add_static_mapping_when_resolved (snat_main_t * sm,
+                                       ip4_address_t l_addr,
+                                       u16 l_port,
+                                       u32 sw_if_index,
+                                       u16 e_port,
+                                       u32 vrf_id,
+                                       snat_protocol_t proto,
+                                       int addr_only,
+                                       int is_add)
+{
+  snat_static_map_resolve_t *rp;
+
+  vec_add2 (sm->to_resolve, rp, 1);
+  rp->l_addr.as_u32 = l_addr.as_u32;
+  rp->l_port = l_port;
+  rp->sw_if_index = sw_if_index;
+  rp->e_port = e_port;
+  rp->vrf_id = vrf_id;
+  rp->proto = proto;
+  rp->addr_only = addr_only;
+  rp->is_add = is_add;
+}
+
+/**
+ * @brief Add static mapping.
+ *
+ * Create static mapping between local addr+port and external addr+port.
+ *
+ * @param l_addr Local IPv4 address.
+ * @param e_addr External IPv4 address.
+ * @param l_port Local port number.
+ * @param e_port External port number.
+ * @param vrf_id VRF ID.
+ * @param addr_only If 0 address port and pair mapping, otherwise address only.
+ * @param sw_if_index External port instead of specific IP address.
+ * @param is_add If 0 delete static mapping, otherwise add.
+ *
+ * @returns
+ */
+int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
+                            u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
+                            u32 sw_if_index, snat_protocol_t proto, int is_add)
+{
+  snat_main_t * sm = &snat_main;
+  snat_static_mapping_t *m;
+  snat_session_key_t m_key;
+  clib_bihash_kv_8_8_t kv, value;
+  snat_address_t *a = 0;
+  u32 fib_index = ~0;
+  uword * p;
+  snat_interface_t *interface;
+  int i;
+
+  /* If the external address is a specific interface address */
+  if (sw_if_index != ~0)
+    {
+      ip4_address_t * first_int_addr;
+
+      /* Might be already set... */
+      first_int_addr = ip4_interface_first_address
+        (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
+
+      /* DHCP resolution required? */
+      if (first_int_addr == 0)
+        {
+          snat_add_static_mapping_when_resolved
+            (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
+             addr_only,  is_add);
+          return 0;
+        }
+        else
+          e_addr.as_u32 = first_int_addr->as_u32;
+    }
+
+  m_key.addr = e_addr;
+  m_key.port = addr_only ? 0 : e_port;
+  m_key.protocol = addr_only ? 0 : proto;
+  m_key.fib_index = sm->outside_fib_index;
+  kv.key = m_key.as_u64;
+  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+    m = 0;
+  else
+    m = pool_elt_at_index (sm->static_mappings, value.value);
+
+  if (is_add)
+    {
+      if (m)
+        return VNET_API_ERROR_VALUE_EXIST;
+
+      /* Convert VRF id to FIB index */
+      if (vrf_id != ~0)
+        {
+          p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
+          if (!p)
+            return VNET_API_ERROR_NO_SUCH_FIB;
+          fib_index = p[0];
+        }
+      /* If not specified use inside VRF id from SNAT plugin startup config */
+      else
+        {
+          fib_index = sm->inside_fib_index;
+          vrf_id = sm->inside_vrf_id;
+        }
+
+      /* Find external address in allocated addresses and reserve port for
+         address and port pair mapping when dynamic translations enabled */
+      if (!addr_only && !(sm->static_mapping_only))
+        {
+          for (i = 0; i < vec_len (sm->addresses); i++)
+            {
+              if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
+                {
+                  a = sm->addresses + i;
+                  /* External port must be unused */
+                  switch (proto)
+                    {
+#define _(N, j, n, s) \
+                    case SNAT_PROTOCOL_##N: \
+                      if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
+                        return VNET_API_ERROR_INVALID_VALUE; \
+                      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
+                      if (e_port > 1024) \
+                        a->busy_##n##_ports++; \
+                      break;
+                      foreach_snat_protocol
+#undef _
+                    default:
+                      clib_warning("unknown_protocol");
+                      return VNET_API_ERROR_INVALID_VALUE_2;
+                    }
+                  break;
+                }
+            }
+          /* External address must be allocated */
+          if (!a)
+            return VNET_API_ERROR_NO_SUCH_ENTRY;
+        }
+
+      pool_get (sm->static_mappings, m);
+      memset (m, 0, sizeof (*m));
+      m->local_addr = l_addr;
+      m->external_addr = e_addr;
+      m->addr_only = addr_only;
+      m->vrf_id = vrf_id;
+      m->fib_index = fib_index;
+      if (!addr_only)
+        {
+          m->local_port = l_port;
+          m->external_port = e_port;
+          m->proto = proto;
+        }
+
+      m_key.addr = m->local_addr;
+      m_key.port = m->local_port;
+      m_key.protocol = m->proto;
+      m_key.fib_index = m->fib_index;
+      kv.key = m_key.as_u64;
+      kv.value = m - sm->static_mappings;
+      clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
+
+      m_key.addr = m->external_addr;
+      m_key.port = m->external_port;
+      m_key.fib_index = sm->outside_fib_index;
+      kv.key = m_key.as_u64;
+      kv.value = m - sm->static_mappings;
+      clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
+
+      /* Assign worker */
+      if (sm->workers)
+        {
+          snat_user_key_t w_key0;
+          snat_worker_key_t w_key1;
+
+          w_key0.addr = m->local_addr;
+          w_key0.fib_index = m->fib_index;
+          kv.key = w_key0.as_u64;
+
+          if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value))
+            {
+              kv.value = sm->first_worker_index +
+                sm->workers[sm->next_worker++ % vec_len (sm->workers)];
+
+              clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv, 1);
+            }
+          else
+            {
+              kv.value = value.value;
+            }
+
+          w_key1.addr = m->external_addr;
+          w_key1.port = clib_host_to_net_u16 (m->external_port);
+          w_key1.fib_index = sm->outside_fib_index;
+          kv.key = w_key1.as_u64;
+          clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv, 1);
+        }
+    }
+  else
+    {
+      if (!m)
+        return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      /* Free external address port */
+      if (!addr_only && !(sm->static_mapping_only))
+        {
+          for (i = 0; i < vec_len (sm->addresses); i++)
+            {
+              if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
+                {
+                  a = sm->addresses + i;
+                  switch (proto)
+                    {
+#define _(N, j, n, s) \
+                    case SNAT_PROTOCOL_##N: \
+                      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
+                      if (e_port > 1024) \
+                        a->busy_##n##_ports--; \
+                      break;
+                      foreach_snat_protocol
+#undef _
+                    default:
+                      clib_warning("unknown_protocol");
+                      return VNET_API_ERROR_INVALID_VALUE_2;
+                    }
+                  break;
+                }
+            }
+        }
+
+      m_key.addr = m->local_addr;
+      m_key.port = m->local_port;
+      m_key.protocol = m->proto;
+      m_key.fib_index = m->fib_index;
+      kv.key = m_key.as_u64;
+      clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
+
+      m_key.addr = m->external_addr;
+      m_key.port = m->external_port;
+      m_key.fib_index = sm->outside_fib_index;
+      kv.key = m_key.as_u64;
+      clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
+
+      /* Delete session(s) for static mapping if exist */
+      if (!(sm->static_mapping_only) ||
+          (sm->static_mapping_only && sm->static_mapping_connection_tracking))
+        {
+          snat_user_key_t u_key;
+          snat_user_t *u;
+          dlist_elt_t * head, * elt;
+          u32 elt_index, head_index, del_elt_index;
+          u32 ses_index;
+          u64 user_index;
+          snat_session_t * s;
+          snat_main_per_thread_data_t *tsm;
+
+          u_key.addr = m->local_addr;
+          u_key.fib_index = m->fib_index;
+          kv.key = u_key.as_u64;
+          if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+            {
+              user_index = value.value;
+              if (!clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value))
+                tsm = vec_elt_at_index (sm->per_thread_data, value.value);
+              else
+                tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
+              u = pool_elt_at_index (tsm->users, user_index);
+              if (u->nstaticsessions)
+                {
+                  head_index = u->sessions_per_user_list_head_index;
+                  head = pool_elt_at_index (tsm->list_pool, head_index);
+                  elt_index = head->next;
+                  elt = pool_elt_at_index (tsm->list_pool, elt_index);
+                  ses_index = elt->value;
+                  while (ses_index != ~0)
+                    {
+                      s =  pool_elt_at_index (tsm->sessions, ses_index);
+                      del_elt_index = elt_index;
+                      elt_index = elt->next;
+                      elt = pool_elt_at_index (tsm->list_pool, elt_index);
+                      ses_index = elt->value;
+
+                      if (!addr_only)
+                        {
+                          if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
+                              (clib_net_to_host_u16 (s->out2in.port) != e_port))
+                            continue;
+                        }
+
+                      if (snat_is_unk_proto_session (s))
+                        {
+                          clib_bihash_kv_16_8_t up_kv;
+                          snat_unk_proto_ses_key_t up_key;
+                          up_key.l_addr = s->in2out.addr;
+                          up_key.r_addr = s->ext_host_addr;
+                          up_key.fib_index = s->in2out.fib_index;
+                          up_key.proto = s->in2out.port;
+                          up_key.rsvd[0] = up_key.rsvd[1] = up_key.rsvd[2] = 0;
+                          up_kv.key[0] = up_key.as_u64[0];
+                          up_kv.key[1] = up_key.as_u64[1];
+                          if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto,
+                                                        &up_kv, 0))
+                            clib_warning ("in2out key del failed");
+
+                          up_key.l_addr = s->out2in.addr;
+                          up_key.fib_index = s->out2in.fib_index;
+                          up_kv.key[0] = up_key.as_u64[0];
+                          up_kv.key[1] = up_key.as_u64[1];
+                          if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto,
+                                                        &up_kv, 0))
+                            clib_warning ("out2in key del failed");
+
+                          goto delete;
+                        }
+                      /* log NAT event */
+                      snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
+                                                          s->out2in.addr.as_u32,
+                                                          s->in2out.protocol,
+                                                          s->in2out.port,
+                                                          s->out2in.port,
+                                                          s->in2out.fib_index);
+
+                      value.key = s->in2out.as_u64;
+                      if (clib_bihash_add_del_8_8 (&sm->in2out, &value, 0))
+                        clib_warning ("in2out key del failed");
+                      value.key = s->out2in.as_u64;
+                      if (clib_bihash_add_del_8_8 (&sm->out2in, &value, 0))
+                        clib_warning ("out2in key del failed");
+delete:
+                      pool_put (tsm->sessions, s);
+
+                      clib_dlist_remove (tsm->list_pool, del_elt_index);
+                      pool_put_index (tsm->list_pool, del_elt_index);
+                      u->nstaticsessions--;
+
+                      if (!addr_only)
+                        break;
+                    }
+                  if (addr_only)
+                    {
+                      pool_put (tsm->users, u);
+                      clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 0);
+                    }
+                }
+            }
+        }
+
+      /* Delete static mapping from pool */
+      pool_put (sm->static_mappings, m);
+    }
+
+  if (!addr_only)
+    return 0;
+
+  /* Add/delete external address to FIB */
+  pool_foreach (interface, sm->interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
+    break;
+  }));
+  pool_foreach (interface, sm->output_feature_interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
+    break;
+  }));
+
+  return 0;
+}
+
+int snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm)
+{
+  snat_address_t *a = 0;
+  snat_session_t *ses;
+  u32 *ses_to_be_removed = 0, *ses_index;
+  clib_bihash_kv_8_8_t kv, value;
+  snat_user_key_t user_key;
+  snat_user_t *u;
+  snat_main_per_thread_data_t *tsm;
+  snat_static_mapping_t *m;
+  snat_interface_t *interface;
+  int i;
+
+  /* Find SNAT address */
+  for (i=0; i < vec_len (sm->addresses); i++)
+    {
+      if (sm->addresses[i].addr.as_u32 == addr.as_u32)
+        {
+          a = sm->addresses + i;
+          break;
+        }
+    }
+  if (!a)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  if (delete_sm)
+    {
+      pool_foreach (m, sm->static_mappings,
+      ({
+          if (m->external_addr.as_u32 == addr.as_u32)
+            (void) snat_add_static_mapping (m->local_addr, m->external_addr,
+                                            m->local_port, m->external_port,
+                                            m->vrf_id, m->addr_only, ~0,
+                                            m->proto, 0);
+      }));
+    }
+  else
+    {
+      /* Check if address is used in some static mapping */
+      if (is_snat_address_used_in_static_mapping(sm, addr))
+        {
+          clib_warning ("address used in static mapping");
+          return VNET_API_ERROR_UNSPECIFIED;
+        }
+    }
+
+  if (a->fib_index != ~0)
+    fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4);
+
+  /* Delete sessions using address */
+  if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
+    {
+      vec_foreach (tsm, sm->per_thread_data)
+        {
+          pool_foreach (ses, tsm->sessions, ({
+            if (ses->out2in.addr.as_u32 == addr.as_u32)
+              {
+                if (snat_is_unk_proto_session (ses))
+                  {
+                    clib_bihash_kv_16_8_t up_kv;
+                    snat_unk_proto_ses_key_t up_key;
+                    up_key.l_addr = ses->in2out.addr;
+                    up_key.r_addr = ses->ext_host_addr;
+                    up_key.fib_index = ses->in2out.fib_index;
+                    up_key.proto = ses->in2out.port;
+                    up_key.rsvd[0] = up_key.rsvd[1] = up_key.rsvd[2] = 0;
+                    up_kv.key[0] = up_key.as_u64[0];
+                    up_kv.key[1] = up_key.as_u64[1];
+                    if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto,
+                                                  &up_kv, 0))
+                      clib_warning ("in2out key del failed");
+
+                    up_key.l_addr = ses->out2in.addr;
+                    up_key.fib_index = ses->out2in.fib_index;
+                    up_kv.key[0] = up_key.as_u64[0];
+                    up_kv.key[1] = up_key.as_u64[1];
+                    if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto,
+                                                  &up_kv, 0))
+                      clib_warning ("out2in key del failed");
+                  }
+                else
+                  {
+                    /* log NAT event */
+                    snat_ipfix_logging_nat44_ses_delete(ses->in2out.addr.as_u32,
+                                                        ses->out2in.addr.as_u32,
+                                                        ses->in2out.protocol,
+                                                        ses->in2out.port,
+                                                        ses->out2in.port,
+                                                        ses->in2out.fib_index);
+                    kv.key = ses->in2out.as_u64;
+                    clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0);
+                    kv.key = ses->out2in.as_u64;
+                    clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0);
+                  }
+                vec_add1 (ses_to_be_removed, ses - tsm->sessions);
+                clib_dlist_remove (tsm->list_pool, ses->per_user_index);
+                user_key.addr = ses->in2out.addr;
+                user_key.fib_index = ses->in2out.fib_index;
+                kv.key = user_key.as_u64;
+                if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+                  {
+                    u = pool_elt_at_index (tsm->users, value.value);
+                    u->nsessions--;
+                  }
+              }
+          }));
+
+          vec_foreach (ses_index, ses_to_be_removed)
+            pool_put_index (tsm->sessions, ses_index[0]);
+
+          vec_free (ses_to_be_removed);
+       }
+    }
+
+  vec_del1 (sm->addresses, i);
+
+  /* Delete external address from FIB */
+  pool_foreach (interface, sm->interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
+    break;
+  }));
+  pool_foreach (interface, sm->output_feature_interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
+    break;
+  }));
+
+  return 0;
+}
+
+int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
+{
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+  const char * feature_name;
+  snat_address_t * ap;
+  snat_static_mapping_t * m;
+  snat_det_map_t * dm;
+
+  if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
+    feature_name = is_inside ?  "nat44-in2out-fast" : "nat44-out2in-fast";
+  else
+    {
+      if (sm->num_workers > 1 && !sm->deterministic)
+        feature_name = is_inside ?  "nat44-in2out-worker-handoff" : "nat44-out2in-worker-handoff";
+      else if (sm->deterministic)
+        feature_name = is_inside ?  "nat44-det-in2out" : "nat44-det-out2in";
+      else
+        feature_name = is_inside ?  "nat44-in2out" : "nat44-out2in";
+    }
+
+  vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index,
+			       !is_del, 0, 0);
+
+  if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
+    sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index, 0);
+
+  if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
+    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
+
+  pool_foreach (i, sm->interfaces,
+  ({
+    if (i->sw_if_index == sw_if_index)
+      {
+        if (is_del)
+          pool_put (sm->interfaces, i);
+        else
+          return VNET_API_ERROR_VALUE_EXIST;
+
+        goto fib;
+      }
+  }));
+
+  if (is_del)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  pool_get (sm->interfaces, i);
+  i->sw_if_index = sw_if_index;
+  i->is_inside = is_inside;
+
+  /* Add/delete external addresses to FIB */
+fib:
+  if (is_inside)
+    return 0;
+
+  vec_foreach (ap, sm->addresses)
+    snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
+
+  pool_foreach (m, sm->static_mappings,
+  ({
+    if (!(m->addr_only))
+      continue;
+
+    snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
+  }));
+
+  pool_foreach (dm, sm->det_maps,
+  ({
+    snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
+  }));
+
+  return 0;
+}
+
+int snat_interface_add_del_output_feature (u32 sw_if_index,
+                                           u8 is_inside,
+                                           int is_del)
+{
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+  snat_address_t * ap;
+  snat_static_mapping_t * m;
+
+  if (sm->deterministic ||
+      (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
+    return VNET_API_ERROR_UNSUPPORTED;
+
+  if (is_inside)
+    {
+      vnet_feature_enable_disable ("ip4-unicast", "nat44-hairpin-dst",
+                                   sw_if_index, !is_del, 0, 0);
+      vnet_feature_enable_disable ("ip4-output", "nat44-hairpin-src",
+                                   sw_if_index, !is_del, 0, 0);
+      goto fq;
+    }
+
+  if (sm->num_workers > 1)
+    {
+      vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in-worker-handoff",
+                                   sw_if_index, !is_del, 0, 0);
+      vnet_feature_enable_disable ("ip4-output",
+                                   "nat44-in2out-output-worker-handoff",
+                                   sw_if_index, !is_del, 0, 0);
+    }
+  else
+    {
+      vnet_feature_enable_disable ("ip4-unicast", "nat44-out2in", sw_if_index,
+                                   !is_del, 0, 0);
+      vnet_feature_enable_disable ("ip4-output", "nat44-in2out-output",
+                                   sw_if_index, !is_del, 0, 0);
+    }
+
+fq:
+  if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
+    sm->fq_in2out_output_index =
+      vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
+
+  if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
+    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
+
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    if (i->sw_if_index == sw_if_index)
+      {
+        if (is_del)
+          pool_put (sm->output_feature_interfaces, i);
+        else
+          return VNET_API_ERROR_VALUE_EXIST;
+
+        goto fib;
+      }
+  }));
+
+  if (is_del)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  pool_get (sm->output_feature_interfaces, i);
+  i->sw_if_index = sw_if_index;
+  i->is_inside = is_inside;
+
+  /* Add/delete external addresses to FIB */
+fib:
+  if (is_inside)
+    return 0;
+
+  vec_foreach (ap, sm->addresses)
+    snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
+
+  pool_foreach (m, sm->static_mappings,
+  ({
+    if (!(m->addr_only))
+      continue;
+
+    snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
+  }));
+
+  return 0;
+}
+
+int snat_set_workers (uword * bitmap)
+{
+  snat_main_t *sm = &snat_main;
+  int i, j = 0;
+
+  if (sm->num_workers < 2)
+    return VNET_API_ERROR_FEATURE_DISABLED;
+
+  if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
+    return VNET_API_ERROR_INVALID_WORKER;
+
+  vec_free (sm->workers);
+  clib_bitmap_foreach (i, bitmap,
+    ({
+      vec_add1(sm->workers, i);
+      sm->per_thread_data[i].snat_thread_index = j;
+      j++;
+    }));
+
+  sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
+  sm->num_snat_thread = _vec_len (sm->workers);
+
+  return 0;
+}
+
+
+static void
+snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
+                                       uword opaque,
+                                       u32 sw_if_index,
+                                       ip4_address_t * address,
+                                       u32 address_length,
+                                       u32 if_address_index,
+                                       u32 is_delete);
+
+static clib_error_t * snat_init (vlib_main_t * vm)
+{
+  snat_main_t * sm = &snat_main;
+  clib_error_t * error = 0;
+  ip4_main_t * im = &ip4_main;
+  ip_lookup_main_t * lm = &im->lookup_main;
+  uword *p;
+  vlib_thread_registration_t *tr;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  uword *bitmap = 0;
+  u32 i;
+  ip4_add_del_interface_address_callback_t cb4;
+
+  sm->vlib_main = vm;
+  sm->vnet_main = vnet_get_main();
+  sm->ip4_main = im;
+  sm->ip4_lookup_main = lm;
+  sm->api_main = &api_main;
+  sm->first_worker_index = 0;
+  sm->next_worker = 0;
+  sm->num_workers = 0;
+  sm->num_snat_thread = 1;
+  sm->workers = 0;
+  sm->port_per_thread = 0xffff - 1024;
+  sm->fq_in2out_index = ~0;
+  sm->fq_out2in_index = ~0;
+  sm->udp_timeout = SNAT_UDP_TIMEOUT;
+  sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
+  sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
+  sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
+
+  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  if (p)
+    {
+      tr = (vlib_thread_registration_t *) p[0];
+      if (tr)
+        {
+          sm->num_workers = tr->count;
+          sm->first_worker_index = tr->first_index;
+        }
+    }
+
+  vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
+
+  /* Use all available workers by default */
+  if (sm->num_workers > 1)
+    {
+      for (i=0; i < sm->num_workers; i++)
+        bitmap = clib_bitmap_set (bitmap, i, 1);
+      snat_set_workers(bitmap);
+      clib_bitmap_free (bitmap);
+    }
+  else
+    {
+      sm->per_thread_data[0].snat_thread_index = 0;
+    }
+
+  error = snat_api_init(vm, sm);
+  if (error)
+    return error;
+
+  /* Set up the interface address add/del callback */
+  cb4.function = snat_ip4_add_del_interface_address_cb;
+  cb4.function_opaque = 0;
+
+  vec_add1 (im->add_del_interface_address_callbacks, cb4);
+
+  /* Init IPFIX logging */
+  snat_ipfix_logging_init(vm);
+
+  error = nat64_init(vm);
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (snat_init);
+
+void snat_free_outside_address_and_port (snat_main_t * sm,
+                                         snat_session_key_t * k,
+                                         u32 address_index)
+{
+  snat_address_t *a;
+  u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
+
+  ASSERT (address_index < vec_len (sm->addresses));
+
+  a = sm->addresses + address_index;
+
+  switch (k->protocol)
+    {
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
+        port_host_byte_order) == 1); \
+      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
+        port_host_byte_order, 0); \
+      a->busy_##n##_ports--; \
+      break;
+      foreach_snat_protocol
+#undef _
+    default:
+      clib_warning("unknown_protocol");
+      return;
+    }
+}
+
+/**
+ * @brief Match NAT44 static mapping.
+ *
+ * @param sm          NAT main.
+ * @param match       Address and port to match.
+ * @param mapping     External or local address and port of the matched mapping.
+ * @param by_external If 0 match by local address otherwise match by external
+ *                    address.
+ * @param is_addr_only If matched mapping is address only
+ *
+ * @returns 0 if match found otherwise 1.
+ */
+int snat_static_mapping_match (snat_main_t * sm,
+                               snat_session_key_t match,
+                               snat_session_key_t * mapping,
+                               u8 by_external,
+                               u8 *is_addr_only)
+{
+  clib_bihash_kv_8_8_t kv, value;
+  snat_static_mapping_t *m;
+  snat_session_key_t m_key;
+  clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
+
+  if (by_external)
+    mapping_hash = &sm->static_mapping_by_external;
+
+  m_key.addr = match.addr;
+  m_key.port = clib_net_to_host_u16 (match.port);
+  m_key.protocol = match.protocol;
+  m_key.fib_index = match.fib_index;
+
+  kv.key = m_key.as_u64;
+
+  if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
+    {
+      /* Try address only mapping */
+      m_key.port = 0;
+      m_key.protocol = 0;
+      kv.key = m_key.as_u64;
+      if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
+        return 1;
+    }
+
+  m = pool_elt_at_index (sm->static_mappings, value.value);
+
+  if (by_external)
+    {
+      mapping->addr = m->local_addr;
+      /* Address only mapping doesn't change port */
+      mapping->port = m->addr_only ? match.port
+        : clib_host_to_net_u16 (m->local_port);
+      mapping->fib_index = m->fib_index;
+    }
+  else
+    {
+      mapping->addr = m->external_addr;
+      /* Address only mapping doesn't change port */
+      mapping->port = m->addr_only ? match.port
+        : clib_host_to_net_u16 (m->external_port);
+      mapping->fib_index = sm->outside_fib_index;
+    }
+
+  if (PREDICT_FALSE(is_addr_only != 0))
+    *is_addr_only = m->addr_only;
+
+  return 0;
+}
+
+static_always_inline u16
+snat_random_port (snat_main_t * sm, u16 min, u16 max)
+{
+  return min + random_u32 (&sm->random_seed) /
+    (random_u32_max() / (max - min + 1) + 1);
+}
+
+int snat_alloc_outside_address_and_port (snat_main_t * sm,
+                                         u32 fib_index,
+                                         u32 thread_index,
+                                         snat_session_key_t * k,
+                                         u32 * address_indexp)
+{
+  int i;
+  snat_address_t *a;
+  u32 portnum;
+
+  for (i = 0; i < vec_len (sm->addresses); i++)
+    {
+      a = sm->addresses + i;
+      if (sm->vrf_mode && a->fib_index != ~0 && a->fib_index != fib_index)
+        continue;
+      switch (k->protocol)
+        {
+#define _(N, j, n, s) \
+        case SNAT_PROTOCOL_##N: \
+          if (a->busy_##n##_ports < (sm->port_per_thread * sm->num_snat_thread)) \
+            { \
+              while (1) \
+                { \
+                  portnum = (sm->port_per_thread * \
+                    sm->per_thread_data[thread_index].snat_thread_index) + \
+                    snat_random_port(sm, 0, sm->port_per_thread) + 1024; \
+                  if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
+                    continue; \
+                  clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
+                  a->busy_##n##_ports++; \
+                  k->addr = a->addr; \
+                  k->port = clib_host_to_net_u16(portnum); \
+                  *address_indexp = i; \
+                  return 0; \
+                } \
+            } \
+          break;
+          foreach_snat_protocol
+#undef _
+        default:
+          clib_warning("unknown protocol");
+          return 1;
+        }
+
+    }
+  /* Totally out of translations to use... */
+  snat_ipfix_logging_addresses_exhausted(0);
+  return 1;
+}
+
+
+static clib_error_t *
+add_address_command_fn (vlib_main_t * vm,
+                        unformat_input_t * input,
+                        vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  snat_main_t * sm = &snat_main;
+  ip4_address_t start_addr, end_addr, this_addr;
+  u32 start_host_order, end_host_order;
+  u32 vrf_id = ~0;
+  int i, count;
+  int is_add = 1;
+  int rv = 0;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U - %U",
+                    unformat_ip4_address, &start_addr,
+                    unformat_ip4_address, &end_addr))
+        ;
+      else if (unformat (line_input, "tenant-vrf %u", &vrf_id))
+        ;
+      else if (unformat (line_input, "%U", unformat_ip4_address, &start_addr))
+        end_addr = start_addr;
+      else if (unformat (line_input, "del"))
+        is_add = 0;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
+     }
+
+  if (sm->static_mapping_only)
+    {
+      error = clib_error_return (0, "static mapping only mode");
+      goto done;
+    }
+
+  start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
+  end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
+
+  if (end_host_order < start_host_order)
+    {
+      error = clib_error_return (0, "end address less than start address");
+      goto done;
+    }
+
+  count = (end_host_order - start_host_order) + 1;
+
+  if (count > 1024)
+    clib_warning ("%U - %U, %d addresses...",
+                  format_ip4_address, &start_addr,
+                  format_ip4_address, &end_addr,
+                  count);
+
+  this_addr = start_addr;
+
+  for (i = 0; i < count; i++)
+    {
+      if (is_add)
+        snat_add_address (sm, &this_addr, vrf_id);
+      else
+        rv = snat_del_address (sm, this_addr, 0);
+
+      switch (rv)
+        {
+        case VNET_API_ERROR_NO_SUCH_ENTRY:
+          error = clib_error_return (0, "S-NAT address not exist.");
+          goto done;
+        case VNET_API_ERROR_UNSPECIFIED:
+          error = clib_error_return (0, "S-NAT address used in static mapping.");
+          goto done;
+        default:
+          break;
+        }
+
+      increment_v4_address (&this_addr);
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+VLIB_CLI_COMMAND (add_address_command, static) = {
+  .path = "nat44 add address",
+  .short_help = "nat44 add addresses  [- ] "
+                "[tenant-vrf ] [del]",
+  .function = add_address_command_fn,
+};
+
+static clib_error_t *
+snat_feature_command_fn (vlib_main_t * vm,
+                          unformat_input_t * input,
+                          vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vnet_main_t * vnm = vnet_get_main();
+  clib_error_t * error = 0;
+  u32 sw_if_index;
+  u32 * inside_sw_if_indices = 0;
+  u32 * outside_sw_if_indices = 0;
+  u8 is_output_feature = 0;
+  int is_del = 0;
+  int i;
+
+  sw_if_index = ~0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "in %U", unformat_vnet_sw_interface,
+                    vnm, &sw_if_index))
+        vec_add1 (inside_sw_if_indices, sw_if_index);
+      else if (unformat (line_input, "out %U", unformat_vnet_sw_interface,
+                         vnm, &sw_if_index))
+        vec_add1 (outside_sw_if_indices, sw_if_index);
+      else if (unformat (line_input, "output-feature"))
+        is_output_feature = 1;
+      else if (unformat (line_input, "del"))
+        is_del = 1;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  if (vec_len (inside_sw_if_indices))
+    {
+      for (i = 0; i < vec_len(inside_sw_if_indices); i++)
+        {
+          sw_if_index = inside_sw_if_indices[i];
+          if (is_output_feature)
+            {
+              if (snat_interface_add_del_output_feature (sw_if_index, 1, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
+          else
+            {
+              if (snat_interface_add_del (sw_if_index, 1, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
+        }
+    }
+
+  if (vec_len (outside_sw_if_indices))
+    {
+      for (i = 0; i < vec_len(outside_sw_if_indices); i++)
+        {
+          sw_if_index = outside_sw_if_indices[i];
+          if (is_output_feature)
+            {
+              if (snat_interface_add_del_output_feature (sw_if_index, 0, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
+          else
+            {
+              if (snat_interface_add_del (sw_if_index, 0, is_del))
+                {
+                  error = clib_error_return (0, "%s %U failed",
+                                             is_del ? "del" : "add",
+                                             format_vnet_sw_interface_name, vnm,
+                                             vnet_get_sw_interface (vnm,
+                                                                    sw_if_index));
+                  goto done;
+                }
+            }
+        }
+    }
+
+done:
+  unformat_free (line_input);
+  vec_free (inside_sw_if_indices);
+  vec_free (outside_sw_if_indices);
+
+  return error;
+}
+
+VLIB_CLI_COMMAND (set_interface_snat_command, static) = {
+  .path = "set interface nat44",
+  .function = snat_feature_command_fn,
+  .short_help = "set interface nat44 in  out  [output-feature] "
+                "[del]",
+};
+
+uword
+unformat_snat_protocol (unformat_input_t * input, va_list * args)
+{
+  u32 *r = va_arg (*args, u32 *);
+
+  if (0);
+#define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
+  foreach_snat_protocol
+#undef _
+  else
+    return 0;
+  return 1;
+}
+
+u8 *
+format_snat_protocol (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  u8 *t = 0;
+
+  switch (i)
+    {
+#define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
+      foreach_snat_protocol
+#undef _
+    default:
+      s = format (s, "unknown");
+      return s;
+    }
+  s = format (s, "%s", t);
+  return s;
+}
+
+static clib_error_t *
+add_static_mapping_command_fn (vlib_main_t * vm,
+                               unformat_input_t * input,
+                               vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t * error = 0;
+  ip4_address_t l_addr, e_addr;
+  u32 l_port = 0, e_port = 0, vrf_id = ~0;
+  int is_add = 1;
+  int addr_only = 1;
+  u32 sw_if_index = ~0;
+  vnet_main_t * vnm = vnet_get_main();
+  int rv;
+  snat_protocol_t proto;
+  u8 proto_set = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "local %U %u", unformat_ip4_address, &l_addr,
+                    &l_port))
+        addr_only = 0;
+      else if (unformat (line_input, "local %U", unformat_ip4_address, &l_addr))
+        ;
+      else if (unformat (line_input, "external %U %u", unformat_ip4_address,
+                         &e_addr, &e_port))
+        addr_only = 0;
+      else if (unformat (line_input, "external %U", unformat_ip4_address,
+                         &e_addr))
+        ;
+      else if (unformat (line_input, "external %U %u",
+                         unformat_vnet_sw_interface, vnm, &sw_if_index,
+                         &e_port))
+        addr_only = 0;
+
+      else if (unformat (line_input, "external %U",
+                         unformat_vnet_sw_interface, vnm, &sw_if_index))
+        ;
+      else if (unformat (line_input, "vrf %u", &vrf_id))
+        ;
+      else if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
+        proto_set = 1;
+      else if (unformat (line_input, "del"))
+        is_add = 0;
+      else
+        {
+          error = clib_error_return (0, "unknown input: '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  if (!addr_only && !proto_set)
+    {
+      error = clib_error_return (0, "missing protocol");
+      goto done;
+    }
+
+  rv = snat_add_static_mapping(l_addr, e_addr, (u16) l_port, (u16) e_port,
+                               vrf_id, addr_only, sw_if_index, proto, is_add);
+
+  switch (rv)
+    {
+    case VNET_API_ERROR_INVALID_VALUE:
+      error = clib_error_return (0, "External port already in use.");
+      goto done;
+    case VNET_API_ERROR_NO_SUCH_ENTRY:
+      if (is_add)
+        error = clib_error_return (0, "External addres must be allocated.");
+      else
+        error = clib_error_return (0, "Mapping not exist.");
+      goto done;
+    case VNET_API_ERROR_NO_SUCH_FIB:
+      error = clib_error_return (0, "No such VRF id.");
+      goto done;
+    case VNET_API_ERROR_VALUE_EXIST:
+      error = clib_error_return (0, "Mapping already exist.");
+      goto done;
+    default:
+      break;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat add static mapping}
+ * Static mapping allows hosts on the external network to initiate connection
+ * to to the local network host.
+ * To create static mapping between local host address 10.0.0.3 port 6303 and
+ * external address 4.4.4.4 port 3606 for TCP protocol use:
+ *  vpp# nat44 add static mapping local tcp 10.0.0.3 6303 external 4.4.4.4 3606
+ * If not runnig "static mapping only" NAT plugin mode use before:
+ *  vpp# nat44 add address 4.4.4.4
+ * To create static mapping between local and external address use:
+ *  vpp# nat44 add static mapping local 10.0.0.3 external 4.4.4.4
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (add_static_mapping_command, static) = {
+  .path = "nat44 add static mapping",
+  .function = add_static_mapping_command_fn,
+  .short_help =
+    "nat44 add static mapping local tcp|udp|icmp  [] external  [] [vrf ] [del]",
+};
+
+static clib_error_t *
+set_workers_command_fn (vlib_main_t * vm,
+                        unformat_input_t * input,
+                        vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  uword *bitmap = 0;
+  int rv = 0;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U", unformat_bitmap_list, &bitmap))
+        ;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
+     }
+
+  if (bitmap == 0)
+    {
+      error = clib_error_return (0, "List of workers must be specified.");
+      goto done;
+    }
+
+  rv = snat_set_workers(bitmap);
+
+  clib_bitmap_free (bitmap);
+
+  switch (rv)
+    {
+    case VNET_API_ERROR_INVALID_WORKER:
+      error = clib_error_return (0, "Invalid worker(s).");
+      goto done;
+    case VNET_API_ERROR_FEATURE_DISABLED:
+      error = clib_error_return (0,
+        "Supported only if 2 or more workes available.");
+      goto done;
+    default:
+      break;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{set snat workers}
+ * Set NAT workers if 2 or more workers available, use:
+ *  vpp# set snat workers 0-2,5
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (set_workers_command, static) = {
+  .path = "set nat workers",
+  .function = set_workers_command_fn,
+  .short_help =
+    "set nat workers ",
+};
+
+static clib_error_t *
+snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm,
+                                              unformat_input_t * input,
+                                              vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 domain_id = 0;
+  u32 src_port = 0;
+  u8 enable = 1;
+  int rv = 0;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "domain %d", &domain_id))
+        ;
+      else if (unformat (line_input, "src-port %d", &src_port))
+        ;
+      else if (unformat (line_input, "disable"))
+        enable = 0;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
+     }
+
+  rv = snat_ipfix_logging_enable_disable (enable, domain_id, (u16) src_port);
+
+  if (rv)
+    {
+      error = clib_error_return (0, "ipfix logging enable failed");
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat ipfix logging}
+ * To enable NAT IPFIX logging use:
+ *  vpp# nat ipfix logging
+ * To set IPFIX exporter use:
+ *  vpp# set ipfix exporter collector 10.10.10.3 src 10.10.10.1
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (snat_ipfix_logging_enable_disable_command, static) = {
+  .path = "nat ipfix logging",
+  .function = snat_ipfix_logging_enable_disable_command_fn,
+  .short_help = "nat ipfix logging [domain ] [src-port ] [disable]",
+};
+
+static u32
+snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
+{
+  snat_main_t *sm = &snat_main;
+  snat_user_key_t key0;
+  clib_bihash_kv_8_8_t kv0, value0;
+  u32 next_worker_index = 0;
+
+  key0.addr = ip0->src_address;
+  key0.fib_index = rx_fib_index0;
+
+  kv0.key = key0.as_u64;
+
+  /* Ever heard of of the "user" before? */
+  if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv0, &value0))
+    {
+      /* No, assign next available worker (RR) */
+      next_worker_index = sm->first_worker_index;
+      if (vec_len (sm->workers))
+        {
+          next_worker_index +=
+            sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
+        }
+
+      /* add non-traslated packets worker lookup */
+      kv0.value = next_worker_index;
+      clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
+    }
+  else
+    next_worker_index = value0.value;
+
+  return next_worker_index;
+}
+
+static u32
+snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
+{
+  snat_main_t *sm = &snat_main;
+  snat_worker_key_t key0;
+  clib_bihash_kv_8_8_t kv0, value0;
+  udp_header_t * udp0;
+  u32 next_worker_index = 0;
+
+  udp0 = ip4_next_header (ip0);
+
+  key0.addr = ip0->dst_address;
+  key0.port = udp0->dst_port;
+  key0.fib_index = rx_fib_index0;
+
+  if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
+    {
+      icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
+      icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
+      key0.port = echo0->identifier;
+    }
+
+  kv0.key = key0.as_u64;
+
+  /* Ever heard of of the "user" before? */
+  if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
+    {
+      key0.port = 0;
+      kv0.key = key0.as_u64;
+
+      if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
+        {
+          /* No, assign next available worker (RR) */
+          next_worker_index = sm->first_worker_index;
+          if (vec_len (sm->workers))
+            {
+              next_worker_index +=
+                sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
+            }
+        }
+      else
+        {
+          /* Static mapping without port */
+          next_worker_index = value0.value;
+        }
+
+      /* Add to translated packets worker lookup */
+      kv0.value = next_worker_index;
+      clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
+    }
+  else
+    next_worker_index = value0.value;
+
+  return next_worker_index;
+}
+
+static clib_error_t *
+snat_config (vlib_main_t * vm, unformat_input_t * input)
+{
+  snat_main_t * sm = &snat_main;
+  u32 translation_buckets = 1024;
+  u32 translation_memory_size = 128<<20;
+  u32 user_buckets = 128;
+  u32 user_memory_size = 64<<20;
+  u32 max_translations_per_user = 100;
+  u32 outside_vrf_id = 0;
+  u32 inside_vrf_id = 0;
+  u32 static_mapping_buckets = 1024;
+  u32 static_mapping_memory_size = 64<<20;
+  u8 static_mapping_only = 0;
+  u8 static_mapping_connection_tracking = 0;
+
+  sm->deterministic = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "translation hash buckets %d", &translation_buckets))
+        ;
+      else if (unformat (input, "translation hash memory %d",
+                         &translation_memory_size));
+      else if (unformat (input, "user hash buckets %d", &user_buckets))
+        ;
+      else if (unformat (input, "user hash memory %d",
+                         &user_memory_size))
+        ;
+      else if (unformat (input, "max translations per user %d",
+                         &max_translations_per_user))
+        ;
+      else if (unformat (input, "outside VRF id %d",
+                         &outside_vrf_id))
+        ;
+      else if (unformat (input, "inside VRF id %d",
+                         &inside_vrf_id))
+        ;
+      else if (unformat (input, "static mapping only"))
+        {
+          static_mapping_only = 1;
+          if (unformat (input, "connection tracking"))
+            static_mapping_connection_tracking = 1;
+        }
+      else if (unformat (input, "deterministic"))
+        sm->deterministic = 1;
+      else
+	return clib_error_return (0, "unknown input '%U'",
+				  format_unformat_error, input);
+    }
+
+  /* for show commands, etc. */
+  sm->translation_buckets = translation_buckets;
+  sm->translation_memory_size = translation_memory_size;
+  sm->user_buckets = user_buckets;
+  sm->user_memory_size = user_memory_size;
+  sm->max_translations_per_user = max_translations_per_user;
+  sm->outside_vrf_id = outside_vrf_id;
+  sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
+                                                             outside_vrf_id);
+  sm->inside_vrf_id = inside_vrf_id;
+  sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
+                                                            inside_vrf_id);
+  sm->static_mapping_only = static_mapping_only;
+  sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
+
+  if (sm->deterministic)
+    {
+      sm->in2out_node_index = snat_det_in2out_node.index;
+      sm->in2out_output_node_index = ~0;
+      sm->out2in_node_index = snat_det_out2in_node.index;
+      sm->icmp_match_in2out_cb = icmp_match_in2out_det;
+      sm->icmp_match_out2in_cb = icmp_match_out2in_det;
+    }
+  else
+    {
+      sm->worker_in2out_cb = snat_get_worker_in2out_cb;
+      sm->worker_out2in_cb = snat_get_worker_out2in_cb;
+      sm->in2out_node_index = snat_in2out_node.index;
+      sm->in2out_output_node_index = snat_in2out_output_node.index;
+      sm->out2in_node_index = snat_out2in_node.index;
+      if (!static_mapping_only ||
+          (static_mapping_only && static_mapping_connection_tracking))
+        {
+          sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
+          sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
+
+          clib_bihash_init_8_8 (&sm->worker_by_in, "worker-by-in", user_buckets,
+                                user_memory_size);
+
+          clib_bihash_init_8_8 (&sm->worker_by_out, "worker-by-out", user_buckets,
+                                user_memory_size);
+
+          clib_bihash_init_8_8 (&sm->in2out, "in2out", translation_buckets,
+                                translation_memory_size);
+
+          clib_bihash_init_8_8 (&sm->out2in, "out2in", translation_buckets,
+                                translation_memory_size);
+
+          clib_bihash_init_8_8 (&sm->user_hash, "users", user_buckets,
+                                user_memory_size);
+
+          clib_bihash_init_16_8 (&sm->in2out_unk_proto, "in2out-unk-proto",
+                                 translation_buckets, translation_memory_size);
+
+          clib_bihash_init_16_8 (&sm->out2in_unk_proto, "out2in-unk-proto",
+                                 translation_buckets, translation_memory_size);
+        }
+      else
+        {
+          sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
+          sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
+        }
+      clib_bihash_init_8_8 (&sm->static_mapping_by_local,
+                            "static_mapping_by_local", static_mapping_buckets,
+                            static_mapping_memory_size);
+
+      clib_bihash_init_8_8 (&sm->static_mapping_by_external,
+                            "static_mapping_by_external", static_mapping_buckets,
+                            static_mapping_memory_size);
+    }
+
+  return 0;
+}
+
+VLIB_CONFIG_FUNCTION (snat_config, "nat");
+
+u8 * format_snat_session_state (u8 * s, va_list * args)
+{
+  u32 i = va_arg (*args, u32);
+  u8 *t = 0;
+
+  switch (i)
+    {
+#define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
+    foreach_snat_session_state
+#undef _
+    default:
+      t = format (t, "unknown");
+    }
+  s = format (s, "%s", t);
+  return s;
+}
+
+u8 * format_snat_key (u8 * s, va_list * args)
+{
+  snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
+  char * protocol_string = "unknown";
+  static char *protocol_strings[] = {
+      "UDP",
+      "TCP",
+      "ICMP",
+  };
+
+  if (key->protocol < ARRAY_LEN(protocol_strings))
+      protocol_string = protocol_strings[key->protocol];
+
+  s = format (s, "%U proto %s port %d fib %d",
+              format_ip4_address, &key->addr, protocol_string,
+              clib_net_to_host_u16 (key->port), key->fib_index);
+  return s;
+}
+
+u8 * format_snat_session (u8 * s, va_list * args)
+{
+  snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *);
+  snat_session_t * sess = va_arg (*args, snat_session_t *);
+
+  if (snat_is_unk_proto_session (sess))
+    {
+      s = format (s, "  i2o %U proto %u fib %u\n",
+                  format_ip4_address, &sess->in2out.addr, sess->in2out.port,
+                  sess->in2out.fib_index);
+      s = format (s, "    o2i %U proto %u fib %u\n",
+                  format_ip4_address, &sess->out2in.addr, sess->out2in.port,
+                  sess->out2in.fib_index);
+    }
+  else
+    {
+      s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
+      s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
+    }
+  s = format (s, "       last heard %.2f\n", sess->last_heard);
+  s = format (s, "       total pkts %d, total bytes %lld\n",
+              sess->total_pkts, sess->total_bytes);
+  if (snat_is_session_static (sess))
+    s = format (s, "       static translation\n");
+  else
+    s = format (s, "       dynamic translation\n");
+
+  return s;
+}
+
+u8 * format_snat_user (u8 * s, va_list * args)
+{
+  snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
+  snat_user_t * u = va_arg (*args, snat_user_t *);
+  int verbose = va_arg (*args, int);
+  dlist_elt_t * head, * elt;
+  u32 elt_index, head_index;
+  u32 session_index;
+  snat_session_t * sess;
+
+  s = format (s, "%U: %d dynamic translations, %d static translations\n",
+              format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
+
+  if (verbose == 0)
+    return s;
+
+  if (u->nsessions || u->nstaticsessions)
+    {
+      head_index = u->sessions_per_user_list_head_index;
+      head = pool_elt_at_index (sm->list_pool, head_index);
+
+      elt_index = head->next;
+      elt = pool_elt_at_index (sm->list_pool, elt_index);
+      session_index = elt->value;
+
+      while (session_index != ~0)
+        {
+          sess = pool_elt_at_index (sm->sessions, session_index);
+
+          s = format (s, "  %U\n", format_snat_session, sm, sess);
+
+          elt_index = elt->next;
+          elt = pool_elt_at_index (sm->list_pool, elt_index);
+          session_index = elt->value;
+        }
+    }
+
+  return s;
+}
+
+u8 * format_snat_static_mapping (u8 * s, va_list * args)
+{
+  snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
+
+  if (m->addr_only)
+      s = format (s, "local %U external %U vrf %d",
+                  format_ip4_address, &m->local_addr,
+                  format_ip4_address, &m->external_addr,
+                  m->vrf_id);
+  else
+      s = format (s, "%U local %U:%d external %U:%d vrf %d",
+                  format_snat_protocol, m->proto,
+                  format_ip4_address, &m->local_addr, m->local_port,
+                  format_ip4_address, &m->external_addr, m->external_port,
+                  m->vrf_id);
+
+  return s;
+}
+
+u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
+{
+  snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
+  vnet_main_t *vnm = vnet_get_main();
+
+  if (m->addr_only)
+      s = format (s, "local %U external %U vrf %d",
+                  format_ip4_address, &m->l_addr,
+                  format_vnet_sw_interface_name, vnm,
+                  vnet_get_sw_interface (vnm, m->sw_if_index),
+                  m->vrf_id);
+  else
+      s = format (s, "%U local %U:%d external %U:%d vrf %d",
+                  format_snat_protocol, m->proto,
+                  format_ip4_address, &m->l_addr, m->l_port,
+                  format_vnet_sw_interface_name, vnm,
+                  vnet_get_sw_interface (vnm, m->sw_if_index), m->e_port,
+                  m->vrf_id);
+
+  return s;
+}
+
+u8 * format_det_map_ses (u8 * s, va_list * args)
+{
+  snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
+  ip4_address_t in_addr, out_addr;
+  u32 in_offset, out_offset;
+  snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
+  u32 * i = va_arg (*args, u32 *);
+
+  u32 user_index = *i / SNAT_DET_SES_PER_USER;
+  in_addr.as_u32 = clib_host_to_net_u32 (
+    clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
+  in_offset = clib_net_to_host_u32(in_addr.as_u32) -
+    clib_net_to_host_u32(det_map->in_addr.as_u32);
+  out_offset = in_offset / det_map->sharing_ratio;
+  out_addr.as_u32 = clib_host_to_net_u32(
+    clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
+  s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
+              format_ip4_address, &in_addr,
+              clib_net_to_host_u16 (ses->in_port),
+              format_ip4_address, &out_addr,
+              clib_net_to_host_u16 (ses->out.out_port),
+              format_ip4_address, &ses->out.ext_host_addr,
+              clib_net_to_host_u16 (ses->out.ext_host_port),
+              format_snat_session_state, ses->state,
+              ses->expire);
+
+  return s;
+}
+
+static clib_error_t *
+show_snat_command_fn (vlib_main_t * vm,
+		 unformat_input_t * input,
+		 vlib_cli_command_t * cmd)
+{
+  int verbose = 0;
+  snat_main_t * sm = &snat_main;
+  snat_user_t * u;
+  snat_static_mapping_t *m;
+  snat_interface_t *i;
+  snat_address_t * ap;
+  vnet_main_t *vnm = vnet_get_main();
+  snat_main_per_thread_data_t *tsm;
+  u32 users_num = 0, sessions_num = 0, *worker, *sw_if_index;
+  uword j = 0;
+  snat_static_map_resolve_t *rp;
+  snat_det_map_t * dm;
+  snat_det_session_t * ses;
+
+  if (unformat (input, "detail"))
+    verbose = 1;
+  else if (unformat (input, "verbose"))
+    verbose = 2;
+
+  if (sm->static_mapping_only)
+    {
+      if (sm->static_mapping_connection_tracking)
+        vlib_cli_output (vm, "NAT plugin mode: static mapping only connection "
+                         "tracking");
+      else
+        vlib_cli_output (vm, "NAT plugin mode: static mapping only");
+    }
+  else if (sm->deterministic)
+    {
+      vlib_cli_output (vm, "NAT plugin mode: deterministic mapping");
+    }
+  else
+    {
+      vlib_cli_output (vm, "NAT plugin mode: dynamic translations enabled");
+    }
+
+  if (verbose > 0)
+    {
+      pool_foreach (i, sm->interfaces,
+      ({
+        vlib_cli_output (vm, "%U %s", format_vnet_sw_interface_name, vnm,
+                         vnet_get_sw_interface (vnm, i->sw_if_index),
+                         i->is_inside ? "in" : "out");
+      }));
+
+      pool_foreach (i, sm->output_feature_interfaces,
+      ({
+        vlib_cli_output (vm, "%U output-feature %s",
+                         format_vnet_sw_interface_name, vnm,
+                         vnet_get_sw_interface (vnm, i->sw_if_index),
+                         i->is_inside ? "in" : "out");
+      }));
+
+      if (vec_len (sm->auto_add_sw_if_indices))
+        {
+          vlib_cli_output (vm, "NAT44 pool addresses interfaces:");
+          vec_foreach (sw_if_index, sm->auto_add_sw_if_indices)
+            {
+              vlib_cli_output (vm, "%U", format_vnet_sw_interface_name, vnm,
+                               vnet_get_sw_interface (vnm, *sw_if_index));
+            }
+        }
+
+      vec_foreach (ap, sm->addresses)
+        {
+          vlib_cli_output (vm, "%U", format_ip4_address, &ap->addr);
+          if (ap->fib_index != ~0)
+              vlib_cli_output (vm, "  tenant VRF: %u",
+                               ip4_fib_get(ap->fib_index)->table_id);
+          else
+            vlib_cli_output (vm, "  tenant VRF independent");
+#define _(N, i, n, s) \
+          vlib_cli_output (vm, "  %d busy %s ports", ap->busy_##n##_ports, s);
+          foreach_snat_protocol
+#undef _
+        }
+    }
+
+  if (sm->num_workers > 1)
+    {
+      vlib_cli_output (vm, "%d workers", vec_len (sm->workers));
+      if (verbose > 0)
+        {
+          vec_foreach (worker, sm->workers)
+            {
+              vlib_worker_thread_t *w =
+                vlib_worker_threads + *worker + sm->first_worker_index;
+              vlib_cli_output (vm, "  %s", w->name);
+            }
+        }
+    }
+
+  if (sm->deterministic)
+    {
+      vlib_cli_output (vm, "udp timeout: %dsec", sm->udp_timeout);
+      vlib_cli_output (vm, "tcp-established timeout: %dsec",
+                       sm->tcp_established_timeout);
+      vlib_cli_output (vm, "tcp-transitory timeout: %dsec",
+                       sm->tcp_transitory_timeout);
+      vlib_cli_output (vm, "icmp timeout: %dsec", sm->icmp_timeout);
+      vlib_cli_output (vm, "%d deterministic mappings",
+                       pool_elts (sm->det_maps));
+      if (verbose > 0)
+        {
+          pool_foreach (dm, sm->det_maps,
+          ({
+            vlib_cli_output (vm, "in %U/%d out %U/%d\n",
+                             format_ip4_address, &dm->in_addr, dm->in_plen,
+                             format_ip4_address, &dm->out_addr, dm->out_plen);
+            vlib_cli_output (vm, " outside address sharing ratio: %d\n",
+                             dm->sharing_ratio);
+            vlib_cli_output (vm, " number of ports per inside host: %d\n",
+                             dm->ports_per_host);
+            vlib_cli_output (vm, " sessions number: %d\n", dm->ses_num);
+            if (verbose > 1)
+              {
+                vec_foreach_index (j, dm->sessions)
+                  {
+                    ses = vec_elt_at_index (dm->sessions, j);
+                    if (ses->in_port)
+                      vlib_cli_output (vm, "  %U", format_det_map_ses, dm, ses,
+                                       &j);
+                  }
+              }
+          }));
+        }
+    }
+  else
+    {
+      if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
+        {
+          vlib_cli_output (vm, "%d static mappings",
+                           pool_elts (sm->static_mappings));
+
+          if (verbose > 0)
+            {
+              pool_foreach (m, sm->static_mappings,
+              ({
+                vlib_cli_output (vm, "%U", format_snat_static_mapping, m);
+              }));
+            }
+        }
+      else
+        {
+          vec_foreach (tsm, sm->per_thread_data)
+            {
+              users_num += pool_elts (tsm->users);
+              sessions_num += pool_elts (tsm->sessions);
+            }
+
+          vlib_cli_output (vm, "%d users, %d outside addresses, %d active sessions,"
+                           " %d static mappings",
+                           users_num,
+                           vec_len (sm->addresses),
+                           sessions_num,
+                           pool_elts (sm->static_mappings));
+
+          if (verbose > 0)
+            {
+              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->in2out,
+                               verbose - 1);
+              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->out2in,
+                               verbose - 1);
+              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_in,
+                               verbose - 1);
+              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_out,
+                               verbose - 1);
+              vec_foreach_index (j, sm->per_thread_data)
+                {
+                  tsm = vec_elt_at_index (sm->per_thread_data, j);
+
+                  if (pool_elts (tsm->users) == 0)
+                    continue;
+
+                  vlib_worker_thread_t *w = vlib_worker_threads + j;
+                  vlib_cli_output (vm, "Thread %d (%s at lcore %u):", j, w->name,
+                                   w->lcore_id);
+                  vlib_cli_output (vm, "  %d list pool elements",
+                                   pool_elts (tsm->list_pool));
+
+                  pool_foreach (u, tsm->users,
+                  ({
+                    vlib_cli_output (vm, "  %U", format_snat_user, tsm, u,
+                                     verbose - 1);
+                  }));
+                }
+
+              if (pool_elts (sm->static_mappings))
+                {
+                  vlib_cli_output (vm, "static mappings:");
+                  pool_foreach (m, sm->static_mappings,
+                  ({
+                    vlib_cli_output (vm, "%U", format_snat_static_mapping, m);
+                  }));
+                  for (j = 0; j < vec_len (sm->to_resolve); j++)
+                    {
+                      rp = sm->to_resolve + j;
+                      vlib_cli_output (vm, "%U",
+                                       format_snat_static_map_to_resolve, rp);
+                    }
+                }
+            }
+        }
+    }
+  return 0;
+}
+
+VLIB_CLI_COMMAND (show_snat_command, static) = {
+    .path = "show nat44",
+    .short_help = "show nat44",
+    .function = show_snat_command_fn,
+};
+
+
+static void
+snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
+                                       uword opaque,
+                                       u32 sw_if_index,
+                                       ip4_address_t * address,
+                                       u32 address_length,
+                                       u32 if_address_index,
+                                       u32 is_delete)
+{
+  snat_main_t *sm = &snat_main;
+  snat_static_map_resolve_t *rp;
+  u32 *indices_to_delete = 0;
+  int i, j;
+  int rv;
+
+  for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
+    {
+      if (sw_if_index == sm->auto_add_sw_if_indices[i])
+        {
+          if (!is_delete)
+            {
+              /* Don't trip over lease renewal, static config */
+              for (j = 0; j < vec_len(sm->addresses); j++)
+                if (sm->addresses[j].addr.as_u32 == address->as_u32)
+                  return;
+
+              snat_add_address (sm, address, ~0);
+              /* Scan static map resolution vector */
+              for (j = 0; j < vec_len (sm->to_resolve); j++)
+                {
+                  rp = sm->to_resolve + j;
+                  /* On this interface? */
+                  if (rp->sw_if_index == sw_if_index)
+                    {
+                      /* Add the static mapping */
+                      rv = snat_add_static_mapping (rp->l_addr,
+                                                    address[0],
+                                                    rp->l_port,
+                                                    rp->e_port,
+                                                    rp->vrf_id,
+                                                    rp->addr_only,
+                                                    ~0 /* sw_if_index */,
+                                                    rp->proto,
+                                                    rp->is_add);
+                      if (rv)
+                        clib_warning ("snat_add_static_mapping returned %d", 
+                                      rv);
+                      vec_add1 (indices_to_delete, j);
+                    }
+                }
+              /* If we resolved any of the outstanding static mappings */
+              if (vec_len(indices_to_delete))
+                {
+                  /* Delete them */
+                  for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
+                    vec_delete(sm->to_resolve, 1, j);
+                  vec_free(indices_to_delete);
+                }
+              return;
+            }
+          else
+            {
+              (void) snat_del_address(sm, address[0], 1);
+              return;
+            }
+        }
+    }
+}
+
+
+int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del)
+{
+  ip4_main_t * ip4_main = sm->ip4_main;
+  ip4_address_t * first_int_addr;
+  snat_static_map_resolve_t *rp;
+  u32 *indices_to_delete = 0;
+  int i, j;
+
+  first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
+                                                0 /* just want the address*/);
+
+  for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
+    {
+      if (sm->auto_add_sw_if_indices[i] == sw_if_index)
+        {
+          if (is_del)
+            {
+              /* if have address remove it */
+              if (first_int_addr)
+                  (void) snat_del_address (sm, first_int_addr[0], 1);
+              else
+                {
+                  for (j = 0; j < vec_len (sm->to_resolve); j++)
+                    {
+                      rp = sm->to_resolve + j;
+                      if (rp->sw_if_index == sw_if_index)
+                        vec_add1 (indices_to_delete, j);
+                    }
+                  if (vec_len(indices_to_delete))
+                    {
+                      for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
+                        vec_del1(sm->to_resolve, j);
+                      vec_free(indices_to_delete);
+                    }
+                }
+              vec_del1(sm->auto_add_sw_if_indices, i);
+            }
+          else
+            return VNET_API_ERROR_VALUE_EXIST;
+
+          return 0;
+        }
+    }
+
+  if (is_del)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  /* add to the auto-address list */
+  vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
+
+  /* If the address is already bound - or static - add it now */
+  if (first_int_addr)
+      snat_add_address (sm, first_int_addr, ~0);
+
+  return 0;
+}
+
+static clib_error_t *
+snat_add_interface_address_command_fn (vlib_main_t * vm,
+                                       unformat_input_t * input,
+                                       vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 sw_if_index;
+  int rv;
+  int is_del = 0;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U", unformat_vnet_sw_interface,
+                    sm->vnet_main, &sw_if_index))
+        ;
+      else if (unformat (line_input, "del"))
+        is_del = 1;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  rv = snat_add_interface_address (sm, sw_if_index, is_del);
+
+  switch (rv)
+    {
+    case 0:
+      break;
+
+    default:
+      error = clib_error_return (0, "snat_add_interface_address returned %d",
+                                 rv);
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+VLIB_CLI_COMMAND (snat_add_interface_address_command, static) = {
+    .path = "nat44 add interface address",
+    .short_help = "nat44 add interface address  [del]",
+    .function = snat_add_interface_address_command_fn,
+};
+
+static clib_error_t *
+snat_det_map_command_fn (vlib_main_t * vm,
+                         unformat_input_t * input,
+                         vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t in_addr, out_addr;
+  u32 in_plen, out_plen;
+  int is_add = 1, rv;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "in %U/%u", unformat_ip4_address, &in_addr, &in_plen))
+        ;
+      else if (unformat (line_input, "out %U/%u", unformat_ip4_address, &out_addr, &out_plen))
+        ;
+      else if (unformat (line_input, "del"))
+        is_add = 0;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+                                     format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  unformat_free (line_input);
+
+  rv = snat_det_add_map(sm, &in_addr, (u8) in_plen, &out_addr, (u8)out_plen,
+                        is_add);
+
+  if (rv)
+    {
+      error = clib_error_return (0, "snat_det_add_map return %d", rv);
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat deterministic add}
+ * Create bijective mapping of inside address to outside address and port range
+ * pairs, with the purpose of enabling deterministic NAT to reduce logging in
+ * CGN deployments.
+ * To create deterministic mapping between inside network 10.0.0.0/18 and
+ * outside network 1.1.1.0/30 use:
+ * # vpp# nat44 deterministic add in 10.0.0.0/18 out 1.1.1.0/30
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (snat_det_map_command, static) = {
+    .path = "nat44 deterministic add",
+    .short_help = "nat44 deterministic add in / out / [del]",
+    .function = snat_det_map_command_fn,
+};
+
+static clib_error_t *
+snat_det_forward_command_fn (vlib_main_t * vm,
+                             unformat_input_t * input,
+                             vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t in_addr, out_addr;
+  u16 lo_port;
+  snat_det_map_t * dm;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U", unformat_ip4_address, &in_addr))
+        ;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+                                     format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  unformat_free (line_input);
+
+  dm = snat_det_map_by_user(sm, &in_addr);
+  if (!dm)
+    vlib_cli_output (vm, "no match");
+  else
+    {
+      snat_det_forward (dm, &in_addr, &out_addr, &lo_port);
+      vlib_cli_output (vm, "%U:<%d-%d>", format_ip4_address, &out_addr,
+                       lo_port, lo_port + dm->ports_per_host - 1);
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat deterministic forward}
+ * Return outside address and port range from inside address for deterministic
+ * NAT.
+ * To obtain outside address and port of inside host use:
+ *  vpp# nat44 deterministic forward 10.0.0.2
+ *  1.1.1.0:<1054-1068>
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (snat_det_forward_command, static) = {
+    .path = "nat44 deterministic forward",
+    .short_help = "nat44 deterministic forward ",
+    .function = snat_det_forward_command_fn,
+};
+
+static clib_error_t *
+snat_det_reverse_command_fn (vlib_main_t * vm,
+                             unformat_input_t * input,
+                             vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t in_addr, out_addr;
+  u32 out_port;
+  snat_det_map_t * dm;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U:%d", unformat_ip4_address, &out_addr, &out_port))
+        ;
+      else
+        {
+          error =  clib_error_return (0, "unknown input '%U'",
+                                      format_unformat_error, line_input);
+        }
+    }
+
+  unformat_free (line_input);
+
+  if (out_port < 1024 || out_port > 65535)
+    {
+      error = clib_error_return (0, "wrong port, must be <1024-65535>");
+      goto done;
+    }
+
+  dm = snat_det_map_by_out(sm, &out_addr);
+  if (!dm)
+    vlib_cli_output (vm, "no match");
+  else
+    {
+      snat_det_reverse (dm, &out_addr, (u16) out_port, &in_addr);
+      vlib_cli_output (vm, "%U", format_ip4_address, &in_addr);
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat deterministic reverse}
+ * Return inside address from outside address and port for deterministic NAT.
+ * To obtain inside host address from outside address and port use:
+ *  #vpp nat44 deterministic reverse 1.1.1.1:1276
+ *  10.0.16.16
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (snat_det_reverse_command, static) = {
+    .path = "nat44 deterministic reverse",
+    .short_help = "nat44 deterministic reverse :",
+    .function = snat_det_reverse_command_fn,
+};
+
+static clib_error_t *
+set_timeout_command_fn (vlib_main_t * vm,
+                        unformat_input_t * input,
+                        vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "udp %u", &sm->udp_timeout))
+        ;
+      else if (unformat (line_input, "tcp-established %u",
+               &sm->tcp_established_timeout))
+        ;
+      else if (unformat (line_input, "tcp-transitory %u",
+               &sm->tcp_transitory_timeout))
+        ;
+      else if (unformat (line_input, "icmp %u", &sm->icmp_timeout))
+        ;
+      else if (unformat (line_input, "reset"))
+        {
+          sm->udp_timeout = SNAT_UDP_TIMEOUT;
+          sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
+          sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
+          sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
+        }
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+                                     format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  unformat_free (line_input);
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{set snat deterministic timeout}
+ * Set values of timeouts for deterministic NAT (in seconds), use:
+ *  vpp# set nat44 deterministic timeout udp 120 tcp-established 7500
+ *  tcp-transitory 250 icmp 90
+ * To reset default values use:
+ *  vpp# set nat44 deterministic timeout reset
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (set_timeout_command, static) = {
+  .path = "set nat44 deterministic timeout",
+  .function = set_timeout_command_fn,
+  .short_help =
+    "set nat44 deterministic timeout [udp  | tcp-established  "
+    "tcp-transitory  | icmp  | reset]",
+};
+
+static clib_error_t *
+snat_det_close_session_out_fn (vlib_main_t *vm,
+                               unformat_input_t * input,
+                               vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t out_addr, ext_addr, in_addr;
+  u16 out_port, ext_port;
+  snat_det_map_t * dm;
+  snat_det_session_t * ses;
+  snat_det_out_key_t key;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U:%d %U:%d",
+                    unformat_ip4_address, &out_addr, &out_port,
+                    unformat_ip4_address, &ext_addr, &ext_port))
+        ;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+                                     format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  unformat_free (line_input);
+
+  dm = snat_det_map_by_out(sm, &out_addr);
+  if (!dm)
+    vlib_cli_output (vm, "no match");
+  else
+    {
+      snat_det_reverse(dm, &ext_addr, out_port, &in_addr);
+      key.ext_host_addr = out_addr;
+      key.ext_host_port = ntohs(ext_port);
+      key.out_port = ntohs(out_port);
+      ses = snat_det_get_ses_by_out(dm, &out_addr, key.as_u64);
+      if (!ses)
+        vlib_cli_output (vm, "no match");
+      else
+       snat_det_ses_close(dm, ses);
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat deterministic close session out}
+ * Close session using outside ip address and port
+ * and external ip address and port, use:
+ *  vpp# nat44 deterministic close session out 1.1.1.1:1276 2.2.2.2:2387
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (snat_det_close_sesion_out_command, static) = {
+  .path = "nat44 deterministic close session out",
+  .short_help = "nat44 deterministic close session out "
+                ": :",
+  .function = snat_det_close_session_out_fn,
+};
+
+static clib_error_t *
+snat_det_close_session_in_fn (vlib_main_t *vm,
+                              unformat_input_t * input,
+                              vlib_cli_command_t * cmd)
+{
+  snat_main_t *sm = &snat_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t in_addr, ext_addr;
+  u16 in_port, ext_port;
+  snat_det_map_t * dm;
+  snat_det_session_t * ses;
+  snat_det_out_key_t key;
+  clib_error_t *error = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U:%d %U:%d",
+                    unformat_ip4_address, &in_addr, &in_port,
+                    unformat_ip4_address, &ext_addr, &ext_port))
+        ;
+      else
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+                                     format_unformat_error, line_input);
+          goto done;
+        }
+    }
+
+  unformat_free (line_input);
+
+  dm = snat_det_map_by_user (sm, &in_addr);
+  if (!dm)
+    vlib_cli_output (vm, "no match");
+  else
+    {
+      key.ext_host_addr = ext_addr;
+      key.ext_host_port = ntohs (ext_port);
+      ses = snat_det_find_ses_by_in (dm, &in_addr, ntohs(in_port), key);
+      if (!ses)
+        vlib_cli_output (vm, "no match");
+      else
+        snat_det_ses_close(dm, ses);
+    }
+
+done:
+  unformat_free(line_input);
+
+  return error;
+}
+
+/*?
+ * @cliexpar
+ * @cliexstart{snat deterministic close_session_in}
+ * Close session using inside ip address and port
+ * and external ip address and port, use:
+ *  vpp# nat44 deterministic close session in 3.3.3.3:3487 2.2.2.2:2387
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (snat_det_close_session_in_command, static) = {
+  .path = "nat44 deterministic close session in",
+  .short_help = "nat44 deterministic close session in "
+                ": :",
+  .function = snat_det_close_session_in_fn,
+};
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
new file mode 100644
index 00000000..04c466dc
--- /dev/null
+++ b/src/plugins/nat/nat.h
@@ -0,0 +1,541 @@
+
+/*
+ * nat.h - NAT plugin definitions
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_nat_h__
+#define __included_nat_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+
+#define SNAT_UDP_TIMEOUT 300
+#define SNAT_UDP_TIMEOUT_MIN 120
+#define SNAT_TCP_TRANSITORY_TIMEOUT 240
+#define SNAT_TCP_ESTABLISHED_TIMEOUT 7440
+#define SNAT_TCP_INCOMING_SYN 6
+#define SNAT_ICMP_TIMEOUT 60
+
+#define SNAT_FLAG_HAIRPINNING (1 << 0)
+
+/* Key */
+typedef struct {
+  union
+  {
+    struct
+    {
+      ip4_address_t addr;
+      u16 port;
+      u16 protocol:3,
+        fib_index:13;
+    };
+    u64 as_u64;
+  };
+} snat_session_key_t;
+
+typedef struct {
+  union
+  {
+    struct
+    {
+      ip4_address_t l_addr;
+      ip4_address_t r_addr;
+      u32 fib_index;
+      u8 proto;
+      u8 rsvd[3];
+    };
+    u64 as_u64[2];
+  };
+} snat_unk_proto_ses_key_t;
+
+typedef struct {
+  union
+  {
+    struct
+    {
+      ip4_address_t ext_host_addr;
+      u16 ext_host_port;
+      u16 out_port;
+    };
+    u64 as_u64;
+  };
+} snat_det_out_key_t;
+
+typedef struct {
+  union
+  {
+    struct
+    {
+      ip4_address_t addr;
+      u32 fib_index;
+    };
+    u64 as_u64;
+  };
+} snat_user_key_t;
+
+typedef struct {
+  union
+  {
+    struct
+    {
+      ip4_address_t addr;
+      u16 port;
+      u16 fib_index;
+    };
+    u64 as_u64;
+  };
+} snat_worker_key_t;
+
+
+#define foreach_snat_protocol \
+  _(UDP, 0, udp, "udp")       \
+  _(TCP, 1, tcp, "tcp")       \
+  _(ICMP, 2, icmp, "icmp")
+
+typedef enum {
+#define _(N, i, n, s) SNAT_PROTOCOL_##N = i,
+  foreach_snat_protocol
+#undef _
+} snat_protocol_t;
+
+
+#define foreach_snat_session_state          \
+  _(0, UNKNOWN, "unknown")                 \
+  _(1, UDP_ACTIVE, "udp-active")           \
+  _(2, TCP_SYN_SENT, "tcp-syn-sent")       \
+  _(3, TCP_ESTABLISHED, "tcp-established") \
+  _(4, TCP_FIN_WAIT, "tcp-fin-wait")       \
+  _(5, TCP_CLOSE_WAIT, "tcp-close-wait")   \
+  _(6, TCP_LAST_ACK, "tcp-last-ack")       \
+  _(7, ICMP_ACTIVE, "icmp-active")
+
+typedef enum {
+#define _(v, N, s) SNAT_SESSION_##N = v,
+  foreach_snat_session_state
+#undef _
+} snat_session_state_t;
+
+
+#define SNAT_SESSION_FLAG_STATIC_MAPPING 1
+#define SNAT_SESSION_FLAG_UNKNOWN_PROTO  2
+
+typedef CLIB_PACKED(struct {
+  snat_session_key_t out2in;    /* 0-15 */
+
+  snat_session_key_t in2out;    /* 16-31 */
+
+  u32 flags;                    /* 32-35 */
+
+  /* per-user translations */
+  u32 per_user_index;           /* 36-39 */
+
+  u32 per_user_list_head_index; /* 40-43 */
+
+  /* Last heard timer */
+  f64 last_heard;               /* 44-51 */
+
+  u64 total_bytes;              /* 52-59 */
+
+  u32 total_pkts;               /* 60-63 */
+
+  /* Outside address */
+  u32 outside_address_index;    /* 64-67 */
+
+  /* External host address */
+  ip4_address_t ext_host_addr;  /* 68-71 */
+
+}) snat_session_t;
+
+
+typedef struct {
+  ip4_address_t addr;
+  u32 fib_index;
+  u32 sessions_per_user_list_head_index;
+  u32 nsessions;
+  u32 nstaticsessions;
+} snat_user_t;
+
+typedef struct {
+  ip4_address_t addr;
+  u32 fib_index;
+#define _(N, i, n, s) \
+  u32 busy_##n##_ports; \
+  uword * busy_##n##_port_bitmap;
+  foreach_snat_protocol
+#undef _
+} snat_address_t;
+
+typedef struct {
+  u16 in_port;
+  snat_det_out_key_t out;
+  u8 state;
+  u32 expire;
+} snat_det_session_t;
+
+typedef struct {
+  ip4_address_t in_addr;
+  u8 in_plen;
+  ip4_address_t out_addr;
+  u8 out_plen;
+  u32 sharing_ratio;
+  u16 ports_per_host;
+  u32 ses_num;
+  /* vector of sessions */
+  snat_det_session_t * sessions;
+} snat_det_map_t;
+
+typedef struct {
+  ip4_address_t local_addr;
+  ip4_address_t external_addr;
+  u16 local_port;
+  u16 external_port;
+  u8 addr_only;
+  u32 vrf_id;
+  u32 fib_index;
+  snat_protocol_t proto;
+} snat_static_mapping_t;
+
+typedef struct {
+  u32 sw_if_index;
+  u8 is_inside;
+} snat_interface_t;
+
+typedef struct {
+  ip4_address_t l_addr;
+  u16 l_port;
+  u16 e_port;
+  u32 sw_if_index;
+  u32 vrf_id;
+  snat_protocol_t proto;
+  int addr_only;
+  int is_add;
+} snat_static_map_resolve_t;
+
+typedef struct {
+  /* User pool */
+  snat_user_t * users;
+
+  /* Session pool */
+  snat_session_t * sessions;
+
+  /* Pool of doubly-linked list elements */
+  dlist_elt_t * list_pool;
+
+  u32 snat_thread_index;
+} snat_main_per_thread_data_t;
+
+struct snat_main_s;
+
+typedef u32 snat_icmp_match_function_t (struct snat_main_s *sm,
+                                        vlib_node_runtime_t *node,
+                                        u32 thread_index,
+                                        vlib_buffer_t *b0,
+                                        u8 *p_proto,
+                                        snat_session_key_t *p_value,
+                                        u8 *p_dont_translate,
+                                        void *d,
+                                        void *e);
+
+typedef u32 (snat_get_worker_function_t) (ip4_header_t * ip, u32 rx_fib_index);
+
+typedef struct snat_main_s {
+  /* Main lookup tables */
+  clib_bihash_8_8_t out2in;
+  clib_bihash_8_8_t in2out;
+
+  /* Unknown protocol sessions lookup tables */
+  clib_bihash_16_8_t out2in_unk_proto;
+  clib_bihash_16_8_t in2out_unk_proto;
+
+  /* Find-a-user => src address lookup */
+  clib_bihash_8_8_t user_hash;
+
+  /* Non-translated packets worker lookup => src address + VRF */
+  clib_bihash_8_8_t worker_by_in;
+
+  /* Translated packets worker lookup => IP address + port number */
+  clib_bihash_8_8_t worker_by_out;
+
+  snat_icmp_match_function_t * icmp_match_in2out_cb;
+  snat_icmp_match_function_t * icmp_match_out2in_cb;
+
+  u32 num_workers;
+  u32 first_worker_index;
+  u32 next_worker;
+  u32 * workers;
+  snat_get_worker_function_t * worker_in2out_cb;
+  snat_get_worker_function_t * worker_out2in_cb;
+  u16 port_per_thread;
+  u32 num_snat_thread;
+
+  /* Per thread data */
+  snat_main_per_thread_data_t * per_thread_data;
+
+  /* Find a static mapping by local */
+  clib_bihash_8_8_t static_mapping_by_local;
+
+  /* Find a static mapping by external */
+  clib_bihash_8_8_t static_mapping_by_external;
+
+  /* Static mapping pool */
+  snat_static_mapping_t * static_mappings;
+
+  /* Interface pool */
+  snat_interface_t * interfaces;
+  snat_interface_t * output_feature_interfaces;
+
+  /* Vector of outside addresses */
+  snat_address_t * addresses;
+
+  /* sw_if_indices whose intfc addresses should be auto-added */
+  u32 * auto_add_sw_if_indices;
+
+  /* vector of interface address static mappings to resolve. */
+  snat_static_map_resolve_t *to_resolve;
+
+  /* Randomize port allocation order */
+  u32 random_seed;
+
+  /* Worker handoff index */
+  u32 fq_in2out_index;
+  u32 fq_in2out_output_index;
+  u32 fq_out2in_index;
+
+  /* in2out and out2in node index */
+  u32 in2out_node_index;
+  u32 in2out_output_node_index;
+  u32 out2in_node_index;
+
+  /* Deterministic NAT */
+  snat_det_map_t * det_maps;
+
+  /* Config parameters */
+  u8 static_mapping_only;
+  u8 static_mapping_connection_tracking;
+  u8 deterministic;
+  u32 translation_buckets;
+  u32 translation_memory_size;
+  u32 user_buckets;
+  u32 user_memory_size;
+  u32 max_translations_per_user;
+  u32 outside_vrf_id;
+  u32 outside_fib_index;
+  u32 inside_vrf_id;
+  u32 inside_fib_index;
+
+  /* tenant VRF aware address pool activation flag */
+  u8 vrf_mode;
+
+  /* values of various timeouts */
+  u32 udp_timeout;
+  u32 tcp_established_timeout;
+  u32 tcp_transitory_timeout;
+  u32 icmp_timeout;
+
+  /* API message ID base */
+  u16 msg_id_base;
+
+  /* convenience */
+  vlib_main_t * vlib_main;
+  vnet_main_t * vnet_main;
+  ip4_main_t * ip4_main;
+  ip_lookup_main_t * ip4_lookup_main;
+  api_main_t * api_main;
+} snat_main_t;
+
+extern snat_main_t snat_main;
+extern vlib_node_registration_t snat_in2out_node;
+extern vlib_node_registration_t snat_in2out_output_node;
+extern vlib_node_registration_t snat_out2in_node;
+extern vlib_node_registration_t snat_in2out_fast_node;
+extern vlib_node_registration_t snat_out2in_fast_node;
+extern vlib_node_registration_t snat_in2out_worker_handoff_node;
+extern vlib_node_registration_t snat_in2out_output_worker_handoff_node;
+extern vlib_node_registration_t snat_out2in_worker_handoff_node;
+extern vlib_node_registration_t snat_det_in2out_node;
+extern vlib_node_registration_t snat_det_out2in_node;
+extern vlib_node_registration_t snat_hairpin_dst_node;
+extern vlib_node_registration_t snat_hairpin_src_node;
+
+void snat_free_outside_address_and_port (snat_main_t * sm,
+                                         snat_session_key_t * k,
+                                         u32 address_index);
+
+int snat_alloc_outside_address_and_port (snat_main_t * sm,
+                                         u32 fib_index,
+                                         u32 thread_index,
+                                         snat_session_key_t * k,
+                                         u32 * address_indexp);
+
+int snat_static_mapping_match (snat_main_t * sm,
+                               snat_session_key_t match,
+                               snat_session_key_t * mapping,
+                               u8 by_external,
+                               u8 *is_addr_only);
+
+void snat_add_del_addr_to_fib (ip4_address_t * addr,
+                               u8 p_len,
+                               u32 sw_if_index,
+                               int is_add);
+
+format_function_t format_snat_user;
+
+typedef struct {
+  u32 cached_sw_if_index;
+  u32 cached_ip4_address;
+} snat_runtime_t;
+
+/** \brief Check if SNAT session is created from static mapping.
+    @param s SNAT session
+    @return 1 if SNAT session is created from static mapping otherwise 0
+*/
+#define snat_is_session_static(s) s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING
+
+/** \brief Check if SNAT session for unknown protocol.
+    @param s SNAT session
+    @return 1 if SNAT session for unknown protocol otherwise 0
+*/
+#define snat_is_unk_proto_session(s) s->flags & SNAT_SESSION_FLAG_UNKNOWN_PROTO
+
+/*
+ * Why is this here? Because we don't need to touch this layer to
+ * simply reply to an icmp. We need to change id to a unique
+ * value to NAT an echo request/reply.
+ */
+
+typedef struct {
+  u16 identifier;
+  u16 sequence;
+} icmp_echo_header_t;
+
+always_inline u32
+ip_proto_to_snat_proto (u8 ip_proto)
+{
+  u32 snat_proto = ~0;
+
+  snat_proto = (ip_proto == IP_PROTOCOL_UDP) ? SNAT_PROTOCOL_UDP : snat_proto;
+  snat_proto = (ip_proto == IP_PROTOCOL_TCP) ? SNAT_PROTOCOL_TCP : snat_proto;
+  snat_proto = (ip_proto == IP_PROTOCOL_ICMP) ? SNAT_PROTOCOL_ICMP : snat_proto;
+  snat_proto = (ip_proto == IP_PROTOCOL_ICMP6) ? SNAT_PROTOCOL_ICMP : snat_proto;
+
+  return snat_proto;
+}
+
+always_inline u8
+snat_proto_to_ip_proto (snat_protocol_t snat_proto)
+{
+  u8 ip_proto = ~0;
+
+  ip_proto = (snat_proto == SNAT_PROTOCOL_UDP) ? IP_PROTOCOL_UDP : ip_proto;
+  ip_proto = (snat_proto == SNAT_PROTOCOL_TCP) ? IP_PROTOCOL_TCP : ip_proto;
+  ip_proto = (snat_proto == SNAT_PROTOCOL_ICMP) ? IP_PROTOCOL_ICMP : ip_proto;
+
+  return ip_proto;
+}
+
+typedef struct {
+  u16 src_port, dst_port;
+} tcp_udp_header_t;
+
+u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e);
+u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e);
+u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
+                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                          snat_session_key_t *p_value,
+                          u8 *p_dont_translate, void *d, void *e);
+u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e);
+u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e);
+u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
+                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                          snat_session_key_t *p_value,
+                          u8 *p_dont_translate, void *d, void *e);
+void increment_v4_address(ip4_address_t * a);
+void snat_add_address(snat_main_t *sm, ip4_address_t *addr, u32 vrf_id);
+int snat_del_address(snat_main_t *sm, ip4_address_t addr, u8 delete_sm);
+int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
+                            u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
+                            u32 sw_if_index, snat_protocol_t proto, int is_add);
+clib_error_t * snat_api_init(vlib_main_t * vm, snat_main_t * sm);
+int snat_set_workers (uword * bitmap);
+int snat_interface_add_del(u32 sw_if_index, u8 is_inside, int is_del);
+int snat_interface_add_del_output_feature(u32 sw_if_index, u8 is_inside,
+                                          int is_del);
+int snat_add_interface_address(snat_main_t *sm, u32 sw_if_index, int is_del);
+uword unformat_snat_protocol(unformat_input_t * input, va_list * args);
+u8 * format_snat_protocol(u8 * s, va_list * args);
+
+static_always_inline u8
+icmp_is_error_message (icmp46_header_t * icmp)
+{
+  switch(icmp->type)
+    {
+    case ICMP4_destination_unreachable:
+    case ICMP4_time_exceeded:
+    case ICMP4_parameter_problem:
+    case ICMP4_source_quench:
+    case ICMP4_redirect:
+    case ICMP4_alternate_host_address:
+      return 1;
+    }
+  return 0;
+}
+
+static_always_inline u8
+is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0,
+                  u32 ip4_addr)
+{
+  snat_runtime_t *rt = (snat_runtime_t *) node->runtime_data;
+  ip4_address_t * first_int_addr;
+
+  if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
+    {
+      first_int_addr =
+        ip4_interface_first_address (sm->ip4_main, sw_if_index0,
+                                     0 /* just want the address */);
+      rt->cached_sw_if_index = sw_if_index0;
+      if (first_int_addr)
+        rt->cached_ip4_address = first_int_addr->as_u32;
+      else
+        rt->cached_ip4_address = 0;
+    }
+
+  if (PREDICT_FALSE(ip4_addr == rt->cached_ip4_address))
+    return 1;
+  else
+    return 0;
+}
+
+#endif /* __included_nat_h__ */
diff --git a/src/plugins/nat/nat64.c b/src/plugins/nat/nat64.c
new file mode 100644
index 00000000..b04901fa
--- /dev/null
+++ b/src/plugins/nat/nat64.c
@@ -0,0 +1,858 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 implementation
+ */
+
+#include 
+#include 
+#include 
+
+
+nat64_main_t nat64_main;
+
+/* *INDENT-OFF* */
+
+/* Hook up input features */
+VNET_FEATURE_INIT (nat64_in2out, static) = {
+  .arc_name = "ip6-unicast",
+  .node_name = "nat64-in2out",
+  .runs_before = VNET_FEATURES ("ip6-lookup"),
+};
+VNET_FEATURE_INIT (nat64_out2in, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat64-out2in",
+  .runs_before = VNET_FEATURES ("ip4-lookup"),
+};
+
+static u8 well_known_prefix[] = {
+  0x00, 0x64, 0xff, 0x9b,
+  0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00
+};
+
+/* *INDENT-ON* */
+
+clib_error_t *
+nat64_init (vlib_main_t * vm)
+{
+  nat64_main_t *nm = &nat64_main;
+  clib_error_t *error = 0;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+
+  nm->is_disabled = 0;
+
+  if (tm->n_vlib_mains > 1)
+    {
+      nm->is_disabled = 1;
+      goto error;
+    }
+
+  if (nat64_db_init (&nm->db))
+    {
+      error = clib_error_return (0, "NAT64 DB init failed");
+      goto error;
+    }
+
+  /* set session timeouts to default values */
+  nm->udp_timeout = SNAT_UDP_TIMEOUT;
+  nm->icmp_timeout = SNAT_ICMP_TIMEOUT;
+  nm->tcp_trans_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
+  nm->tcp_est_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
+  nm->tcp_incoming_syn_timeout = SNAT_TCP_INCOMING_SYN;
+
+error:
+  return error;
+}
+
+int
+nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add)
+{
+  nat64_main_t *nm = &nat64_main;
+  snat_address_t *a = 0;
+  snat_interface_t *interface;
+  int i;
+
+  /* Check if address already exists */
+  for (i = 0; i < vec_len (nm->addr_pool); i++)
+    {
+      if (nm->addr_pool[i].addr.as_u32 == addr->as_u32)
+	{
+	  a = nm->addr_pool + i;
+	  break;
+	}
+    }
+
+  if (is_add)
+    {
+      if (a)
+	return VNET_API_ERROR_VALUE_EXIST;
+
+      vec_add2 (nm->addr_pool, a, 1);
+      a->addr = *addr;
+      a->fib_index = 0;
+      if (vrf_id != ~0)
+	a->fib_index =
+	  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
+#define _(N, i, n, s) \
+      clib_bitmap_alloc (a->busy_##n##_port_bitmap, 65535);
+      foreach_snat_protocol
+#undef _
+    }
+  else
+    {
+      if (!a)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      if (a->fib_index)
+	fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP6);
+
+#define _(N, id, n, s) \
+      clib_bitmap_free (a->busy_##n##_port_bitmap);
+      foreach_snat_protocol
+#undef _
+	/* Delete sessions using address */
+	nat64_db_free_out_addr (&nm->db, &a->addr);
+      vec_del1 (nm->addr_pool, i);
+    }
+
+  /* Add/del external address to FIB */
+  /* *INDENT-OFF* */
+  pool_foreach (interface, nm->interfaces,
+  ({
+    if (interface->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib (addr, 32, interface->sw_if_index, is_add);
+    break;
+  }));
+  /* *INDENT-ON* */
+
+  return 0;
+}
+
+void
+nat64_pool_addr_walk (nat64_pool_addr_walk_fn_t fn, void *ctx)
+{
+  nat64_main_t *nm = &nat64_main;
+  snat_address_t *a = 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (a, nm->addr_pool)
+    {
+      if (fn (a, ctx))
+        break;
+    };
+  /* *INDENT-ON* */
+}
+
+int
+nat64_add_del_interface (u32 sw_if_index, u8 is_inside, u8 is_add)
+{
+  nat64_main_t *nm = &nat64_main;
+  snat_interface_t *interface = 0, *i;
+  snat_address_t *ap;
+  const char *feature_name, *arc_name;
+
+  /* Check if address already exists */
+  /* *INDENT-OFF* */
+  pool_foreach (i, nm->interfaces,
+  ({
+    if (i->sw_if_index == sw_if_index)
+      {
+        interface = i;
+        break;
+      }
+  }));
+  /* *INDENT-ON* */
+
+  if (is_add)
+    {
+      if (interface)
+	return VNET_API_ERROR_VALUE_EXIST;
+
+      pool_get (nm->interfaces, interface);
+      interface->sw_if_index = sw_if_index;
+      interface->is_inside = is_inside;
+
+    }
+  else
+    {
+      if (!interface)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      pool_put (nm->interfaces, interface);
+    }
+
+  if (!is_inside)
+    {
+      /* *INDENT-OFF* */
+      vec_foreach (ap, nm->addr_pool)
+        snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, is_add);
+      /* *INDENT-ON* */
+    }
+
+  arc_name = is_inside ? "ip6-unicast" : "ip4-unicast";
+  feature_name = is_inside ? "nat64-in2out" : "nat64-out2in";
+
+  return vnet_feature_enable_disable (arc_name, feature_name, sw_if_index,
+				      is_add, 0, 0);
+}
+
+void
+nat64_interfaces_walk (nat64_interface_walk_fn_t fn, void *ctx)
+{
+  nat64_main_t *nm = &nat64_main;
+  snat_interface_t *i = 0;
+
+  /* *INDENT-OFF* */
+  pool_foreach (i, nm->interfaces,
+  ({
+    if (fn (i, ctx))
+      break;
+  }));
+  /* *INDENT-ON* */
+}
+
+int
+nat64_alloc_out_addr_and_port (u32 fib_index, snat_protocol_t proto,
+			       ip4_address_t * addr, u16 * port)
+{
+  nat64_main_t *nm = &nat64_main;
+  snat_main_t *sm = &snat_main;
+  int i;
+  snat_address_t *a, *ga = 0;
+  u32 portnum;
+
+  for (i = 0; i < vec_len (nm->addr_pool); i++)
+    {
+      a = nm->addr_pool + i;
+      switch (proto)
+	{
+#define _(N, j, n, s) \
+        case SNAT_PROTOCOL_##N: \
+          if (a->busy_##n##_ports < (65535-1024)) \
+            { \
+              if (a->fib_index == fib_index) \
+                { \
+                  while (1) \
+                    { \
+                      portnum = random_u32 (&sm->random_seed); \
+                      portnum &= 0xFFFF; \
+                      if (portnum < 1024) \
+                        continue; \
+                      if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
+                                                    portnum)) \
+                        continue; \
+                      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
+                                                portnum, 1); \
+                      a->busy_##n##_ports++; \
+                      *port = portnum; \
+                      addr->as_u32 = a->addr.as_u32; \
+                      return 0; \
+                    } \
+                 } \
+               else if (a->fib_index == 0) \
+                 ga = a; \
+            } \
+          break;
+	  foreach_snat_protocol
+#undef _
+	default:
+	  clib_warning ("unknown protocol");
+	  return 1;
+	}
+    }
+
+  if (ga)
+    {
+      switch (proto)
+	{
+#define _(N, j, n, s) \
+        case SNAT_PROTOCOL_##N: \
+          while (1) \
+            { \
+              portnum = random_u32 (&sm->random_seed); \
+              portnum &= 0xFFFF; \
+              if (portnum < 1024) \
+                continue; \
+              if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
+                                            portnum)) \
+                continue; \
+              clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
+                                        portnum, 1); \
+              a->busy_##n##_ports++; \
+              *port = portnum; \
+              addr->as_u32 = a->addr.as_u32; \
+              return 0; \
+            }
+	  break;
+	  foreach_snat_protocol
+#undef _
+	default:
+	  clib_warning ("unknown protocol");
+	  return 1;
+	}
+    }
+
+  /* Totally out of translations to use... */
+  //TODO: IPFix
+  return 1;
+}
+
+void
+nat64_free_out_addr_and_port (ip4_address_t * addr, u16 port,
+			      snat_protocol_t proto)
+{
+  nat64_main_t *nm = &nat64_main;
+  int i;
+  snat_address_t *a;
+
+  for (i = 0; i < vec_len (nm->addr_pool); i++)
+    {
+      a = nm->addr_pool + i;
+      if (addr->as_u32 != a->addr.as_u32)
+	continue;
+      switch (proto)
+	{
+#define _(N, j, n, s) \
+        case SNAT_PROTOCOL_##N: \
+          ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
+                  port) == 1); \
+          clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port, 0); \
+          a->busy_##n##_ports--; \
+          break;
+	  foreach_snat_protocol
+#undef _
+	default:
+	  clib_warning ("unknown protocol");
+	  return;
+	}
+      break;
+    }
+}
+
+int
+nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
+				ip4_address_t * out_addr, u16 in_port,
+				u16 out_port, u8 proto, u32 vrf_id, u8 is_add)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_db_bib_entry_t *bibe;
+  u32 fib_index =
+    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
+  snat_protocol_t p = ip_proto_to_snat_proto (proto);
+  ip46_address_t addr;
+  int i;
+  snat_address_t *a;
+
+  addr.as_u64[0] = in_addr->as_u64[0];
+  addr.as_u64[1] = in_addr->as_u64[1];
+  bibe =
+    nat64_db_bib_entry_find (&nm->db, &addr, clib_host_to_net_u16 (in_port),
+			     proto, fib_index, 1);
+
+  if (is_add)
+    {
+      if (bibe)
+	return VNET_API_ERROR_VALUE_EXIST;
+
+      for (i = 0; i < vec_len (nm->addr_pool); i++)
+	{
+	  a = nm->addr_pool + i;
+	  if (out_addr->as_u32 != a->addr.as_u32)
+	    continue;
+	  switch (p)
+	    {
+#define _(N, j, n, s) \
+            case SNAT_PROTOCOL_##N: \
+              if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
+                                            out_port)) \
+                return VNET_API_ERROR_INVALID_VALUE; \
+              clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
+                                        out_port, 1); \
+              if (out_port > 1024) \
+                a->busy_##n##_ports++; \
+              break;
+	      foreach_snat_protocol
+#undef _
+	    default:
+	      memset (&addr, 0, sizeof (addr));
+	      addr.ip4.as_u32 = out_addr->as_u32;
+	      if (nat64_db_bib_entry_find
+		  (&nm->db, &addr, 0, proto, fib_index, 0))
+		return VNET_API_ERROR_INVALID_VALUE;
+	    }
+	  break;
+	}
+      bibe =
+	nat64_db_bib_entry_create (&nm->db, in_addr, out_addr,
+				   clib_host_to_net_u16 (in_port),
+				   clib_host_to_net_u16 (out_port), fib_index,
+				   proto, 1);
+      if (!bibe)
+	return VNET_API_ERROR_UNSPECIFIED;
+    }
+  else
+    {
+      if (!bibe)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      nat64_free_out_addr_and_port (out_addr, out_port, p);
+      nat64_db_bib_entry_free (&nm->db, bibe);
+    }
+
+  return 0;
+}
+
+int
+nat64_set_udp_timeout (u32 timeout)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (timeout == 0)
+    nm->udp_timeout = SNAT_UDP_TIMEOUT;
+  else if (timeout < SNAT_UDP_TIMEOUT_MIN)
+    return VNET_API_ERROR_INVALID_VALUE;
+  else
+    nm->udp_timeout = timeout;
+
+  return 0;
+}
+
+u32
+nat64_get_udp_timeout (void)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  return nm->udp_timeout;
+}
+
+int
+nat64_set_icmp_timeout (u32 timeout)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (timeout == 0)
+    nm->icmp_timeout = SNAT_ICMP_TIMEOUT;
+  else
+    nm->icmp_timeout = timeout;
+
+  return 0;
+}
+
+u32
+nat64_get_icmp_timeout (void)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  return nm->icmp_timeout;
+}
+
+int
+nat64_set_tcp_timeouts (u32 trans, u32 est, u32 incoming_syn)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (trans == 0)
+    nm->tcp_trans_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
+  else
+    nm->tcp_trans_timeout = trans;
+
+  if (est == 0)
+    nm->tcp_est_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
+  else
+    nm->tcp_est_timeout = est;
+
+  if (incoming_syn == 0)
+    nm->tcp_incoming_syn_timeout = SNAT_TCP_INCOMING_SYN;
+  else
+    nm->tcp_incoming_syn_timeout = incoming_syn;
+
+  return 0;
+}
+
+u32
+nat64_get_tcp_trans_timeout (void)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  return nm->tcp_trans_timeout;
+}
+
+u32
+nat64_get_tcp_est_timeout (void)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  return nm->tcp_est_timeout;
+}
+
+u32
+nat64_get_tcp_incoming_syn_timeout (void)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  return nm->tcp_incoming_syn_timeout;
+}
+
+void
+nat64_session_reset_timeout (nat64_db_st_entry_t * ste, vlib_main_t * vm)
+{
+  nat64_main_t *nm = &nat64_main;
+  u32 now = (u32) vlib_time_now (vm);
+
+  switch (ip_proto_to_snat_proto (ste->proto))
+    {
+    case SNAT_PROTOCOL_ICMP:
+      ste->expire = now + nm->icmp_timeout;
+      return;
+    case SNAT_PROTOCOL_TCP:
+      {
+	switch (ste->tcp_state)
+	  {
+	  case NAT64_TCP_STATE_V4_INIT:
+	  case NAT64_TCP_STATE_V6_INIT:
+	  case NAT64_TCP_STATE_V4_FIN_RCV:
+	  case NAT64_TCP_STATE_V6_FIN_RCV:
+	  case NAT64_TCP_STATE_V6_FIN_V4_FIN_RCV:
+	  case NAT64_TCP_STATE_TRANS:
+	    ste->expire = now + nm->tcp_trans_timeout;
+	    return;
+	  case NAT64_TCP_STATE_ESTABLISHED:
+	    ste->expire = now + nm->tcp_est_timeout;
+	    return;
+	  default:
+	    return;
+	  }
+      }
+    case SNAT_PROTOCOL_UDP:
+      ste->expire = now + nm->udp_timeout;
+      return;
+    default:
+      ste->expire = now + nm->udp_timeout;
+      return;
+    }
+}
+
+void
+nat64_tcp_session_set_state (nat64_db_st_entry_t * ste, tcp_header_t * tcp,
+			     u8 is_ip6)
+{
+  switch (ste->tcp_state)
+    {
+    case NAT64_TCP_STATE_CLOSED:
+      {
+	if (tcp->flags & TCP_FLAG_SYN)
+	  {
+	    if (is_ip6)
+	      ste->tcp_state = NAT64_TCP_STATE_V6_INIT;
+	    else
+	      ste->tcp_state = NAT64_TCP_STATE_V4_INIT;
+	  }
+	return;
+      }
+    case NAT64_TCP_STATE_V4_INIT:
+      {
+	if (is_ip6 && (tcp->flags & TCP_FLAG_SYN))
+	  ste->tcp_state = NAT64_TCP_STATE_ESTABLISHED;
+	return;
+      }
+    case NAT64_TCP_STATE_V6_INIT:
+      {
+	if (!is_ip6 && (tcp->flags & TCP_FLAG_SYN))
+	  ste->tcp_state = NAT64_TCP_STATE_ESTABLISHED;
+	return;
+      }
+    case NAT64_TCP_STATE_ESTABLISHED:
+      {
+	if (tcp->flags & TCP_FLAG_FIN)
+	  {
+	    if (is_ip6)
+	      ste->tcp_state = NAT64_TCP_STATE_V6_FIN_RCV;
+	    else
+	      ste->tcp_state = NAT64_TCP_STATE_V4_FIN_RCV;
+	  }
+	else if (tcp->flags & TCP_FLAG_RST)
+	  {
+	    ste->tcp_state = NAT64_TCP_STATE_TRANS;
+	  }
+	return;
+      }
+    case NAT64_TCP_STATE_V4_FIN_RCV:
+      {
+	if (is_ip6 && (tcp->flags & TCP_FLAG_FIN))
+	  ste->tcp_state = NAT64_TCP_STATE_V6_FIN_V4_FIN_RCV;
+	return;
+      }
+    case NAT64_TCP_STATE_V6_FIN_RCV:
+      {
+	if (!is_ip6 && (tcp->flags & TCP_FLAG_FIN))
+	  ste->tcp_state = NAT64_TCP_STATE_V6_FIN_V4_FIN_RCV;
+	return;
+      }
+    case NAT64_TCP_STATE_TRANS:
+      {
+	if (!(tcp->flags & TCP_FLAG_RST))
+	  ste->tcp_state = NAT64_TCP_STATE_ESTABLISHED;
+	return;
+      }
+    default:
+      return;
+    }
+}
+
+int
+nat64_add_del_prefix (ip6_address_t * prefix, u8 plen, u32 vrf_id, u8 is_add)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_prefix_t *p = 0;
+  int i;
+
+  /* Verify prefix length */
+  if (plen != 32 && plen != 40 && plen != 48 && plen != 56 && plen != 64
+      && plen != 96)
+    return VNET_API_ERROR_INVALID_VALUE;
+
+  /* Check if tenant already have prefix */
+  for (i = 0; i < vec_len (nm->pref64); i++)
+    {
+      if (nm->pref64[i].vrf_id == vrf_id)
+	{
+	  p = nm->pref64 + i;
+	  break;
+	}
+    }
+
+  if (is_add)
+    {
+      if (!p)
+	{
+	  vec_add2 (nm->pref64, p, 1);
+	  p->fib_index =
+	    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
+	  p->vrf_id = vrf_id;
+	}
+
+      p->prefix.as_u64[0] = prefix->as_u64[0];
+      p->prefix.as_u64[1] = prefix->as_u64[1];
+      p->plen = plen;
+    }
+  else
+    {
+      if (!p)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      vec_del1 (nm->pref64, i);
+    }
+
+  return 0;
+}
+
+void
+nat64_prefix_walk (nat64_prefix_walk_fn_t fn, void *ctx)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_prefix_t *p = 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (p, nm->pref64)
+    {
+      if (fn (p, ctx))
+        break;
+    };
+  /* *INDENT-ON* */
+}
+
+void
+nat64_compose_ip6 (ip6_address_t * ip6, ip4_address_t * ip4, u32 fib_index)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_prefix_t *p, *gp = 0, *prefix = 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (p, nm->pref64)
+    {
+      if (p->fib_index == fib_index)
+        {
+          prefix = p;
+          break;
+        }
+
+      if (p->fib_index == 0)
+        gp = p;
+    };
+  /* *INDENT-ON* */
+
+  if (!prefix)
+    prefix = gp;
+
+  if (prefix)
+    {
+      memset (ip6, 0, 16);
+      memcpy (ip6, &p->prefix, p->plen);
+      switch (p->plen)
+	{
+	case 32:
+	  ip6->as_u32[1] = ip4->as_u32;
+	  break;
+	case 40:
+	  ip6->as_u8[5] = ip4->as_u8[0];
+	  ip6->as_u8[6] = ip4->as_u8[1];
+	  ip6->as_u8[7] = ip4->as_u8[2];
+	  ip6->as_u8[9] = ip4->as_u8[3];
+	  break;
+	case 48:
+	  ip6->as_u8[6] = ip4->as_u8[0];
+	  ip6->as_u8[7] = ip4->as_u8[1];
+	  ip6->as_u8[9] = ip4->as_u8[2];
+	  ip6->as_u8[10] = ip4->as_u8[3];
+	  break;
+	case 56:
+	  ip6->as_u8[7] = ip4->as_u8[0];
+	  ip6->as_u8[9] = ip4->as_u8[1];
+	  ip6->as_u8[10] = ip4->as_u8[2];
+	  ip6->as_u8[11] = ip4->as_u8[3];
+	  break;
+	case 64:
+	  ip6->as_u8[9] = ip4->as_u8[0];
+	  ip6->as_u8[10] = ip4->as_u8[1];
+	  ip6->as_u8[11] = ip4->as_u8[2];
+	  ip6->as_u8[12] = ip4->as_u8[3];
+	  break;
+	case 96:
+	  ip6->as_u32[3] = ip4->as_u32;
+	  break;
+	default:
+	  clib_warning ("invalid prefix length");
+	  break;
+	}
+    }
+  else
+    {
+      memcpy (ip6, well_known_prefix, 16);
+      ip6->as_u32[3] = ip4->as_u32;
+    }
+}
+
+void
+nat64_extract_ip4 (ip6_address_t * ip6, ip4_address_t * ip4, u32 fib_index)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_prefix_t *p, *gp = 0;
+  u8 plen = 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (p, nm->pref64)
+    {
+      if (p->fib_index == fib_index)
+        {
+          plen = p->plen;
+          break;
+        }
+
+      if (p->vrf_id == 0)
+        gp = p;
+    };
+  /* *INDENT-ON* */
+
+  if (!plen)
+    {
+      if (gp)
+	plen = gp->plen;
+      else
+	plen = 96;
+    }
+
+  switch (plen)
+    {
+    case 32:
+      ip4->as_u32 = ip6->as_u32[1];
+      break;
+    case 40:
+      ip4->as_u8[0] = ip6->as_u8[5];
+      ip4->as_u8[1] = ip6->as_u8[6];
+      ip4->as_u8[2] = ip6->as_u8[7];
+      ip4->as_u8[3] = ip6->as_u8[9];
+      break;
+    case 48:
+      ip4->as_u8[0] = ip6->as_u8[6];
+      ip4->as_u8[1] = ip6->as_u8[7];
+      ip4->as_u8[2] = ip6->as_u8[9];
+      ip4->as_u8[3] = ip6->as_u8[10];
+      break;
+    case 56:
+      ip4->as_u8[0] = ip6->as_u8[7];
+      ip4->as_u8[1] = ip6->as_u8[9];
+      ip4->as_u8[2] = ip6->as_u8[10];
+      ip4->as_u8[3] = ip6->as_u8[11];
+      break;
+    case 64:
+      ip4->as_u8[0] = ip6->as_u8[9];
+      ip4->as_u8[1] = ip6->as_u8[10];
+      ip4->as_u8[2] = ip6->as_u8[11];
+      ip4->as_u8[3] = ip6->as_u8[12];
+      break;
+    case 96:
+      ip4->as_u32 = ip6->as_u32[3];
+      break;
+    default:
+      clib_warning ("invalid prefix length");
+      break;
+    }
+}
+
+/**
+ * @brief The 'nat64-expire-walk' process's main loop.
+ *
+ * Check expire time for NAT64 sessions.
+ */
+static uword
+nat64_expire_walk_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
+		      vlib_frame_t * f)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  while (!nm->is_disabled)
+    {
+      vlib_process_wait_for_event_or_clock (vm, 10.0);
+      vlib_process_get_events (vm, NULL);
+      u32 now = (u32) vlib_time_now (vm);
+
+      nad64_db_st_free_expired (&nm->db, now);
+    }
+
+  return 0;
+}
+
+static vlib_node_registration_t nat64_expire_walk_node;
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat64_expire_walk_node, static) = {
+    .function = nat64_expire_walk_fn,
+    .type = VLIB_NODE_TYPE_PROCESS,
+    .name = "nat64-expire-walk",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat64.h b/src/plugins/nat/nat64.h
new file mode 100644
index 00000000..68224cab
--- /dev/null
+++ b/src/plugins/nat/nat64.h
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 global declarations
+ */
+#ifndef __included_nat64_h__
+#define __included_nat64_h__
+
+#include 
+#include 
+
+#define foreach_nat64_tcp_ses_state            \
+  _(0, CLOSED, "closed")                       \
+  _(1, V4_INIT, "v4-init")                     \
+  _(2, V6_INIT, "v6-init")                     \
+  _(3, ESTABLISHED, "established")             \
+  _(4, V4_FIN_RCV, "v4-fin-rcv")               \
+  _(5, V6_FIN_RCV, "v6-fin-rcv")               \
+  _(6, V6_FIN_V4_FIN_RCV, "v6-fin-v4-fin-rcv") \
+  _(7, TRANS, "trans")
+
+typedef enum
+{
+#define _(v, N, s) NAT64_TCP_STATE_##N = v,
+  foreach_nat64_tcp_ses_state
+#undef _
+} nat64_tcp_ses_state_t;
+
+typedef struct
+{
+  ip6_address_t prefix;
+  u8 plen;
+  u32 vrf_id;
+  u32 fib_index;
+} nat64_prefix_t;
+
+typedef struct
+{
+  /** Interface pool */
+  snat_interface_t *interfaces;
+
+  /** Address pool vector */
+  snat_address_t *addr_pool;
+
+  /** Pref64 vector */
+  nat64_prefix_t *pref64;
+
+  /** BIB and session DB */
+  nat64_db_t db;
+
+  /* values of various timeouts */
+  u32 udp_timeout;
+  u32 icmp_timeout;
+  u32 tcp_trans_timeout;
+  u32 tcp_est_timeout;
+  u32 tcp_incoming_syn_timeout;
+
+  u8 is_disabled;
+
+  snat_main_t *sm;
+} nat64_main_t;
+
+extern nat64_main_t nat64_main;
+extern vlib_node_registration_t nat64_in2out_node;
+extern vlib_node_registration_t nat64_out2in_node;
+
+/**
+ * @brief Add/delete address to NAT64 pool.
+ *
+ * @param addr   IPv4 address.
+ * @param vrf_id VRF id of tenant, ~0 means independent of VRF.
+ * @param is_add 1 if add, 0 if delete.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add);
+
+/**
+ * @brief Call back function when walking addresses in NAT64 pool, non-zero
+ * return value stop walk.
+ */
+typedef int (*nat64_pool_addr_walk_fn_t) (snat_address_t * addr, void *ctx);
+
+/**
+ * @brief Walk NAT64 pool.
+ *
+ * @param fn The function to invoke on each entry visited.
+ * @param ctx A context passed in the visit function.
+ */
+void nat64_pool_addr_walk (nat64_pool_addr_walk_fn_t fn, void *ctx);
+
+/**
+ * @brief Enable/disable NAT64 feature on the interface.
+ *
+ * @param sw_if_index Index of the interface.
+ * @param is_inside   1 if inside, 0 if outside.
+ * @param is_add      1 if add, 0 if delete.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_add_del_interface (u32 sw_if_index, u8 is_inside, u8 is_add);
+
+/**
+ * @brief Call back function when walking interfaces with NAT64 feature,
+ * non-zero return value stop walk.
+ */
+typedef int (*nat64_interface_walk_fn_t) (snat_interface_t * i, void *ctx);
+
+/**
+ * @brief Walk NAT64 interfaces.
+ *
+ * @param fn The function to invoke on each entry visited.
+ * @param ctx A context passed in the visit function.
+ */
+void nat64_interfaces_walk (nat64_interface_walk_fn_t fn, void *ctx);
+
+/**
+ * @brief Initialize NAT64.
+ *
+ * @param vm vlib main.
+ *
+ * @return error code.
+ */
+clib_error_t *nat64_init (vlib_main_t * vm);
+
+/**
+ * @brief Add/delete static NAT64 BIB entry.
+ *
+ * @param in_addr  Inside IPv6 address.
+ * @param out_addr Outside IPv4 address.
+ * @param in_port  Inside port number.
+ * @param out_port Outside port number.
+ * @param proto    L4 protocol.
+ * @param vrf_id   VRF id of tenant.
+ * @param is_add   1 if add, 0 if delete.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
+				    ip4_address_t * out_addr, u16 in_port,
+				    u16 out_port, u8 proto, u32 vrf_id,
+				    u8 is_add);
+
+/**
+ * @brief Alloce IPv4 address and port pair from NAT64 pool.
+ *
+ * @param fib_index FIB index of tenant.
+ * @param proto     L4 protocol.
+ * @param addr      Allocated IPv4 address.
+ * @param port      Allocated port number.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_alloc_out_addr_and_port (u32 fib_index, snat_protocol_t proto,
+				   ip4_address_t * addr, u16 * port);
+
+/**
+ * @brief Free IPv4 address and port pair from NAT64 pool.
+ *
+ * @param addr  IPv4 address to free.
+ * @param port  Port number to free.
+ * @param proto L4 protocol.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+void nat64_free_out_addr_and_port (ip4_address_t * addr, u16 port,
+				   snat_protocol_t proto);
+
+/**
+ * @brief Set UDP session timeout.
+ *
+ * @param timeout Timeout value in seconds (if 0 reset to default value 300sec).
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_set_udp_timeout (u32 timeout);
+
+/**
+ * @brief Get UDP session timeout.
+ *
+ * @returns UDP session timeout in seconds.
+ */
+u32 nat64_get_udp_timeout (void);
+
+/**
+ * @brief Set ICMP session timeout.
+ *
+ * @param timeout Timeout value in seconds (if 0 reset to default value 60sec).
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_set_icmp_timeout (u32 timeout);
+
+/**
+ * @brief Get ICMP session timeout.
+ *
+ * @returns ICMP session timeout in seconds.
+ */
+u32 nat64_get_icmp_timeout (void);
+
+/**
+ * @brief Set TCP session timeouts.
+ *
+ * @param trans Transitory timeout in seconds (if 0 reset to default value 240sec).
+ * @param est Established timeout in seconds (if 0 reset to default value 7440sec).
+ * @param incoming_syn Incoming SYN timeout in seconds (if 0 reset to default value 6sec).
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_set_tcp_timeouts (u32 trans, u32 est, u32 incoming_syn);
+
+/**
+ * @brief Get TCP transitory timeout.
+ *
+ * @returns TCP transitory timeout in seconds.
+ */
+u32 nat64_get_tcp_trans_timeout (void);
+
+/**
+ * @brief Get TCP established timeout.
+ *
+ * @returns TCP established timeout in seconds.
+ */
+u32 nat64_get_tcp_est_timeout (void);
+
+/**
+ * @brief Get TCP incoming SYN timeout.
+ *
+ * @returns TCP incoming SYN timeout in seconds.
+ */
+u32 nat64_get_tcp_incoming_syn_timeout (void);
+
+/**
+ * @brief Reset NAT64 session timeout.
+ *
+ * @param ste Session table entry.
+ * @param vm VLIB main.
+ **/
+void nat64_session_reset_timeout (nat64_db_st_entry_t * ste,
+				  vlib_main_t * vm);
+
+/**
+ * @brief Set NAT64 TCP session state.
+ *
+ * @param ste Session table entry.
+ * @param tcp TCP header.
+ * @param is_ip6 1 if IPv6 packet, 0 if IPv4.
+ */
+void nat64_tcp_session_set_state (nat64_db_st_entry_t * ste,
+				  tcp_header_t * tcp, u8 is_ip6);
+
+/**
+ * @brief Add/delete NAT64 prefix.
+ *
+ * @param prefix NAT64 prefix.
+ * @param plen Prefix length.
+ * @param vrf_id VRF id of tenant.
+ * @param is_add 1 if add, 0 if delete.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_add_del_prefix (ip6_address_t * prefix, u8 plen, u32 vrf_id,
+			  u8 is_add);
+
+/**
+ * @brief Call back function when walking addresses in NAT64 prefixes, non-zero
+ * return value stop walk.
+ */
+typedef int (*nat64_prefix_walk_fn_t) (nat64_prefix_t * pref64, void *ctx);
+
+/**
+ * @brief Walk NAT64 prefixes.
+ *
+ * @param fn The function to invoke on each entry visited.
+ * @param ctx A context passed in the visit function.
+ */
+void nat64_prefix_walk (nat64_prefix_walk_fn_t fn, void *ctx);
+
+/**
+ * Compose IPv4-embedded IPv6 addresses.
+ * @param ip6 IPv4-embedded IPv6 addresses.
+ * @param ip4 IPv4 address.
+ * @param fib_index Tenant FIB index.
+ */
+void nat64_compose_ip6 (ip6_address_t * ip6, ip4_address_t * ip4,
+			u32 fib_index);
+
+/**
+ * Extract IPv4 address from the IPv4-embedded IPv6 addresses.
+ *
+ * @param ip6 IPv4-embedded IPv6 addresses.
+ * @param ip4 IPv4 address.
+ * @param fib_index Tenant FIB index.
+ */
+void nat64_extract_ip4 (ip6_address_t * ip6, ip4_address_t * ip4,
+			u32 fib_index);
+
+#define u8_ptr_add(ptr, index) (((u8 *)ptr) + index)
+#define u16_net_add(u, val) clib_host_to_net_u16(clib_net_to_host_u16(u) + (val))
+
+#endif /* __included_nat64_h__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat64_cli.c b/src/plugins/nat/nat64_cli.c
new file mode 100644
index 00000000..bb62ecf2
--- /dev/null
+++ b/src/plugins/nat/nat64_cli.c
@@ -0,0 +1,983 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 CLI
+ */
+
+#include 
+#include 
+#include 
+
+static clib_error_t *
+nat64_add_del_pool_addr_command_fn (vlib_main_t * vm,
+				    unformat_input_t * input,
+				    vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t start_addr, end_addr, this_addr;
+  u32 start_host_order, end_host_order;
+  int i, count, rv;
+  u32 vrf_id = ~0;
+  u8 is_add = 1;
+  clib_error_t *error = 0;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U - %U",
+		    unformat_ip4_address, &start_addr,
+		    unformat_ip4_address, &end_addr))
+	;
+      else if (unformat (line_input, "tenant-vrf %u", &vrf_id))
+	;
+      else if (unformat (line_input, "%U", unformat_ip4_address, &start_addr))
+	end_addr = start_addr;
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else
+	{
+	  error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
+  end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
+
+  if (end_host_order < start_host_order)
+    {
+      error = clib_error_return (0, "end address less than start address");
+      goto done;
+    }
+
+  count = (end_host_order - start_host_order) + 1;
+  this_addr = start_addr;
+
+  for (i = 0; i < count; i++)
+    {
+      rv = nat64_add_del_pool_addr (&this_addr, vrf_id, is_add);
+
+      switch (rv)
+	{
+	case VNET_API_ERROR_NO_SUCH_ENTRY:
+	  error =
+	    clib_error_return (0, "NAT64 pool address %U not exist.",
+			       format_ip4_address, &this_addr);
+	  goto done;
+	case VNET_API_ERROR_VALUE_EXIST:
+	  error =
+	    clib_error_return (0, "NAT64 pool address %U exist.",
+			       format_ip4_address, &this_addr);
+	  goto done;
+	default:
+	  break;
+
+	}
+      increment_v4_address (&this_addr);
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static int
+nat64_cli_pool_walk (snat_address_t * ap, void *ctx)
+{
+  vlib_main_t *vm = ctx;
+
+  if (ap->fib_index != ~0)
+    {
+      fib_table_t *fib;
+      fib = fib_table_get (ap->fib_index, FIB_PROTOCOL_IP6);
+      if (!fib)
+	return -1;
+      vlib_cli_output (vm, " %U tenant VRF: %u", format_ip4_address,
+		       &ap->addr, fib->ft_table_id);
+    }
+  else
+    vlib_cli_output (vm, " %U", format_ip4_address, &ap->addr);
+
+  return 0;
+}
+
+static clib_error_t *
+nat64_show_pool_command_fn (vlib_main_t * vm,
+			    unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  vlib_cli_output (vm, "NAT64 pool:");
+  nat64_pool_addr_walk (nat64_cli_pool_walk, vm);
+
+  return 0;
+}
+
+static clib_error_t *
+nat64_interface_feature_command_fn (vlib_main_t * vm,
+				    unformat_input_t *
+				    input, vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *error = 0;
+  u32 sw_if_index;
+  u32 *inside_sw_if_indices = 0;
+  u32 *outside_sw_if_indices = 0;
+  u8 is_add = 1;
+  int i, rv;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "in %U", unformat_vnet_sw_interface,
+		    vnm, &sw_if_index))
+	vec_add1 (inside_sw_if_indices, sw_if_index);
+      else if (unformat (line_input, "out %U", unformat_vnet_sw_interface,
+			 vnm, &sw_if_index))
+	vec_add1 (outside_sw_if_indices, sw_if_index);
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else
+	{
+	  error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (vec_len (inside_sw_if_indices))
+    {
+      for (i = 0; i < vec_len (inside_sw_if_indices); i++)
+	{
+	  sw_if_index = inside_sw_if_indices[i];
+	  rv = nat64_add_del_interface (sw_if_index, 1, is_add);
+	  switch (rv)
+	    {
+	    case VNET_API_ERROR_NO_SUCH_ENTRY:
+	      error =
+		clib_error_return (0, "%U NAT64 feature not enabled.",
+				   format_vnet_sw_interface_name, vnm,
+				   vnet_get_sw_interface (vnm, sw_if_index));
+	      goto done;
+	    case VNET_API_ERROR_VALUE_EXIST:
+	      error =
+		clib_error_return (0, "%U NAT64 feature already enabled.",
+				   format_vnet_sw_interface_name, vnm,
+				   vnet_get_sw_interface (vnm, sw_if_index));
+	      goto done;
+	    case VNET_API_ERROR_INVALID_VALUE:
+	    case VNET_API_ERROR_INVALID_VALUE_2:
+	      error =
+		clib_error_return (0,
+				   "%U NAT64 feature enable/disable failed.",
+				   format_vnet_sw_interface_name, vnm,
+				   vnet_get_sw_interface (vnm, sw_if_index));
+	      goto done;
+	    default:
+	      break;
+
+	    }
+	}
+    }
+
+  if (vec_len (outside_sw_if_indices))
+    {
+      for (i = 0; i < vec_len (outside_sw_if_indices); i++)
+	{
+	  sw_if_index = outside_sw_if_indices[i];
+	  rv = nat64_add_del_interface (sw_if_index, 0, is_add);
+	  switch (rv)
+	    {
+	    case VNET_API_ERROR_NO_SUCH_ENTRY:
+	      error =
+		clib_error_return (0, "%U NAT64 feature not enabled.",
+				   format_vnet_sw_interface_name, vnm,
+				   vnet_get_sw_interface (vnm, sw_if_index));
+	      goto done;
+	    case VNET_API_ERROR_VALUE_EXIST:
+	      error =
+		clib_error_return (0, "%U NAT64 feature already enabled.",
+				   format_vnet_sw_interface_name, vnm,
+				   vnet_get_sw_interface (vnm, sw_if_index));
+	      goto done;
+	    case VNET_API_ERROR_INVALID_VALUE:
+	    case VNET_API_ERROR_INVALID_VALUE_2:
+	      error =
+		clib_error_return (0,
+				   "%U NAT64 feature enable/disable failed.",
+				   format_vnet_sw_interface_name, vnm,
+				   vnet_get_sw_interface (vnm, sw_if_index));
+	      goto done;
+	    default:
+	      break;
+
+	    }
+	}
+    }
+
+done:
+  unformat_free (line_input);
+  vec_free (inside_sw_if_indices);
+  vec_free (outside_sw_if_indices);
+
+  return error;
+}
+
+static int
+nat64_cli_interface_walk (snat_interface_t * i, void *ctx)
+{
+  vlib_main_t *vm = ctx;
+  vnet_main_t *vnm = vnet_get_main ();
+
+  vlib_cli_output (vm, " %U %s", format_vnet_sw_interface_name, vnm,
+		   vnet_get_sw_interface (vnm, i->sw_if_index),
+		   i->is_inside ? "in" : "out");
+  return 0;
+}
+
+static clib_error_t *
+nat64_show_interfaces_command_fn (vlib_main_t * vm,
+				  unformat_input_t *
+				  input, vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  vlib_cli_output (vm, "NAT64 interfaces:");
+  nat64_interfaces_walk (nat64_cli_interface_walk, vm);
+
+  return 0;
+}
+
+static clib_error_t *
+nat64_add_del_static_bib_command_fn (vlib_main_t *
+				     vm,
+				     unformat_input_t
+				     * input, vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
+  u8 is_add = 1;
+  ip6_address_t in_addr;
+  ip4_address_t out_addr;
+  u16 in_port = 0;
+  u16 out_port = 0;
+  u32 vrf_id = 0, protocol;
+  snat_protocol_t proto = 0;
+  u8 p = 0;
+  int rv;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%U %u", unformat_ip6_address,
+		    &in_addr, &in_port))
+	;
+      else if (unformat (line_input, "%U %u", unformat_ip4_address,
+			 &out_addr, &out_port))
+	;
+      else if (unformat (line_input, "vrf %u", &vrf_id))
+	;
+      else if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
+	;
+      else
+	if (unformat
+	    (line_input, "%U %U %u", unformat_ip6_address, &in_addr,
+	     unformat_ip4_address, &out_addr, &protocol))
+	p = (u8) protocol;
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else
+	{
+	  error = clib_error_return (0, "unknown input: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (!p)
+    {
+      if (!in_port)
+	{
+	  error =
+	    clib_error_return (0, "inside port and address  must be set");
+	  goto done;
+	}
+
+      if (!out_port)
+	{
+	  error =
+	    clib_error_return (0, "outside port and address  must be set");
+	  goto done;
+	}
+
+      p = snat_proto_to_ip_proto (proto);
+    }
+
+  rv =
+    nat64_add_del_static_bib_entry (&in_addr, &out_addr, in_port, out_port, p,
+				    vrf_id, is_add);
+
+  switch (rv)
+    {
+    case VNET_API_ERROR_NO_SUCH_ENTRY:
+      error = clib_error_return (0, "NAT64 BIB entry not exist.");
+      goto done;
+    case VNET_API_ERROR_VALUE_EXIST:
+      error = clib_error_return (0, "NAT64 BIB entry exist.");
+      goto done;
+    case VNET_API_ERROR_UNSPECIFIED:
+      error = clib_error_return (0, "Crerate NAT64 BIB entry failed.");
+      goto done;
+    case VNET_API_ERROR_INVALID_VALUE:
+      error =
+	clib_error_return (0, "Outside addres %U and port %u already in use.",
+			   format_ip4_address, &out_addr, out_port);
+      goto done;
+    default:
+      break;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static int
+nat64_cli_bib_walk (nat64_db_bib_entry_t * bibe, void *ctx)
+{
+  vlib_main_t *vm = ctx;
+  fib_table_t *fib;
+
+  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
+  if (!fib)
+    return -1;
+
+  switch (bibe->proto)
+    {
+    case IP_PROTOCOL_ICMP:
+    case IP_PROTOCOL_TCP:
+    case IP_PROTOCOL_UDP:
+      vlib_cli_output (vm, " %U %u %U %u protocol %U vrf %u %s %u sessions",
+		       format_ip6_address, &bibe->in_addr,
+		       clib_net_to_host_u16 (bibe->in_port),
+		       format_ip4_address, &bibe->out_addr,
+		       clib_net_to_host_u16 (bibe->out_port),
+		       format_snat_protocol,
+		       ip_proto_to_snat_proto (bibe->proto), fib->ft_table_id,
+		       bibe->is_static ? "static" : "dynamic", bibe->ses_num);
+      break;
+    default:
+      vlib_cli_output (vm, " %U %U protocol %u vrf %u %s %u sessions",
+		       format_ip6_address, &bibe->in_addr,
+		       format_ip4_address, &bibe->out_addr,
+		       bibe->proto, fib->ft_table_id,
+		       bibe->is_static ? "static" : "dynamic", bibe->ses_num);
+    }
+  return 0;
+}
+
+static clib_error_t *
+nat64_show_bib_command_fn (vlib_main_t * vm,
+			   unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
+  u32 proto = ~0;
+  u8 p = 255;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
+    p = snat_proto_to_ip_proto (proto);
+  else if (unformat (line_input, "unknown"))
+    p = 0;
+  else if (unformat (line_input, "all"))
+    ;
+  else
+    {
+      error = clib_error_return (0, "unknown input: '%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
+
+  if (p == 255)
+    vlib_cli_output (vm, "NAT64 BIB entries:");
+  else
+    vlib_cli_output (vm, "NAT64 %U BIB entries:", format_snat_protocol,
+		     proto);
+  nat64_db_bib_walk (&nm->db, p, nat64_cli_bib_walk, vm);
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static clib_error_t *
+nat64_set_timeouts_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			       vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
+  u32 timeout, tcp_trans, tcp_est, tcp_incoming_syn;
+
+  tcp_trans = nat64_get_tcp_trans_timeout ();
+  tcp_est = nat64_get_tcp_est_timeout ();
+  tcp_incoming_syn = nat64_get_tcp_incoming_syn_timeout ();
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "udp %u", &timeout))
+	{
+	  if (nat64_set_udp_timeout (timeout))
+	    {
+	      error = clib_error_return (0, "Invalid UDP timeout value");
+	      goto done;
+	    }
+	}
+      else if (unformat (line_input, "icmp %u", &timeout))
+	{
+	  if (nat64_set_icmp_timeout (timeout))
+	    {
+	      error = clib_error_return (0, "Invalid ICMP timeout value");
+	      goto done;
+	    }
+	}
+      else if (unformat (line_input, "tcp-trans %u", &tcp_trans))
+	{
+	  if (nat64_set_tcp_timeouts (tcp_trans, tcp_est, tcp_incoming_syn))
+	    {
+	      error =
+		clib_error_return (0, "Invalid TCP transitory tiemout value");
+	      goto done;
+	    }
+	}
+      else if (unformat (line_input, "tcp-est %u", &tcp_est))
+	{
+	  if (nat64_set_tcp_timeouts (tcp_trans, tcp_est, tcp_incoming_syn))
+	    {
+	      error =
+		clib_error_return (0,
+				   "Invalid TCP established tiemout value");
+	      goto done;
+	    }
+	}
+      else
+	if (unformat (line_input, "tcp-incoming-syn %u", &tcp_incoming_syn))
+	{
+	  if (nat64_set_tcp_timeouts (tcp_trans, tcp_est, tcp_incoming_syn))
+	    {
+	      error =
+		clib_error_return (0,
+				   "Invalid TCP incoming SYN tiemout value");
+	      goto done;
+	    }
+	}
+      else if (unformat (line_input, "reset"))
+	{
+	  nat64_set_udp_timeout (0);
+	  nat64_set_icmp_timeout (0);
+	  nat64_set_tcp_timeouts (0, 0, 0);
+	}
+      else
+	{
+	  error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static clib_error_t *
+nat64_show_timeouts_command_fn (vlib_main_t * vm, unformat_input_t * input,
+				vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  vlib_cli_output (vm, "NAT64 session timeouts:");
+  vlib_cli_output (vm, " UDP %usec", nat64_get_udp_timeout ());
+  vlib_cli_output (vm, " ICMP %usec", nat64_get_icmp_timeout ());
+  vlib_cli_output (vm, " TCP transitory %usec",
+		   nat64_get_tcp_trans_timeout ());
+  vlib_cli_output (vm, " TCP established %usec",
+		   nat64_get_tcp_est_timeout ());
+  vlib_cli_output (vm, " TCP incoming SYN %usec",
+		   nat64_get_tcp_incoming_syn_timeout ());
+
+  return 0;
+}
+
+static int
+nat64_cli_st_walk (nat64_db_st_entry_t * ste, void *ctx)
+{
+  vlib_main_t *vm = ctx;
+  nat64_main_t *nm = &nat64_main;
+  nat64_db_bib_entry_t *bibe;
+  fib_table_t *fib;
+
+  bibe = nat64_db_bib_entry_by_index (&nm->db, ste->proto, ste->bibe_index);
+  if (!bibe)
+    return -1;
+
+  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
+  if (!fib)
+    return -1;
+
+  u32 vrf_id = fib->ft_table_id;
+
+  if (ste->proto == IP_PROTOCOL_ICMP)
+    vlib_cli_output (vm, " %U %U %u %U %U %u protocol %U vrf %u",
+		     format_ip6_address, &bibe->in_addr,
+		     format_ip6_address, &ste->in_r_addr,
+		     clib_net_to_host_u16 (bibe->in_port),
+		     format_ip4_address, &bibe->out_addr,
+		     format_ip4_address, &ste->out_r_addr,
+		     clib_net_to_host_u16 (bibe->out_port),
+		     format_snat_protocol,
+		     ip_proto_to_snat_proto (bibe->proto), vrf_id);
+  else if (ste->proto == IP_PROTOCOL_TCP || ste->proto == IP_PROTOCOL_UDP)
+    vlib_cli_output (vm, " %U %u %U %u %U %u %U %u protcol %U vrf %u",
+		     format_ip6_address, &bibe->in_addr,
+		     clib_net_to_host_u16 (bibe->in_port),
+		     format_ip6_address, &ste->in_r_addr,
+		     clib_net_to_host_u16 (ste->r_port),
+		     format_ip4_address, &bibe->out_addr,
+		     clib_net_to_host_u16 (bibe->out_port),
+		     format_ip4_address, &ste->out_r_addr,
+		     clib_net_to_host_u16 (ste->r_port),
+		     format_snat_protocol,
+		     ip_proto_to_snat_proto (bibe->proto), vrf_id);
+  else
+    vlib_cli_output (vm, " %U %U %U %U protocol %u vrf %u",
+		     format_ip6_address, &bibe->in_addr,
+		     format_ip6_address, &ste->in_r_addr,
+		     format_ip4_address, &bibe->out_addr,
+		     format_ip4_address, &ste->out_r_addr,
+		     bibe->proto, vrf_id);
+
+  return 0;
+}
+
+static clib_error_t *
+nat64_show_st_command_fn (vlib_main_t * vm,
+			  unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
+  u32 proto = ~0;
+  u8 p = 255;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
+    p = snat_proto_to_ip_proto (proto);
+  else if (unformat (line_input, "unknown"))
+    p = 0;
+  else if (unformat (line_input, "all"))
+    ;
+  else
+    {
+      error = clib_error_return (0, "unknown input: '%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
+
+  if (p == 255)
+    vlib_cli_output (vm, "NAT64 sessions:");
+  else
+    vlib_cli_output (vm, "NAT64 %U sessions:", format_snat_protocol, proto);
+  nat64_db_st_walk (&nm->db, p, nat64_cli_st_walk, vm);
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static clib_error_t *
+nat64_add_del_prefix_command_fn (vlib_main_t * vm, unformat_input_t * input,
+				 vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+  clib_error_t *error = 0;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u8 is_add = 1;
+  u32 vrf_id = 0;
+  ip6_address_t prefix;
+  u32 plen = 0;
+  int rv;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U/%u", unformat_ip6_address, &prefix, &plen))
+	;
+      else if (unformat (line_input, "tenant-vrf %u", &vrf_id))
+	;
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else
+	{
+	  error = clib_error_return (0, "unknown input: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (!plen)
+    {
+      error = clib_error_return (0, "NAT64 prefix must be set.");
+      goto done;
+    }
+
+  rv = nat64_add_del_prefix (&prefix, (u8) plen, vrf_id, is_add);
+
+  switch (rv)
+    {
+    case VNET_API_ERROR_NO_SUCH_ENTRY:
+      error = clib_error_return (0, "NAT64 prefix not exist.");
+      goto done;
+    case VNET_API_ERROR_INVALID_VALUE:
+      error = clib_error_return (0, "Invalid prefix length.");
+      goto done;
+    default:
+      break;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static int
+nat64_cli_prefix_walk (nat64_prefix_t * p, void *ctx)
+{
+  vlib_main_t *vm = ctx;
+
+  vlib_cli_output (vm, " %U/%u tenant-vrf %u",
+		   format_ip6_address, &p->prefix, p->plen, p->vrf_id);
+
+  return 0;
+}
+
+static clib_error_t *
+nat64_show_prefix_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			      vlib_cli_command_t * cmd)
+{
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return clib_error_return (0,
+			      "NAT64 disabled, multi thread not supported");
+
+  vlib_cli_output (vm, "NAT64 prefix:");
+  nat64_prefix_walk (nat64_cli_prefix_walk, vm);
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+
+/*?
+ * @cliexpar
+ * @cliexstart{nat64 add pool address}
+ * Add/delete NAT64 pool address.
+ * To add single NAT64 pool address use:
+ *  vpp# nat64 add pool address 10.1.1.10
+ * To add NAT64 pool address range use:
+ *  vpp# nat64 add pool address 10.1.1.2 - 10.1.1.5
+ * To add NAT64 pool address for specific tenant use:
+ *  vpp# nat64 add pool address 10.1.1.100 tenant-vrf 100
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (nat64_add_pool_address_command, static) = {
+  .path = "nat64 add pool address",
+  .short_help = "nat64 add pool address  [- ] "
+                "[tenant-vrf ] [del]",
+  .function = nat64_add_del_pool_addr_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{show nat64 pool}
+ * Show NAT64 pool.
+ *  vpp# show nat64 pool
+ *  NAT64 pool:
+ *   10.1.1.3 tenant VRF: 0
+ *   10.1.1.10 tenant VRF: 10
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (show_nat64_pool_command, static) = {
+  .path = "show nat64 pool",
+  .short_help = "show nat64 pool",
+  .function = nat64_show_pool_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{set interface nat64}
+ * Enable/disable NAT64 feature on the interface.
+ * To enable NAT64 feature with local (IPv6) network interface
+ * GigabitEthernet0/8/0 and external (IPv4) network interface
+ * GigabitEthernet0/a/0 use:
+ *  vpp# set interface nat64 in GigabitEthernet0/8/0 out GigabitEthernet0/a/0
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (set_interface_nat64_command, static) = {
+  .path = "set interface nat64",
+  .short_help = "set interface nat64 in|out  [del]",
+  .function = nat64_interface_feature_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{show nat64 interfaces}
+ * Show interfaces with NAT64 feature.
+ * To show interfaces with NAT64 feature use:
+ *  vpp# show nat64 interfaces
+ *  NAT64 interfaces:
+ *   GigabitEthernet0/8/0 in
+ *   GigabitEthernet0/a/0 out
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (show_nat64_interfaces_command, static) = {
+  .path = "show nat64 interfaces",
+  .short_help = "show nat64 interfaces",
+  .function = nat64_show_interfaces_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{nat64 add static bib}
+ * Add/delete NAT64 static BIB entry.
+ * To create NAT64 satatic BIB entry use:
+ *  vpp# nat64 add static bib 2001:db8:c000:221:: 1234 10.1.1.3 5678 tcp
+ *  vpp# nat64 add static bib 2001:db8:c000:221:: 1234 10.1.1.3 5678 udp vrf 10
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (nat64_add_del_static_bib_command, static) = {
+  .path = "nat64 add static bib",
+  .short_help = "nat64 add static bib     "
+                "tcp|udp|icmp [vfr ] [del]",
+  .function = nat64_add_del_static_bib_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{show nat64 bib}
+ * Show NAT64 BIB entries.
+ * To show NAT64 TCP BIB entries use:
+ *  vpp# show nat64 bib tcp
+ *  NAT64 tcp BIB:
+ *   fd01:1::2 6303 10.0.0.3 62303 tcp vrf 0 dynamic 1 sessions
+ *   2001:db8:c000:221:: 1234 10.1.1.3 5678 tcp vrf 0 static 2 sessions
+ * To show NAT64 UDP BIB entries use:
+ *  vpp# show nat64 bib udp
+ *  NAT64 udp BIB:
+ *   fd01:1::2 6304 10.0.0.3 10546 udp vrf 0 dynamic 10 sessions
+ *   2001:db8:c000:221:: 1234 10.1.1.3 5678 udp vrf 10 static 0 sessions
+ * To show NAT64 ICMP BIB entries use:
+ *  vpp# show nat64 bib icmp
+ *  NAT64 icmp BIB:
+ *   fd01:1::2 6305 10.0.0.3 63209 icmp vrf 10 dynamic 1 sessions
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (show_nat64_bib_command, static) = {
+  .path = "show nat64 bib",
+  .short_help = "show nat64 bib all|tcp|udp|icmp|unknown",
+  .function = nat64_show_bib_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{set nat64 timeouts}
+ * Set NAT64 session timeouts (in seconds).
+ * To set NAT64 session timeoutes use use:
+ *  vpp# set nat64 timeouts udp 200 icmp 30 tcp-trans 250 tcp-est 7450
+ * To reset NAT64 session timeoutes to default values use:
+ *  vpp# set nat64 timeouts reset
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (set_nat64_timeouts_command, static) = {
+  .path = "set nat64 timeouts",
+  .short_help = "set nat64 timeouts udp  icmp  tcp-trans  "
+                "tcp-est  tcp-incoming-syn  | reset",
+  .function = nat64_set_timeouts_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{show nat64 tiemouts}
+ * Show NAT64 session timeouts:
+ *  vpp# show nat64 tiemouts
+ *  NAT64 session timeouts:
+ *   UDP 300sec
+ *   ICMP 60sec
+ *   TCP transitory 240sec
+ *   TCP established 7440sec
+ *   TCP incoming SYN 6sec
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (show_nat64_timeouts_command, static) = {
+  .path = "show nat64 tiemouts",
+  .short_help = "show nat64 tiemouts",
+  .function = nat64_show_timeouts_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{show nat64 session table}
+ * Show NAT64 session table.
+ * To show NAT64 TCP session table use:
+ *  vpp# show nat64 session table tcp
+ *  NAT64 tcp session table:
+ *   fd01:1::2 6303 64:ff9b::ac10:202 20 10.0.0.3 62303 172.16.2.2 20 tcp vrf 0
+ *   fd01:3::2 6303 64:ff9b::ac10:202 20 10.0.10.3 21300 172.16.2.2 20 tcp vrf 10
+ * To show NAT64 UDP session table use:
+ * #vpp show nat64 session table udp
+ * NAT64 udp session table:
+ *  fd01:1::2 6304 64:ff9b::ac10:202 20 10.0.0.3 10546 172.16.2.2 20 udp vrf 0
+ *  fd01:3::2 6304 64:ff9b::ac10:202 20 10.0.10.3 58627 172.16.2.2 20 udp vrf 10
+ *  fd01:1::2 1235 64:ff9b::a00:3 4023 10.0.0.3 24488 10.0.0.3 4023 udp vrf 0
+ *  fd01:1::3 23 64:ff9b::a00:3 24488 10.0.0.3 4023 10.0.0.3 24488 udp vrf 0
+ * To show NAT64 ICMP session table use:
+ * #vpp show nat64 session table icmp
+ * NAT64 icmp session table:
+ *  fd01:1::2 64:ff9b::ac10:202 6305 10.0.0.3 172.16.2.2 63209 icmp vrf 0
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (show_nat64_st_command, static) = {
+  .path = "show nat64 session table",
+  .short_help = "show nat64 session table all|tcp|udp|icmp|unknown",
+  .function = nat64_show_st_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{nat64 add prefix}
+ * Set NAT64 prefix for generating IPv6 representations of IPv4 addresses.
+ * To set NAT64 global prefix use:
+ *  vpp# nat64 add prefix 2001:db8::/32
+ * To set NAT64 prefix for specific tenant use:
+ *  vpp# nat64 add prefix 2001:db8:122:300::/56 tenant-vrf 10
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (nat64_add_del_prefix_command, static) = {
+  .path = "nat64 add prefix",
+  .short_help = "nat64 add prefix / [tenant-vrf ] "
+                "[del]",
+  .function = nat64_add_del_prefix_command_fn,
+};
+
+/*?
+ * @cliexpar
+ * @cliexstart{show nat64 prefix}
+ * Show NAT64 prefix.
+ * To show NAT64 prefix use:
+ *  vpp# show nat64 prefix
+ *  NAT64 prefix:
+ *   2001:db8::/32 tenant-vrf 0
+ *   2001:db8:122:300::/56 tenant-vrf 10
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (show_nat64_prefix_command, static) = {
+  .path = "show nat64 prefix",
+  .short_help = "show nat64 prefix",
+  .function = nat64_show_prefix_command_fn,
+};
+
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat64_db.c b/src/plugins/nat/nat64_db.c
new file mode 100644
index 00000000..da73ceee
--- /dev/null
+++ b/src/plugins/nat/nat64_db.c
@@ -0,0 +1,603 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 DB
+ */
+#include 
+
+int
+nat64_db_init (nat64_db_t * db)
+{
+  u32 bib_buckets = 1024;
+  u32 bib_memory_size = 128 << 20;
+  u32 st_buckets = 2048;
+  u32 st_memory_size = 256 << 20;
+
+  clib_bihash_init_24_8 (&db->bib.in2out, "bib-in2out", bib_buckets,
+			 bib_memory_size);
+
+  clib_bihash_init_24_8 (&db->bib.out2in, "bib-out2in", bib_buckets,
+			 bib_memory_size);
+
+  clib_bihash_init_48_8 (&db->st.in2out, "st-in2out", st_buckets,
+			 st_memory_size);
+
+  clib_bihash_init_48_8 (&db->st.out2in, "st-out2in", st_buckets,
+			 st_memory_size);
+
+  return 0;
+}
+
+nat64_db_bib_entry_t *
+nat64_db_bib_entry_create (nat64_db_t * db, ip6_address_t * in_addr,
+			   ip4_address_t * out_addr, u16 in_port,
+			   u16 out_port, u32 fib_index, u8 proto,
+			   u8 is_static)
+{
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_bib_entry_key_t bibe_key;
+  clib_bihash_kv_24_8_t kv;
+
+  /* create pool entry */
+  switch (ip_proto_to_snat_proto (proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      pool_get (db->bib._##n##_bib, bibe); \
+      kv.value = bibe - db->bib._##n##_bib; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      pool_get (db->bib._unk_proto_bib, bibe);
+      kv.value = bibe - db->bib._unk_proto_bib;
+      break;
+    }
+  memset (bibe, 0, sizeof (*bibe));
+  bibe->in_addr.as_u64[0] = in_addr->as_u64[0];
+  bibe->in_addr.as_u64[1] = in_addr->as_u64[1];
+  bibe->in_port = in_port;
+  bibe->out_addr.as_u32 = out_addr->as_u32;
+  bibe->out_port = out_port;
+  bibe->fib_index = fib_index;
+  bibe->proto = proto;
+  bibe->is_static = is_static;
+
+  /* create hash lookup */
+  bibe_key.addr.as_u64[0] = bibe->in_addr.as_u64[0];
+  bibe_key.addr.as_u64[1] = bibe->in_addr.as_u64[1];
+  bibe_key.fib_index = bibe->fib_index;
+  bibe_key.port = bibe->in_port;
+  bibe_key.proto = bibe->proto;
+  bibe_key.rsvd = 0;
+  kv.key[0] = bibe_key.as_u64[0];
+  kv.key[1] = bibe_key.as_u64[1];
+  kv.key[2] = bibe_key.as_u64[2];
+  clib_bihash_add_del_24_8 (&db->bib.in2out, &kv, 1);
+
+  memset (&bibe_key.addr, 0, sizeof (bibe_key.addr));
+  bibe_key.addr.ip4.as_u32 = bibe->out_addr.as_u32;
+  bibe_key.fib_index = 0;
+  bibe_key.port = bibe->out_port;
+  kv.key[0] = bibe_key.as_u64[0];
+  kv.key[1] = bibe_key.as_u64[1];
+  kv.key[2] = bibe_key.as_u64[2];
+  clib_bihash_add_del_24_8 (&db->bib.out2in, &kv, 1);
+
+  return bibe;
+}
+
+void
+nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe)
+{
+  nat64_db_bib_entry_key_t bibe_key;
+  clib_bihash_kv_24_8_t kv;
+  nat64_db_bib_entry_t *bib;
+  u32 *ste_to_be_free = 0, *ste_index, bibe_index;
+  nat64_db_st_entry_t *st, *ste;
+
+  switch (ip_proto_to_snat_proto (bibe->proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      bib = db->bib._##n##_bib; \
+      st = db->st._##n##_st; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      bib = db->bib._unk_proto_bib;
+      st = db->st._unk_proto_st;
+      break;
+    }
+
+  bibe_index = bibe - bib;
+
+  /* delete ST entries for static BIB entry */
+  if (bibe->is_static)
+    {
+      pool_foreach (ste, st, (
+			       {
+			       if (ste->bibe_index == bibe_index)
+			       vec_add1 (ste_to_be_free, ste - st);}
+		    ));
+      vec_foreach (ste_index, ste_to_be_free)
+	nat64_db_st_entry_free (db, pool_elt_at_index (st, ste_index[0]));
+      vec_free (ste_to_be_free);
+    }
+
+  /* delete hash lookup */
+  bibe_key.addr.as_u64[0] = bibe->in_addr.as_u64[0];
+  bibe_key.addr.as_u64[1] = bibe->in_addr.as_u64[1];
+  bibe_key.fib_index = bibe->fib_index;
+  bibe_key.port = bibe->in_port;
+  bibe_key.proto = bibe->proto;
+  bibe_key.rsvd = 0;
+  kv.key[0] = bibe_key.as_u64[0];
+  kv.key[1] = bibe_key.as_u64[1];
+  kv.key[2] = bibe_key.as_u64[2];
+  clib_bihash_add_del_24_8 (&db->bib.in2out, &kv, 0);
+
+  memset (&bibe_key.addr, 0, sizeof (bibe_key.addr));
+  bibe_key.addr.ip4.as_u32 = bibe->out_addr.as_u32;
+  bibe_key.fib_index = 0;
+  bibe_key.port = bibe->out_port;
+  kv.key[0] = bibe_key.as_u64[0];
+  kv.key[1] = bibe_key.as_u64[1];
+  kv.key[2] = bibe_key.as_u64[2];
+  clib_bihash_add_del_24_8 (&db->bib.out2in, &kv, 0);
+
+  /* delete from pool */
+  pool_put (bib, bibe);
+
+}
+
+nat64_db_bib_entry_t *
+nat64_db_bib_entry_find (nat64_db_t * db, ip46_address_t * addr, u16 port,
+			 u8 proto, u32 fib_index, u8 is_ip6)
+{
+  nat64_db_bib_entry_t *bibe = 0;
+  nat64_db_bib_entry_key_t bibe_key;
+  clib_bihash_kv_24_8_t kv, value;
+  nat64_db_bib_entry_t *bib;
+
+  switch (ip_proto_to_snat_proto (proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      bib = db->bib._##n##_bib; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      bib = db->bib._unk_proto_bib;
+      break;
+    }
+
+  bibe_key.addr.as_u64[0] = addr->as_u64[0];
+  bibe_key.addr.as_u64[1] = addr->as_u64[1];
+  bibe_key.fib_index = fib_index;
+  bibe_key.port = port;
+  bibe_key.proto = proto;
+  bibe_key.rsvd = 0;
+
+  kv.key[0] = bibe_key.as_u64[0];
+  kv.key[1] = bibe_key.as_u64[1];
+  kv.key[2] = bibe_key.as_u64[2];
+
+  if (!clib_bihash_search_24_8
+      (is_ip6 ? &db->bib.in2out : &db->bib.out2in, &kv, &value))
+    bibe = pool_elt_at_index (bib, value.value);
+
+  return bibe;
+}
+
+void
+nat64_db_bib_walk (nat64_db_t * db, u8 proto,
+		   nat64_db_bib_walk_fn_t fn, void *ctx)
+{
+  nat64_db_bib_entry_t *bib, *bibe;
+
+  if (proto == 255)
+    {
+    /* *INDENT-OFF* */
+    #define _(N, i, n, s) \
+      bib = db->bib._##n##_bib; \
+      pool_foreach (bibe, bib, ({ \
+        if (fn (bibe, ctx)) \
+          return; \
+      }));
+      foreach_snat_protocol
+    #undef _
+      bib = db->bib._unk_proto_bib;
+      pool_foreach (bibe, bib, ({
+        if (fn (bibe, ctx))
+          return;
+      }));
+    /* *INDENT-ON* */
+    }
+  else
+    {
+      switch (ip_proto_to_snat_proto (proto))
+	{
+    /* *INDENT-OFF* */
+    #define _(N, i, n, s) \
+        case SNAT_PROTOCOL_##N: \
+          bib = db->bib._##n##_bib; \
+          break;
+          foreach_snat_protocol
+    #undef _
+    /* *INDENT-ON* */
+	default:
+	  bib = db->bib._unk_proto_bib;
+	  break;
+	}
+
+      /* *INDENT-OFF* */
+      pool_foreach (bibe, bib,
+      ({
+        if (fn (bibe, ctx))
+          return;
+      }));
+      /* *INDENT-ON* */
+    }
+}
+
+nat64_db_bib_entry_t *
+nat64_db_bib_entry_by_index (nat64_db_t * db, u8 proto, u32 bibe_index)
+{
+  nat64_db_bib_entry_t *bib;
+
+  switch (ip_proto_to_snat_proto (proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      bib = db->bib._##n##_bib; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      bib = db->bib._unk_proto_bib;
+      break;
+    }
+
+  return pool_elt_at_index (bib, bibe_index);
+}
+
+void
+nat64_db_st_walk (nat64_db_t * db, u8 proto,
+		  nat64_db_st_walk_fn_t fn, void *ctx)
+{
+  nat64_db_st_entry_t *st, *ste;
+
+  if (proto == 255)
+    {
+    /* *INDENT-OFF* */
+    #define _(N, i, n, s) \
+      st = db->st._##n##_st; \
+      pool_foreach (ste, st, ({ \
+        if (fn (ste, ctx)) \
+          return; \
+      }));
+      foreach_snat_protocol
+    #undef _
+      st = db->st._unk_proto_st;
+      pool_foreach (ste, st, ({
+        if (fn (ste, ctx))
+          return;
+      }));
+    /* *INDENT-ON* */
+    }
+  else
+    {
+      switch (ip_proto_to_snat_proto (proto))
+	{
+    /* *INDENT-OFF* */
+    #define _(N, i, n, s) \
+        case SNAT_PROTOCOL_##N: \
+          st = db->st._##n##_st; \
+          break;
+          foreach_snat_protocol
+    #undef _
+    /* *INDENT-ON* */
+	default:
+	  st = db->st._unk_proto_st;
+	  break;
+	}
+
+      /* *INDENT-OFF* */
+      pool_foreach (ste, st,
+      ({
+        if (fn (ste, ctx))
+          return;
+      }));
+      /* *INDENT-ON* */
+    }
+}
+
+nat64_db_st_entry_t *
+nat64_db_st_entry_create (nat64_db_t * db, nat64_db_bib_entry_t * bibe,
+			  ip6_address_t * in_r_addr,
+			  ip4_address_t * out_r_addr, u16 r_port)
+{
+  nat64_db_st_entry_t *ste;
+  nat64_db_bib_entry_t *bib;
+  nat64_db_st_entry_key_t ste_key;
+  clib_bihash_kv_48_8_t kv;
+
+  /* create pool entry */
+  switch (ip_proto_to_snat_proto (bibe->proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      pool_get (db->st._##n##_st, ste); \
+      kv.value = ste - db->st._##n##_st; \
+      bib = db->bib._##n##_bib; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      pool_get (db->st._unk_proto_st, ste);
+      kv.value = ste - db->st._unk_proto_st;
+      bib = db->bib._unk_proto_bib;
+      break;
+    }
+  memset (ste, 0, sizeof (*ste));
+  ste->in_r_addr.as_u64[0] = in_r_addr->as_u64[0];
+  ste->in_r_addr.as_u64[1] = in_r_addr->as_u64[1];
+  ste->out_r_addr.as_u32 = out_r_addr->as_u32;
+  ste->r_port = r_port;
+  ste->bibe_index = bibe - bib;
+  ste->proto = bibe->proto;
+
+  /* increment session number for BIB entry */
+  bibe->ses_num++;
+
+  /* create hash lookup */
+  memset (&ste_key, 0, sizeof (ste_key));
+  ste_key.l_addr.as_u64[0] = bibe->in_addr.as_u64[0];
+  ste_key.l_addr.as_u64[1] = bibe->in_addr.as_u64[1];
+  ste_key.r_addr.as_u64[0] = ste->in_r_addr.as_u64[0];
+  ste_key.r_addr.as_u64[1] = ste->in_r_addr.as_u64[1];
+  ste_key.fib_index = bibe->fib_index;
+  ste_key.l_port = bibe->in_port;
+  ste_key.r_port = ste->r_port;
+  ste_key.proto = ste->proto;
+  kv.key[0] = ste_key.as_u64[0];
+  kv.key[1] = ste_key.as_u64[1];
+  kv.key[2] = ste_key.as_u64[2];
+  kv.key[3] = ste_key.as_u64[3];
+  kv.key[4] = ste_key.as_u64[4];
+  kv.key[5] = ste_key.as_u64[5];
+  clib_bihash_add_del_48_8 (&db->st.in2out, &kv, 1);
+
+  memset (&ste_key, 0, sizeof (ste_key));
+  ste_key.l_addr.ip4.as_u32 = bibe->out_addr.as_u32;
+  ste_key.r_addr.ip4.as_u32 = ste->out_r_addr.as_u32;
+  ste_key.l_port = bibe->out_port;
+  ste_key.r_port = ste->r_port;
+  ste_key.proto = ste->proto;
+  kv.key[0] = ste_key.as_u64[0];
+  kv.key[1] = ste_key.as_u64[1];
+  kv.key[2] = ste_key.as_u64[2];
+  kv.key[3] = ste_key.as_u64[3];
+  kv.key[4] = ste_key.as_u64[4];
+  kv.key[5] = ste_key.as_u64[5];
+  clib_bihash_add_del_48_8 (&db->st.out2in, &kv, 1);
+
+  return ste;
+}
+
+void
+nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste)
+{
+  nat64_db_st_entry_t *st;
+  nat64_db_bib_entry_t *bib, *bibe;
+  nat64_db_st_entry_key_t ste_key;
+  clib_bihash_kv_48_8_t kv;
+
+  switch (ip_proto_to_snat_proto (ste->proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      st = db->st._##n##_st; \
+      bib = db->bib._##n##_bib; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      st = db->st._unk_proto_st;
+      bib = db->bib._unk_proto_bib;
+      break;
+    }
+
+  bibe = pool_elt_at_index (bib, ste->bibe_index);
+
+  /* delete hash lookup */
+  memset (&ste_key, 0, sizeof (ste_key));
+  ste_key.l_addr.as_u64[0] = bibe->in_addr.as_u64[0];
+  ste_key.l_addr.as_u64[1] = bibe->in_addr.as_u64[1];
+  ste_key.r_addr.as_u64[0] = ste->in_r_addr.as_u64[0];
+  ste_key.r_addr.as_u64[1] = ste->in_r_addr.as_u64[1];
+  ste_key.fib_index = bibe->fib_index;
+  ste_key.l_port = bibe->in_port;
+  ste_key.r_port = ste->r_port;
+  ste_key.proto = ste->proto;
+  kv.key[0] = ste_key.as_u64[0];
+  kv.key[1] = ste_key.as_u64[1];
+  kv.key[2] = ste_key.as_u64[2];
+  kv.key[3] = ste_key.as_u64[3];
+  kv.key[4] = ste_key.as_u64[4];
+  kv.key[5] = ste_key.as_u64[5];
+  clib_bihash_add_del_48_8 (&db->st.in2out, &kv, 0);
+
+  memset (&ste_key, 0, sizeof (ste_key));
+  ste_key.l_addr.ip4.as_u32 = bibe->out_addr.as_u32;
+  ste_key.r_addr.ip4.as_u32 = ste->out_r_addr.as_u32;
+  ste_key.l_port = bibe->out_port;
+  ste_key.r_port = ste->r_port;
+  ste_key.proto = ste->proto;
+  kv.key[0] = ste_key.as_u64[0];
+  kv.key[1] = ste_key.as_u64[1];
+  kv.key[2] = ste_key.as_u64[2];
+  kv.key[3] = ste_key.as_u64[3];
+  kv.key[4] = ste_key.as_u64[4];
+  kv.key[5] = ste_key.as_u64[5];
+  clib_bihash_add_del_48_8 (&db->st.out2in, &kv, 0);
+
+  /* delete from pool */
+  pool_put (st, ste);
+
+  /* decrement session number for BIB entry */
+  bibe->ses_num--;
+
+  /* delete BIB entry if last session and dynamic */
+  if (!bibe->is_static && !bibe->ses_num)
+    nat64_db_bib_entry_free (db, bibe);
+}
+
+nat64_db_st_entry_t *
+nat64_db_st_entry_find (nat64_db_t * db, ip46_address_t * l_addr,
+			ip46_address_t * r_addr, u16 l_port, u16 r_port,
+			u8 proto, u32 fib_index, u8 is_ip6)
+{
+  nat64_db_st_entry_t *ste = 0;
+  nat64_db_st_entry_t *st;
+  nat64_db_st_entry_key_t ste_key;
+  clib_bihash_kv_48_8_t kv, value;
+
+  switch (ip_proto_to_snat_proto (proto))
+    {
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+    case SNAT_PROTOCOL_##N: \
+      st = db->st._##n##_st; \
+      break;
+      foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+    default:
+      st = db->st._unk_proto_st;
+      break;
+    }
+
+  memset (&ste_key, 0, sizeof (ste_key));
+  ste_key.l_addr.as_u64[0] = l_addr->as_u64[0];
+  ste_key.l_addr.as_u64[1] = l_addr->as_u64[1];
+  ste_key.r_addr.as_u64[0] = r_addr->as_u64[0];
+  ste_key.r_addr.as_u64[1] = r_addr->as_u64[1];
+  ste_key.fib_index = fib_index;
+  ste_key.l_port = l_port;
+  ste_key.r_port = r_port;
+  ste_key.proto = proto;
+  kv.key[0] = ste_key.as_u64[0];
+  kv.key[1] = ste_key.as_u64[1];
+  kv.key[2] = ste_key.as_u64[2];
+  kv.key[3] = ste_key.as_u64[3];
+  kv.key[4] = ste_key.as_u64[4];
+  kv.key[5] = ste_key.as_u64[5];
+
+  if (!clib_bihash_search_48_8
+      (is_ip6 ? &db->st.in2out : &db->st.out2in, &kv, &value))
+    ste = pool_elt_at_index (st, value.value);
+
+  return ste;
+}
+
+void
+nad64_db_st_free_expired (nat64_db_t * db, u32 now)
+{
+  u32 *ste_to_be_free = 0, *ste_index;
+  nat64_db_st_entry_t *st, *ste;
+
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+  st = db->st._##n##_st; \
+  pool_foreach (ste, st, ({\
+    if (i == SNAT_PROTOCOL_TCP && !ste->tcp_state) \
+      continue; \
+    if (ste->expire < now) \
+      vec_add1 (ste_to_be_free, ste - st); \
+  })); \
+  vec_foreach (ste_index, ste_to_be_free) \
+    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0])); \
+  vec_free (ste_to_be_free); \
+  ste_to_be_free = 0;
+  foreach_snat_protocol
+#undef _
+  st = db->st._unk_proto_st;
+  pool_foreach (ste, st, ({
+    if (ste->expire < now)
+      vec_add1 (ste_to_be_free, ste - st);
+  }));
+  vec_foreach (ste_index, ste_to_be_free)
+    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0]));
+  vec_free (ste_to_be_free);
+/* *INDENT-ON* */
+}
+
+void
+nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr)
+{
+  u32 *ste_to_be_free = 0, *ste_index;
+  nat64_db_st_entry_t *st, *ste;
+  nat64_db_bib_entry_t *bibe;
+
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+  st = db->st._##n##_st; \
+  pool_foreach (ste, st, ({ \
+    bibe = pool_elt_at_index (db->bib._##n##_bib, ste->bibe_index); \
+    if (bibe->out_addr.as_u32 == out_addr->as_u32) \
+      vec_add1 (ste_to_be_free, ste - st); \
+  })); \
+  vec_foreach (ste_index, ste_to_be_free) \
+    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0])); \
+  vec_free (ste_to_be_free); \
+  ste_to_be_free = 0;
+  foreach_snat_protocol
+#undef _
+  st = db->st._unk_proto_st;
+  pool_foreach (ste, st, ({
+    bibe = pool_elt_at_index (db->bib._unk_proto_bib, ste->bibe_index);
+    if (bibe->out_addr.as_u32 == out_addr->as_u32)
+      vec_add1 (ste_to_be_free, ste - st);
+  }));
+  vec_foreach (ste_index, ste_to_be_free)
+    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0]));
+  vec_free (ste_to_be_free);
+/* *INDENT-ON* */
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat64_db.h b/src/plugins/nat/nat64_db.h
new file mode 100644
index 00000000..394ca875
--- /dev/null
+++ b/src/plugins/nat/nat64_db.h
@@ -0,0 +1,307 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 DB
+ */
+#ifndef __included_nat64_db_h__
+#define __included_nat64_db_h__
+
+#include 
+#include 
+#include 
+
+
+typedef struct
+{
+  union
+  {
+    struct
+    {
+      ip46_address_t addr;
+      u32 fib_index;
+      u16 port;
+      u8 proto;
+      u8 rsvd;
+    };
+    u64 as_u64[3];
+  };
+} nat64_db_bib_entry_key_t;
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED(struct
+{
+  ip6_address_t in_addr;
+  u16 in_port;
+  ip4_address_t out_addr;
+  u16 out_port;
+  u32 fib_index;
+  u32 ses_num;
+  u8 proto;
+  u8 is_static;
+}) nat64_db_bib_entry_t;
+/* *INDENT-ON* */
+
+typedef struct
+{
+  /* BIBs */
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+  nat64_db_bib_entry_t *_##n##_bib;
+  foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+  nat64_db_bib_entry_t *_unk_proto_bib;
+
+  /* BIB lookup */
+  clib_bihash_24_8_t in2out;
+  clib_bihash_24_8_t out2in;
+} nat64_db_bib_t;
+
+typedef struct
+{
+  union
+  {
+    struct
+    {
+      ip46_address_t l_addr;
+      ip46_address_t r_addr;
+      u32 fib_index;
+      u16 l_port;
+      u16 r_port;
+      u8 proto;
+      u8 rsvd[7];
+    };
+    u64 as_u64[6];
+  };
+} nat64_db_st_entry_key_t;
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED(struct
+{
+  ip6_address_t in_r_addr;
+  ip4_address_t out_r_addr;
+  u16 r_port;
+  u32 bibe_index;
+  u32 expire;
+  u8 proto;
+  u8 tcp_state;
+}) nat64_db_st_entry_t;
+/* *INDENT-ON* */
+
+typedef struct
+{
+  /* session tables */
+/* *INDENT-OFF* */
+#define _(N, i, n, s) \
+  nat64_db_st_entry_t *_##n##_st;
+  foreach_snat_protocol
+#undef _
+/* *INDENT-ON* */
+  nat64_db_st_entry_t *_unk_proto_st;
+
+  /* session lookup */
+  clib_bihash_48_8_t in2out;
+  clib_bihash_48_8_t out2in;
+} nat64_db_st_t;
+
+typedef struct
+{
+  nat64_db_bib_t bib;
+  nat64_db_st_t st;
+} nat64_db_t;
+
+/**
+ * @brief Initialize NAT64 DB.
+ *
+ * @param db NAT64 DB.
+ *
+ * @returns 0 on success, non-zero value otherwise.
+ */
+int nat64_db_init (nat64_db_t * db);
+
+/**
+ * @brief Create new NAT64 BIB entry.
+ *
+ * @param db NAT64 DB.
+ * @param in_addr Inside IPv6 address.
+ * @param out_addr Outside IPv4 address.
+ * @param in_port Inside port number.
+ * @param out_port Outside port number.
+ * @param fib_index FIB index.
+ * @param proto L4 protocol.
+ * @param is_static 1 if static, 0 if dynamic.
+ *
+ * @returns BIB entry on success, 0 otherwise.
+ */
+nat64_db_bib_entry_t *nat64_db_bib_entry_create (nat64_db_t * db,
+						 ip6_address_t * in_addr,
+						 ip4_address_t * out_addr,
+						 u16 in_port, u16 out_port,
+						 u32 fib_index,
+						 u8 proto, u8 is_static);
+
+/**
+ * @brief Free NAT64 BIB entry.
+ *
+ * @param db NAT64 DB.
+ * @param bibe BIB entry.
+ */
+void nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe);
+
+/**
+ * @brief Call back function when walking NAT64 BIB, non-zero
+ * return value stop walk.
+ */
+typedef int (*nat64_db_bib_walk_fn_t) (nat64_db_bib_entry_t * bibe,
+				       void *ctx);
+/**
+ * @brief Walk NAT64 BIB.
+ *
+ * @param db NAT64 DB.
+ * @param proto BIB L4 protocol:
+ *  - 255 all BIBs
+ *  - 6 TCP BIB
+ *  - 17 UDP BIB
+ *  - 1/58 ICMP BIB
+ *  - otherwise "unknown" protocol BIB
+ * @param fn The function to invoke on each entry visited.
+ * @param ctx A context passed in the visit function.
+ */
+void nat64_db_bib_walk (nat64_db_t * db, u8 proto,
+			nat64_db_bib_walk_fn_t fn, void *ctx);
+
+/**
+ * @brief Find NAT64 BIB entry.
+ *
+ * @param db NAT64 DB.
+ * @param addr IP address.
+ * @param port Port number.
+ * @param proto L4 protocol.
+ * @param fib_index FIB index.
+ * @param is_ip6 1 if find by IPv6 (inside) address, 0 by IPv4 (outside).
+ *
+ * @return BIB entry if found.
+ */
+nat64_db_bib_entry_t *nat64_db_bib_entry_find (nat64_db_t * db,
+					       ip46_address_t * addr,
+					       u16 port,
+					       u8 proto,
+					       u32 fib_index, u8 is_ip6);
+
+/**
+ * @brief Get BIB entry by index and protocol.
+ *
+ * @param db NAT64 DB.
+ * @param proto L4 protocol.
+ * @param bibe_index BIB entry index.
+ *
+ * @return BIB entry if found.
+ */
+nat64_db_bib_entry_t *nat64_db_bib_entry_by_index (nat64_db_t * db,
+						   u8 proto, u32 bibe_index);
+/**
+ * @brief Create new NAT64 session table entry.
+ *
+ * @param db NAT64 DB.
+ * @param bibe Corresponding BIB entry.
+ * @param in_r_addr Inside IPv6 address of the remote host.
+ * @param out_r_addr Outside IPv4 address of the remote host.
+ * @param r_port Remote host port number.
+ *
+ * @returns BIB entry on success, 0 otherwise.
+ */
+nat64_db_st_entry_t *nat64_db_st_entry_create (nat64_db_t * db,
+					       nat64_db_bib_entry_t * bibe,
+					       ip6_address_t * in_r_addr,
+					       ip4_address_t * out_r_addr,
+					       u16 r_port);
+
+/**
+ * @brief Free NAT64 session table entry.
+ *
+ * @param db NAT64 DB.
+ * @param ste Session table entry.
+ */
+void nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste);
+
+/**
+ * @brief Find NAT64 session table entry.
+ *
+ * @param db NAT64 DB.
+ * @param l_addr Local host address.
+ * @param r_addr Remote host address.
+ * @param l_port Local host port number.
+ * @param r_port Remote host port number.
+ * @param proto L4 protocol.
+ * @param fib_index FIB index.
+ * @param is_ip6 1 if find by IPv6 (inside) address, 0 by IPv4 (outside).
+ *
+ * @return BIB entry if found.
+ */
+nat64_db_st_entry_t *nat64_db_st_entry_find (nat64_db_t * db,
+					     ip46_address_t * l_addr,
+					     ip46_address_t * r_addr,
+					     u16 l_port, u16 r_port,
+					     u8 proto,
+					     u32 fib_index, u8 is_ip6);
+
+/**
+ * @brief Call back function when walking NAT64 session table, non-zero
+ * return value stop walk.
+ */
+typedef int (*nat64_db_st_walk_fn_t) (nat64_db_st_entry_t * ste, void *ctx);
+
+/**
+ * @brief Walk NAT64 session table.
+ *
+ * @param db NAT64 DB.
+ * @param proto L4 protocol:
+ *  - 255 all session tables
+ *  - 6 TCP session table
+ *  - 17 UDP session table
+ *  - 1/58 ICMP session table
+ *  - otherwise "unknown" protocol session table
+ * @param fn The function to invoke on each entry visited.
+ * @param ctx A context passed in the visit function.
+ */
+void nat64_db_st_walk (nat64_db_t * db, u8 proto,
+		       nat64_db_st_walk_fn_t fn, void *ctx);
+
+/**
+ * @brief Free expired session entries in session tables.
+ *
+ * @param db NAT64 DB.
+ * @param now Current time.
+ */
+void nad64_db_st_free_expired (nat64_db_t * db, u32 now);
+
+/**
+ * @brief Free sessions using specific outside address.
+ *
+ * @param db NAT64 DB.
+ * @param out_addr Outside address to match.
+ */
+void nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr);
+
+#endif /* __included_nat64_db_h__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat64_doc.md b/src/plugins/nat/nat64_doc.md
new file mode 100644
index 00000000..f94467da
--- /dev/null
+++ b/src/plugins/nat/nat64_doc.md
@@ -0,0 +1,73 @@
+# Stateful NAT64: Network Address and Protocol Translation from IPv6 Clients to IPv4 Servers {#nat64_doc}
+
+## Introduction
+
+Stateful NAT64 in VPP allows IPv6-only clients to contact IPv4 servers using unicast UDP, TCP, or ICMP based on RFC 6146.
+
+## Configuration
+
+### Enable/disable NAT64 feature on the interface
+
+> set interface nat64 in|out  [del]
+
+in: inside/local/IPv6 network
+out: outside/external/IPv4 network
+intfc: interface name
+
+### Add/delete NAT64 pool address
+
+One or more public IPv4 addresses assigned to a NAT64 are shared among several IPv6-only clients.
+
+> nat64 add pool address  [- ] [tenant-vrf ] [del]
+
+ip4-range-start: First IPv4 address of the range 
+ip4-range-end: Last IPv4 address of the range (optional, not used for single address)
+tenant-vrf-id: VRF id of the tenant associated with the pool address (optional, if not set pool address is global)
+
+### Add/delete static BIB entry
+
+Stateful NAT64 also supports IPv4-initiated communications to a subset of the IPv6 hosts through staticaly configured bindings.
+
+> nat64 add static bib     tcp|udp|icmp [vfr ] [del]
+
+ip6-addr: inside IPv6 address of the host
+in-port: inside port or ICMPv6 identifier
+ip4-addr: outside IPv4 address of the host
+out-port: outside port or ICMPv4 identifier
+table-id: VRF id of the tenant associated with the BIB entry (optional, default use global VRF)
+
+### Set NAT64 session timeouts
+
+Session is deleted when timer expires. If all sessions corresponding to a dynamically create BIB entry are deleted, then the BIB entry is also deleted. When packets are flowing sessiom timer is refreshed to keep the session alive.
+
+> set nat64 timeouts udp  icmp  tcp-trans  tcp-est  tcp-incoming-syn  | reset
+
+udp: UDP session timeout value (default 300sec)
+icmp: ICMP session timeout value (default 60sec)
+tcp-trans: transitory TCP session timeout value (default 240sec)
+tcp-est: established TCP session timeout value (default 7440sec)
+tcp-incoming-syn: incoming SYN TCP session timeout value (default 6sec)
+reset: reset timers to default values
+
+### Set NAT64 prefix 
+
+Stateful NAT64 support the algorithm for generating IPv6 representations of IPv4 addresses defined in RFC 6052. If no prefix is configured, Well-Known Prefix (64:ff9b::/96) is used. 
+
+> nat64 add prefix / [tenant-vrf ] [del]
+
+ip6-prefix: IPv6 prefix
+plen: prefix length (valid values: 32, 40, 48, 56, 64, or 96)
+tenant-vrf: VRF id of the tenant associated with the prefix
+
+### Show commands
+
+> show nat64 pool
+> show nat64 interfaces
+> show nat64 bib tcp|udp|icmp
+> show nat64 session table tcp|udp|icmp
+> show nat64 tiemouts
+> show nat64 prefix
+
+## Notes
+
+Multi thread is not supported yet (CLI/API commands are disabled when VPP runs with multiple threads).
diff --git a/src/plugins/nat/nat64_in2out.c b/src/plugins/nat/nat64_in2out.c
new file mode 100644
index 00000000..f78baff4
--- /dev/null
+++ b/src/plugins/nat/nat64_in2out.c
@@ -0,0 +1,1118 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 IPv6 to IPv4 translation (inside to outside network)
+ */
+
+#include 
+#include 
+#include 
+
+typedef struct
+{
+  u32 sw_if_index;
+  u32 next_index;
+  u8 is_slow_path;
+} nat64_in2out_trace_t;
+
+static u8 *
+format_nat64_in2out_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  nat64_in2out_trace_t *t = va_arg (*args, nat64_in2out_trace_t *);
+  char *tag;
+
+  tag = t->is_slow_path ? "NAT64-in2out-slowpath" : "NAT64-in2out";
+
+  s =
+    format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
+	    t->next_index);
+
+  return s;
+}
+
+vlib_node_registration_t nat64_in2out_node;
+vlib_node_registration_t nat64_in2out_slowpath_node;
+
+#define foreach_nat64_in2out_error                 \
+_(UNSUPPORTED_PROTOCOL, "unsupported protocol")    \
+_(IN2OUT_PACKETS, "good in2out packets processed") \
+_(NO_TRANSLATION, "no translation")                \
+_(UNKNOWN, "unknown")
+
+typedef enum
+{
+#define _(sym,str) NAT64_IN2OUT_ERROR_##sym,
+  foreach_nat64_in2out_error
+#undef _
+    NAT64_IN2OUT_N_ERROR,
+} nat64_in2out_error_t;
+
+static char *nat64_in2out_error_strings[] = {
+#define _(sym,string) string,
+  foreach_nat64_in2out_error
+#undef _
+};
+
+typedef enum
+{
+  NAT64_IN2OUT_NEXT_IP4_LOOKUP,
+  NAT64_IN2OUT_NEXT_IP6_LOOKUP,
+  NAT64_IN2OUT_NEXT_DROP,
+  NAT64_IN2OUT_NEXT_SLOWPATH,
+  NAT64_IN2OUT_N_NEXT,
+} nat64_in2out_next_t;
+
+typedef struct nat64_in2out_set_ctx_t_
+{
+  vlib_buffer_t *b;
+  vlib_main_t *vm;
+} nat64_in2out_set_ctx_t;
+
+/**
+ * @brief Check whether is a hairpinning.
+ *
+ * If the destination IP address of the packet is an IPv4 address assigned to
+ * the NAT64 itself, then the packet is a hairpin packet.
+ *
+ * param dst_addr Destination address of the packet.
+ *
+ * @returns 1 if hairpinning, otherwise 0.
+ */
+static_always_inline int
+is_hairpinning (ip6_address_t * dst_addr)
+{
+  nat64_main_t *nm = &nat64_main;
+  int i;
+
+  for (i = 0; i < vec_len (nm->addr_pool); i++)
+    {
+      if (nm->addr_pool[i].addr.as_u32 == dst_addr->as_u32[3])
+	return 1;
+    }
+
+  return 0;
+}
+
+static int
+nat64_in2out_tcp_udp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
+			     void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_in2out_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  u32 sw_if_index, fib_index;
+  udp_header_t *udp = ip6_next_header (ip6);
+  u8 proto = ip6->protocol;
+  u16 sport = udp->src_port;
+  u16 dport = udp->dst_port;
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto,
+			    fib_index, 1);
+
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe =
+	nat64_db_bib_entry_find (&nm->db, &saddr, sport, proto, fib_index, 1);
+
+      if (!bibe)
+	{
+	  u16 out_port;
+	  ip4_address_t out_addr;
+	  if (nat64_alloc_out_addr_and_port
+	      (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
+	       &out_port))
+	    return -1;
+
+	  bibe =
+	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address, &out_addr,
+				       sport, clib_host_to_net_u16 (out_port),
+				       fib_index, proto, 0);
+	  if (!bibe)
+	    return -1;
+	}
+
+      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
+				  &daddr.ip4, dport);
+      if (!ste)
+	return -1;
+    }
+
+  nat64_session_reset_timeout (ste, ctx->vm);
+
+  ip4->src_address.as_u32 = bibe->out_addr.as_u32;
+  udp->src_port = bibe->out_port;
+
+  ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
+
+  if (proto == IP_PROTOCOL_TCP)
+    {
+      u16 *checksum;
+      ip_csum_t csum;
+      tcp_header_t *tcp = ip6_next_header (ip6);
+
+      checksum = &tcp->checksum;
+      csum = ip_csum_sub_even (*checksum, sport);
+      csum = ip_csum_add_even (csum, udp->src_port);
+      *checksum = ip_csum_fold (csum);
+    }
+
+  return 0;
+}
+
+static int
+nat64_in2out_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_in2out_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  u32 sw_if_index, fib_index;
+  icmp46_header_t *icmp = ip6_next_header (ip6);
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
+
+  if (icmp->type == ICMP4_echo_request || icmp->type == ICMP4_echo_reply)
+    {
+      u16 in_id = ((u16 *) (icmp))[2];
+      ste =
+	nat64_db_st_entry_find (&nm->db, &saddr, &daddr, in_id, 0,
+				IP_PROTOCOL_ICMP, fib_index, 1);
+
+      if (ste)
+	{
+	  bibe =
+	    nat64_db_bib_entry_by_index (&nm->db, IP_PROTOCOL_ICMP,
+					 ste->bibe_index);
+	  if (!bibe)
+	    return -1;
+	}
+      else
+	{
+	  bibe =
+	    nat64_db_bib_entry_find (&nm->db, &saddr, in_id,
+				     IP_PROTOCOL_ICMP, fib_index, 1);
+
+	  if (!bibe)
+	    {
+	      u16 out_id;
+	      ip4_address_t out_addr;
+	      if (nat64_alloc_out_addr_and_port
+		  (fib_index, SNAT_PROTOCOL_ICMP, &out_addr, &out_id))
+		return -1;
+
+	      bibe =
+		nat64_db_bib_entry_create (&nm->db, &ip6->src_address,
+					   &out_addr, in_id,
+					   clib_host_to_net_u16 (out_id),
+					   fib_index, IP_PROTOCOL_ICMP, 0);
+	      if (!bibe)
+		return -1;
+	    }
+
+	  nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
+	  ste =
+	    nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
+				      &daddr.ip4, 0);
+	  if (!ste)
+	    return -1;
+	}
+
+      nat64_session_reset_timeout (ste, ctx->vm);
+
+      ip4->src_address.as_u32 = bibe->out_addr.as_u32;
+      ((u16 *) (icmp))[2] = bibe->out_port;
+
+      ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
+    }
+  else
+    {
+      if (!vec_len (nm->addr_pool))
+	return -1;
+
+      ip4->src_address.as_u32 = nm->addr_pool[0].addr.as_u32;
+      nat64_extract_ip4 (&ip6->dst_address, &ip4->dst_address, fib_index);
+    }
+
+  return 0;
+}
+
+static int
+nat64_in2out_inner_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
+				void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_in2out_set_ctx_t *ctx = arg;
+  nat64_db_st_entry_t *ste;
+  nat64_db_bib_entry_t *bibe;
+  ip46_address_t saddr, daddr;
+  u32 sw_if_index, fib_index;
+  u8 proto = ip6->protocol;
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
+
+  if (proto == IP_PROTOCOL_ICMP6)
+    {
+      icmp46_header_t *icmp = ip6_next_header (ip6);
+      u16 in_id = ((u16 *) (icmp))[2];
+      proto = IP_PROTOCOL_ICMP;
+
+      if (!
+	  (icmp->type == ICMP4_echo_request
+	   || icmp->type == ICMP4_echo_reply))
+	return -1;
+
+      ste =
+	nat64_db_st_entry_find (&nm->db, &daddr, &saddr, in_id, 0, proto,
+				fib_index, 1);
+      if (!ste)
+	return -1;
+
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+
+      ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
+      ((u16 *) (icmp))[2] = bibe->out_port;
+      ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
+    }
+  else
+    {
+      udp_header_t *udp = ip6_next_header (ip6);
+      tcp_header_t *tcp = ip6_next_header (ip6);
+      u16 *checksum;
+      ip_csum_t csum;
+
+      u16 sport = udp->src_port;
+      u16 dport = udp->dst_port;
+
+      ste =
+	nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto,
+				fib_index, 1);
+      if (!ste)
+	return -1;
+
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+
+      ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
+      udp->dst_port = bibe->out_port;
+      ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
+
+      if (proto == IP_PROTOCOL_TCP)
+	checksum = &tcp->checksum;
+      else
+	checksum = &udp->checksum;
+      csum = ip_csum_sub_even (*checksum, dport);
+      csum = ip_csum_add_even (csum, udp->dst_port);
+      *checksum = ip_csum_fold (csum);
+    }
+
+  return 0;
+}
+
+typedef struct unk_proto_st_walk_ctx_t_
+{
+  ip6_address_t src_addr;
+  ip6_address_t dst_addr;
+  ip4_address_t out_addr;
+  u32 fib_index;
+  u8 proto;
+} unk_proto_st_walk_ctx_t;
+
+static int
+unk_proto_st_walk (nat64_db_st_entry_t * ste, void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  unk_proto_st_walk_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  ip46_address_t saddr, daddr;
+
+  if (ip46_address_is_equal (&ste->in_r_addr, &ctx->dst_addr))
+    {
+      bibe =
+	nat64_db_bib_entry_by_index (&nm->db, ste->proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+
+      if (ip46_address_is_equal (&bibe->in_addr, &ctx->src_addr)
+	  && bibe->fib_index == ctx->fib_index)
+	{
+	  memset (&saddr, 0, sizeof (saddr));
+	  saddr.ip4.as_u32 = bibe->out_addr.as_u32;
+	  memset (&daddr, 0, sizeof (daddr));
+	  nat64_extract_ip4 (&ctx->dst_addr, &daddr.ip4, ctx->fib_index);
+
+	  if (nat64_db_st_entry_find
+	      (&nm->db, &daddr, &saddr, 0, 0, ctx->proto, ctx->fib_index, 0))
+	    return -1;
+
+	  ctx->out_addr.as_u32 = bibe->out_addr.as_u32;
+	  return 1;
+	}
+    }
+
+  return 0;
+}
+
+static int
+nat64_in2out_unk_proto_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
+			       void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_in2out_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr, addr;
+  u32 sw_if_index, fib_index;
+  u8 proto = ip6->protocol;
+  int i;
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, 0, 0, proto, fib_index,
+			    1);
+
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe =
+	nat64_db_bib_entry_find (&nm->db, &saddr, 0, proto, fib_index, 1);
+
+      if (!bibe)
+	{
+	  /* Choose same out address as for TCP/UDP session to same dst */
+	  unk_proto_st_walk_ctx_t ctx = {
+	    .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
+	    .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
+	    .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
+	    .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
+	    .out_addr.as_u32 = 0,
+	    .fib_index = fib_index,
+	    .proto = proto,
+	  };
+
+	  nat64_db_st_walk (&nm->db, IP_PROTOCOL_TCP, unk_proto_st_walk,
+			    &ctx);
+
+	  if (!ctx.out_addr.as_u32)
+	    nat64_db_st_walk (&nm->db, IP_PROTOCOL_UDP, unk_proto_st_walk,
+			      &ctx);
+
+	  /* Verify if out address is not already in use for protocol */
+	  memset (&addr, 0, sizeof (addr));
+	  addr.ip4.as_u32 = ctx.out_addr.as_u32;
+	  if (nat64_db_bib_entry_find (&nm->db, &addr, 0, proto, 0, 0))
+	    ctx.out_addr.as_u32 = 0;
+
+	  if (!ctx.out_addr.as_u32)
+	    {
+	      for (i = 0; i < vec_len (nm->addr_pool); i++)
+		{
+		  addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
+		  if (!nat64_db_bib_entry_find
+		      (&nm->db, &addr, 0, proto, 0, 0))
+		    break;
+		}
+	    }
+
+	  if (!ctx.out_addr.as_u32)
+	    return -1;
+
+	  bibe =
+	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address,
+				       &ctx.out_addr, 0, 0, fib_index, proto,
+				       0);
+	  if (!bibe)
+	    return -1;
+	}
+
+      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
+				  &daddr.ip4, 0);
+      if (!ste)
+	return -1;
+    }
+
+  nat64_session_reset_timeout (ste, ctx->vm);
+
+  ip4->src_address.as_u32 = bibe->out_addr.as_u32;
+  ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
+
+  return 0;
+}
+
+
+
+static int
+nat64_in2out_tcp_udp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
+				  ip6_header_t * ip6)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  u32 sw_if_index, fib_index;
+  udp_header_t *udp = ip6_next_header (ip6);
+  tcp_header_t *tcp = ip6_next_header (ip6);
+  u8 proto = ip6->protocol;
+  u16 sport = udp->src_port;
+  u16 dport = udp->dst_port;
+  u16 *checksum;
+  ip_csum_t csum;
+
+  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
+
+  if (proto == IP_PROTOCOL_UDP)
+    checksum = &udp->checksum;
+  else
+    checksum = &tcp->checksum;
+
+  csum = ip_csum_sub_even (*checksum, ip6->src_address.as_u64[0]);
+  csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
+  csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
+  csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
+  csum = ip_csum_sub_even (csum, sport);
+  csum = ip_csum_sub_even (csum, dport);
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto,
+			    fib_index, 1);
+
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe =
+	nat64_db_bib_entry_find (&nm->db, &saddr, sport, proto, fib_index, 1);
+
+      if (!bibe)
+	{
+	  u16 out_port;
+	  ip4_address_t out_addr;
+	  if (nat64_alloc_out_addr_and_port
+	      (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
+	       &out_port))
+	    return -1;
+
+	  bibe =
+	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address, &out_addr,
+				       sport, clib_host_to_net_u16 (out_port),
+				       fib_index, proto, 0);
+	  if (!bibe)
+	    return -1;
+	}
+
+      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
+				  &daddr.ip4, dport);
+      if (!ste)
+	return -1;
+    }
+
+  nat64_session_reset_timeout (ste, vm);
+
+  sport = udp->src_port = bibe->out_port;
+  nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  memset (&daddr, 0, sizeof (daddr));
+  saddr.ip4.as_u32 = bibe->out_addr.as_u32;
+  daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto, 0,
+			    0);
+
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe = nat64_db_bib_entry_find (&nm->db, &daddr, dport, proto, 0, 0);
+
+      if (!bibe)
+	return -1;
+
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6->src_address,
+				  &saddr.ip4, sport);
+    }
+
+  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
+  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
+  udp->dst_port = bibe->in_port;
+
+  csum = ip_csum_add_even (csum, ip6->src_address.as_u64[0]);
+  csum = ip_csum_add_even (csum, ip6->src_address.as_u64[1]);
+  csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[0]);
+  csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[1]);
+  csum = ip_csum_add_even (csum, udp->src_port);
+  csum = ip_csum_add_even (csum, udp->dst_port);
+  *checksum = ip_csum_fold (csum);
+
+  return 0;
+}
+
+static int
+nat64_in2out_icmp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
+			       ip6_header_t * ip6)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  icmp46_header_t *icmp = ip6_next_header (ip6);
+  ip6_header_t *inner_ip6;
+  ip46_address_t saddr, daddr;
+  u32 sw_if_index, fib_index;
+  u8 proto;
+  udp_header_t *udp;
+  tcp_header_t *tcp;
+  u16 *checksum, sport, dport;
+  ip_csum_t csum;
+
+  if (icmp->type == ICMP6_echo_request || icmp->type == ICMP6_echo_reply)
+    return -1;
+
+  inner_ip6 = (ip6_header_t *) u8_ptr_add (icmp, 8);
+
+  proto = inner_ip6->protocol;
+
+  if (proto == IP_PROTOCOL_ICMP6)
+    return -1;
+
+  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = inner_ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = inner_ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = inner_ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = inner_ip6->dst_address.as_u64[1];
+
+  udp = ip6_next_header (inner_ip6);
+  tcp = ip6_next_header (inner_ip6);
+
+  sport = udp->src_port;
+  dport = udp->dst_port;
+
+  if (proto == IP_PROTOCOL_UDP)
+    checksum = &udp->checksum;
+  else
+    checksum = &tcp->checksum;
+
+  csum = ip_csum_sub_even (*checksum, inner_ip6->src_address.as_u64[0]);
+  csum = ip_csum_sub_even (csum, inner_ip6->src_address.as_u64[1]);
+  csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[0]);
+  csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[1]);
+  csum = ip_csum_sub_even (csum, sport);
+  csum = ip_csum_sub_even (csum, dport);
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto,
+			    fib_index, 1);
+  if (!ste)
+    return -1;
+
+  bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+  if (!bibe)
+    return -1;
+
+  dport = udp->dst_port = bibe->out_port;
+  nat64_compose_ip6 (&inner_ip6->dst_address, &bibe->out_addr, fib_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  memset (&daddr, 0, sizeof (daddr));
+  saddr.ip4.as_u32 = ste->out_r_addr.as_u32;
+  daddr.ip4.as_u32 = bibe->out_addr.as_u32;
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto, 0,
+			    0);
+  if (!ste)
+    return -1;
+
+  bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+  if (!bibe)
+    return -1;
+
+  inner_ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
+  inner_ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
+  udp->src_port = bibe->in_port;
+
+  csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[0]);
+  csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[1]);
+  csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[0]);
+  csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[1]);
+  csum = ip_csum_add_even (csum, udp->src_port);
+  csum = ip_csum_add_even (csum, udp->dst_port);
+  *checksum = ip_csum_fold (csum);
+
+  if (!vec_len (nm->addr_pool))
+    return -1;
+
+  nat64_compose_ip6 (&ip6->src_address, &nm->addr_pool[0].addr, fib_index);
+  ip6->dst_address.as_u64[0] = inner_ip6->src_address.as_u64[0];
+  ip6->dst_address.as_u64[1] = inner_ip6->src_address.as_u64[1];
+
+  icmp->checksum = 0;
+  csum = ip_csum_with_carry (0, ip6->payload_length);
+  csum = ip_csum_with_carry (csum, clib_host_to_net_u16 (ip6->protocol));
+  csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[0]);
+  csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[1]);
+  csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[0]);
+  csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[1]);
+  csum =
+    ip_incremental_checksum (csum, icmp,
+			     clib_net_to_host_u16 (ip6->payload_length));
+  icmp->checksum = ~ip_csum_fold (csum);
+
+  return 0;
+}
+
+static int
+nat64_in2out_unk_proto_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
+				    ip6_header_t * ip6)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr, addr;
+  u32 sw_if_index, fib_index;
+  u8 proto = ip6->protocol;
+  int i;
+
+  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  saddr.as_u64[0] = ip6->src_address.as_u64[0];
+  saddr.as_u64[1] = ip6->src_address.as_u64[1];
+  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
+  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, 0, 0, proto, fib_index,
+			    1);
+
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe =
+	nat64_db_bib_entry_find (&nm->db, &saddr, 0, proto, fib_index, 1);
+
+      if (!bibe)
+	{
+	  /* Choose same out address as for TCP/UDP session to same dst */
+	  unk_proto_st_walk_ctx_t ctx = {
+	    .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
+	    .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
+	    .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
+	    .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
+	    .out_addr.as_u32 = 0,
+	    .fib_index = fib_index,
+	    .proto = proto,
+	  };
+
+	  nat64_db_st_walk (&nm->db, IP_PROTOCOL_TCP, unk_proto_st_walk,
+			    &ctx);
+
+	  if (!ctx.out_addr.as_u32)
+	    nat64_db_st_walk (&nm->db, IP_PROTOCOL_UDP, unk_proto_st_walk,
+			      &ctx);
+
+	  /* Verify if out address is not already in use for protocol */
+	  memset (&addr, 0, sizeof (addr));
+	  addr.ip4.as_u32 = ctx.out_addr.as_u32;
+	  if (nat64_db_bib_entry_find (&nm->db, &addr, 0, proto, 0, 0))
+	    ctx.out_addr.as_u32 = 0;
+
+	  if (!ctx.out_addr.as_u32)
+	    {
+	      for (i = 0; i < vec_len (nm->addr_pool); i++)
+		{
+		  addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
+		  if (!nat64_db_bib_entry_find
+		      (&nm->db, &addr, 0, proto, 0, 0))
+		    break;
+		}
+	    }
+
+	  if (!ctx.out_addr.as_u32)
+	    return -1;
+
+	  bibe =
+	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address,
+				       &ctx.out_addr, 0, 0, fib_index, proto,
+				       0);
+	  if (!bibe)
+	    return -1;
+	}
+
+      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
+				  &daddr.ip4, 0);
+      if (!ste)
+	return -1;
+    }
+
+  nat64_session_reset_timeout (ste, vm);
+
+  nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  memset (&daddr, 0, sizeof (daddr));
+  saddr.ip4.as_u32 = bibe->out_addr.as_u32;
+  daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
+
+  ste = nat64_db_st_entry_find (&nm->db, &daddr, &saddr, 0, 0, proto, 0, 0);
+
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe = nat64_db_bib_entry_find (&nm->db, &daddr, 0, proto, 0, 0);
+
+      if (!bibe)
+	return -1;
+
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6->src_address,
+				  &saddr.ip4, 0);
+    }
+
+  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
+  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
+
+  return 0;
+}
+
+static inline uword
+nat64_in2out_node_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+			     vlib_frame_t * frame, u8 is_slow_path)
+{
+  u32 n_left_from, *from, *to_next;
+  nat64_in2out_next_t next_index;
+  u32 pkts_processed = 0;
+  u32 stats_node_index;
+
+  stats_node_index =
+    is_slow_path ? nat64_in2out_slowpath_node.index : nat64_in2out_node.index;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  u32 next0;
+	  ip6_header_t *ip60;
+	  u16 l4_offset0, frag_offset0;
+	  u8 l4_protocol0;
+	  u32 proto0;
+	  nat64_in2out_set_ctx_t ctx0;
+
+	  /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  ip60 = vlib_buffer_get_current (b0);
+
+	  ctx0.b = b0;
+	  ctx0.vm = vm;
+
+	  next0 = NAT64_IN2OUT_NEXT_IP4_LOOKUP;
+
+	  if (PREDICT_FALSE
+	      (ip6_parse
+	       (ip60, b0->current_length, &l4_protocol0, &l4_offset0,
+		&frag_offset0)))
+	    {
+	      next0 = NAT64_IN2OUT_NEXT_DROP;
+	      b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
+	      goto trace0;
+	    }
+
+	  proto0 = ip_proto_to_snat_proto (l4_protocol0);
+	  if (frag_offset0 != 0)
+	    {
+	      next0 = NAT64_IN2OUT_NEXT_DROP;
+	      b0->error =
+		node->errors[NAT64_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
+	      goto trace0;
+	    }
+
+	  if (is_slow_path)
+	    {
+	      if (PREDICT_TRUE (proto0 == ~0))
+		{
+		  if (is_hairpinning (&ip60->dst_address))
+		    {
+		      next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
+		      if (nat64_in2out_unk_proto_hairpinning (vm, b0, ip60))
+			{
+			  next0 = NAT64_IN2OUT_NEXT_DROP;
+			  b0->error =
+			    node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
+			}
+		      goto trace0;
+		    }
+
+		  if (ip6_to_ip4 (b0, nat64_in2out_unk_proto_set_cb, &ctx0))
+		    {
+		      next0 = NAT64_IN2OUT_NEXT_DROP;
+		      b0->error =
+			node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
+		      goto trace0;
+		    }
+		}
+	      goto trace0;
+	    }
+	  else
+	    {
+	      if (PREDICT_FALSE (proto0 == ~0))
+		{
+		  next0 = NAT64_IN2OUT_NEXT_SLOWPATH;
+		  goto trace0;
+		}
+	    }
+
+	  if (proto0 == SNAT_PROTOCOL_ICMP)
+	    {
+	      if (is_hairpinning (&ip60->dst_address))
+		{
+		  next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
+		  if (nat64_in2out_icmp_hairpinning (vm, b0, ip60))
+		    {
+		      next0 = NAT64_IN2OUT_NEXT_DROP;
+		      b0->error =
+			node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
+		    }
+		  goto trace0;
+		}
+
+	      if (icmp6_to_icmp
+		  (b0, nat64_in2out_icmp_set_cb, &ctx0,
+		   nat64_in2out_inner_icmp_set_cb, &ctx0))
+		{
+		  next0 = NAT64_IN2OUT_NEXT_DROP;
+		  b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
+		  goto trace0;
+		}
+	    }
+	  else if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
+	    {
+	      if (is_hairpinning (&ip60->dst_address))
+		{
+		  next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
+		  if (nat64_in2out_tcp_udp_hairpinning (vm, b0, ip60))
+		    {
+		      next0 = NAT64_IN2OUT_NEXT_DROP;
+		      b0->error =
+			node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
+		    }
+		  goto trace0;
+		}
+
+	      if (ip6_to_ip4_tcp_udp
+		  (b0, nat64_in2out_tcp_udp_set_cb, &ctx0, 0))
+		{
+		  next0 = NAT64_IN2OUT_NEXT_DROP;
+		  b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
+		  goto trace0;
+		}
+	    }
+
+	trace0:
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	    {
+	      nat64_in2out_trace_t *t =
+		vlib_add_trace (vm, node, b0, sizeof (*t));
+	      t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
+	      t->next_index = next0;
+	      t->is_slow_path = is_slow_path;
+	    }
+
+	  pkts_processed += next0 != NAT64_IN2OUT_NEXT_DROP;
+
+	  /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  vlib_node_increment_counter (vm, stats_node_index,
+			       NAT64_IN2OUT_ERROR_IN2OUT_PACKETS,
+			       pkts_processed);
+  return frame->n_vectors;
+}
+
+static uword
+nat64_in2out_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		      vlib_frame_t * frame)
+{
+  return nat64_in2out_node_fn_inline (vm, node, frame, 0);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat64_in2out_node) = {
+  .function = nat64_in2out_node_fn,
+  .name = "nat64-in2out",
+  .vector_size = sizeof (u32),
+  .format_trace = format_nat64_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
+  .error_strings = nat64_in2out_error_strings,
+  .n_next_nodes = NAT64_IN2OUT_N_NEXT,
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
+    [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
+    [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
+    [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
+  },
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_node, nat64_in2out_node_fn);
+
+static uword
+nat64_in2out_slowpath_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+			       vlib_frame_t * frame)
+{
+  return nat64_in2out_node_fn_inline (vm, node, frame, 1);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat64_in2out_slowpath_node) = {
+  .function = nat64_in2out_slowpath_node_fn,
+  .name = "nat64-in2out-slowpath",
+  .vector_size = sizeof (u32),
+  .format_trace = format_nat64_in2out_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
+  .error_strings = nat64_in2out_error_strings,
+  .n_next_nodes = NAT64_IN2OUT_N_NEXT,
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
+    [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
+    [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
+    [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
+  },
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_slowpath_node,
+			      nat64_in2out_slowpath_node_fn);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat64_out2in.c b/src/plugins/nat/nat64_out2in.c
new file mode 100644
index 00000000..61e88a7f
--- /dev/null
+++ b/src/plugins/nat/nat64_out2in.c
@@ -0,0 +1,494 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT64 IPv4 to IPv6 translation (otside to inside network)
+ */
+
+#include 
+#include 
+#include 
+
+typedef struct
+{
+  u32 sw_if_index;
+  u32 next_index;
+} nat64_out2in_trace_t;
+
+static u8 *
+format_nat64_out2in_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  nat64_out2in_trace_t *t = va_arg (*args, nat64_out2in_trace_t *);
+
+  s =
+    format (s, "NAT64-out2in: sw_if_index %d, next index %d", t->sw_if_index,
+	    t->next_index);
+
+  return s;
+}
+
+vlib_node_registration_t nat64_out2in_node;
+
+#define foreach_nat64_out2in_error                 \
+_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")    \
+_(OUT2IN_PACKETS, "Good out2in packets processed") \
+_(NO_TRANSLATION, "No translation")                \
+_(UNKNOWN, "unknown")
+
+typedef enum
+{
+#define _(sym,str) NAT64_OUT2IN_ERROR_##sym,
+  foreach_nat64_out2in_error
+#undef _
+    NAT64_OUT2IN_N_ERROR,
+} nat64_out2in_error_t;
+
+static char *nat64_out2in_error_strings[] = {
+#define _(sym,string) string,
+  foreach_nat64_out2in_error
+#undef _
+};
+
+typedef enum
+{
+  NAT64_OUT2IN_NEXT_LOOKUP,
+  NAT64_OUT2IN_NEXT_DROP,
+  NAT64_OUT2IN_N_NEXT,
+} nat64_out2in_next_t;
+
+typedef struct nat64_out2in_set_ctx_t_
+{
+  vlib_buffer_t *b;
+  vlib_main_t *vm;
+} nat64_out2in_set_ctx_t;
+
+static int
+nat64_out2in_tcp_udp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
+			     void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_out2in_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  ip6_address_t ip6_saddr;
+  udp_header_t *udp = ip4_next_header (ip4);
+  tcp_header_t *tcp = ip4_next_header (ip4);
+  u8 proto = ip4->protocol;
+  u16 dport = udp->dst_port;
+  u16 sport = udp->src_port;
+  u32 sw_if_index, fib_index;
+  u16 *checksum;
+  ip_csum_t csum;
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  saddr.ip4.as_u32 = ip4->src_address.as_u32;
+  memset (&daddr, 0, sizeof (daddr));
+  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto,
+			    fib_index, 0);
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe =
+	nat64_db_bib_entry_find (&nm->db, &daddr, dport, proto, fib_index, 0);
+
+      if (!bibe)
+	return -1;
+
+      nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6_saddr, &saddr.ip4,
+				  sport);
+    }
+
+  nat64_session_reset_timeout (ste, ctx->vm);
+
+  ip6->src_address.as_u64[0] = ste->in_r_addr.as_u64[0];
+  ip6->src_address.as_u64[1] = ste->in_r_addr.as_u64[1];
+
+  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
+  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
+  udp->dst_port = bibe->in_port;
+
+  if (proto == IP_PROTOCOL_UDP)
+    checksum = &udp->checksum;
+  else
+    checksum = &tcp->checksum;
+  csum = ip_csum_sub_even (*checksum, dport);
+  csum = ip_csum_add_even (csum, udp->dst_port);
+  *checksum = ip_csum_fold (csum);
+
+  vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
+
+  return 0;
+}
+
+static int
+nat64_out2in_icmp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6, void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_out2in_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  ip6_address_t ip6_saddr;
+  u32 sw_if_index, fib_index;
+  icmp46_header_t *icmp = ip4_next_header (ip4);
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  saddr.ip4.as_u32 = ip4->src_address.as_u32;
+  memset (&daddr, 0, sizeof (daddr));
+  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
+
+  if (icmp->type == ICMP6_echo_request || icmp->type == ICMP6_echo_reply)
+    {
+      u16 out_id = ((u16 *) (icmp))[2];
+      ste =
+	nat64_db_st_entry_find (&nm->db, &daddr, &saddr, out_id, 0,
+				IP_PROTOCOL_ICMP, fib_index, 0);
+
+      if (ste)
+	{
+	  bibe =
+	    nat64_db_bib_entry_by_index (&nm->db, IP_PROTOCOL_ICMP,
+					 ste->bibe_index);
+	  if (!bibe)
+	    return -1;
+	}
+      else
+	{
+	  bibe =
+	    nat64_db_bib_entry_find (&nm->db, &daddr, out_id,
+				     IP_PROTOCOL_ICMP, fib_index, 0);
+	  if (!bibe)
+	    return -1;
+
+	  nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
+	  ste =
+	    nat64_db_st_entry_create (&nm->db, bibe, &ip6_saddr, &saddr.ip4,
+				      0);
+	}
+
+      nat64_session_reset_timeout (ste, ctx->vm);
+
+      ip6->src_address.as_u64[0] = ste->in_r_addr.as_u64[0];
+      ip6->src_address.as_u64[1] = ste->in_r_addr.as_u64[1];
+
+      ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
+      ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
+      ((u16 *) (icmp))[2] = bibe->in_port;
+
+      vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
+    }
+  else
+    {
+      ip6_header_t *inner_ip6 = (ip6_header_t *) u8_ptr_add (icmp, 8);
+
+      nat64_compose_ip6 (&ip6->src_address, &ip4->src_address,
+			 vnet_buffer (ctx->b)->sw_if_index[VLIB_TX]);
+      ip6->dst_address.as_u64[0] = inner_ip6->src_address.as_u64[0];
+      ip6->dst_address.as_u64[1] = inner_ip6->src_address.as_u64[1];
+    }
+
+  return 0;
+}
+
+static int
+nat64_out2in_inner_icmp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
+				void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_out2in_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  u32 sw_if_index, fib_index;
+  u8 proto = ip4->protocol;
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index =
+    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  saddr.ip4.as_u32 = ip4->src_address.as_u32;
+  memset (&daddr, 0, sizeof (daddr));
+  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
+
+  if (proto == IP_PROTOCOL_ICMP6)
+    {
+      icmp46_header_t *icmp = ip4_next_header (ip4);
+      u16 out_id = ((u16 *) (icmp))[2];
+      proto = IP_PROTOCOL_ICMP;
+
+      if (!
+	  (icmp->type == ICMP6_echo_request
+	   || icmp->type == ICMP6_echo_reply))
+	return -1;
+
+      ste =
+	nat64_db_st_entry_find (&nm->db, &saddr, &daddr, out_id, 0, proto,
+				fib_index, 0);
+      if (!ste)
+	return -1;
+
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+
+      ip6->dst_address.as_u64[0] = ste->in_r_addr.as_u64[0];
+      ip6->dst_address.as_u64[1] = ste->in_r_addr.as_u64[1];
+      ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
+      ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
+      ((u16 *) (icmp))[2] = bibe->in_port;
+
+      vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
+    }
+  else
+    {
+      udp_header_t *udp = ip4_next_header (ip4);
+      tcp_header_t *tcp = ip4_next_header (ip4);
+      u16 dport = udp->dst_port;
+      u16 sport = udp->src_port;
+      u16 *checksum;
+      ip_csum_t csum;
+
+      ste =
+	nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto,
+				fib_index, 0);
+      if (!ste)
+	return -1;
+
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+
+      nat64_compose_ip6 (&ip6->dst_address, &daddr.ip4, bibe->fib_index);
+      ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
+      ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
+      udp->src_port = bibe->in_port;
+
+      if (proto == IP_PROTOCOL_UDP)
+	checksum = &udp->checksum;
+      else
+	checksum = &tcp->checksum;
+      if (*checksum)
+	{
+	  csum = ip_csum_sub_even (*checksum, sport);
+	  csum = ip_csum_add_even (csum, udp->src_port);
+	  *checksum = ip_csum_fold (csum);
+	}
+
+      vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
+    }
+
+  return 0;
+}
+
+static int
+nat64_out2in_unk_proto_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
+			       void *arg)
+{
+  nat64_main_t *nm = &nat64_main;
+  nat64_out2in_set_ctx_t *ctx = arg;
+  nat64_db_bib_entry_t *bibe;
+  nat64_db_st_entry_t *ste;
+  ip46_address_t saddr, daddr;
+  ip6_address_t ip6_saddr;
+  u32 sw_if_index, fib_index;
+  u8 proto = ip4->protocol;
+
+  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
+  fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
+
+  memset (&saddr, 0, sizeof (saddr));
+  saddr.ip4.as_u32 = ip4->src_address.as_u32;
+  memset (&daddr, 0, sizeof (daddr));
+  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
+
+  ste =
+    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, 0, 0, proto, fib_index,
+			    0);
+  if (ste)
+    {
+      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
+      if (!bibe)
+	return -1;
+    }
+  else
+    {
+      bibe =
+	nat64_db_bib_entry_find (&nm->db, &daddr, 0, proto, fib_index, 0);
+
+      if (!bibe)
+	return -1;
+
+      nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
+      ste =
+	nat64_db_st_entry_create (&nm->db, bibe, &ip6_saddr, &saddr.ip4, 0);
+    }
+
+  nat64_session_reset_timeout (ste, ctx->vm);
+
+  ip6->src_address.as_u64[0] = ste->in_r_addr.as_u64[0];
+  ip6->src_address.as_u64[1] = ste->in_r_addr.as_u64[1];
+
+  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
+  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
+
+  vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
+
+  return 0;
+}
+
+static uword
+nat64_out2in_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+		      vlib_frame_t * frame)
+{
+  u32 n_left_from, *from, *to_next;
+  nat64_out2in_next_t next_index;
+  u32 pkts_processed = 0;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+	  u32 next0;
+	  ip4_header_t *ip40;
+	  u32 proto0;
+	  nat64_out2in_set_ctx_t ctx0;
+
+	  /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  ip40 = vlib_buffer_get_current (b0);
+
+	  ctx0.b = b0;
+	  ctx0.vm = vm;
+
+	  next0 = NAT64_OUT2IN_NEXT_LOOKUP;
+
+	  proto0 = ip_proto_to_snat_proto (ip40->protocol);
+
+	  if (proto0 == SNAT_PROTOCOL_ICMP)
+	    {
+	      if (icmp_to_icmp6
+		  (b0, nat64_out2in_icmp_set_cb, &ctx0,
+		   nat64_out2in_inner_icmp_set_cb, &ctx0))
+		{
+		  next0 = NAT64_OUT2IN_NEXT_DROP;
+		  b0->error = node->errors[NAT64_OUT2IN_ERROR_NO_TRANSLATION];
+		  goto trace0;
+		}
+	    }
+	  else if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
+	    {
+	      if (ip4_to_ip6_tcp_udp (b0, nat64_out2in_tcp_udp_set_cb, &ctx0))
+		{
+		  next0 = NAT64_OUT2IN_NEXT_DROP;
+		  b0->error = node->errors[NAT64_OUT2IN_ERROR_NO_TRANSLATION];
+		  goto trace0;
+		}
+	    }
+	  else
+	    {
+	      if (ip4_to_ip6 (b0, nat64_out2in_unk_proto_set_cb, &ctx0))
+		{
+		  next0 = NAT64_OUT2IN_NEXT_DROP;
+		  b0->error = node->errors[NAT64_OUT2IN_ERROR_NO_TRANSLATION];
+		  goto trace0;
+		}
+	    }
+
+	trace0:
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	    {
+	      nat64_out2in_trace_t *t =
+		vlib_add_trace (vm, node, b0, sizeof (*t));
+	      t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
+	      t->next_index = next0;
+	    }
+
+	  pkts_processed += next0 != NAT64_OUT2IN_NEXT_DROP;
+
+	  /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+					   n_left_to_next, bi0, next0);
+	}
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+  vlib_node_increment_counter (vm, nat64_out2in_node.index,
+			       NAT64_OUT2IN_ERROR_OUT2IN_PACKETS,
+			       pkts_processed);
+  return frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat64_out2in_node) = {
+  .function = nat64_out2in_node_fn,
+  .name = "nat64-out2in",
+  .vector_size = sizeof (u32),
+  .format_trace = format_nat64_out2in_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = ARRAY_LEN (nat64_out2in_error_strings),
+  .error_strings = nat64_out2in_error_strings,.n_next_nodes = 2,
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [NAT64_OUT2IN_NEXT_DROP] = "error-drop",
+    [NAT64_OUT2IN_NEXT_LOOKUP] = "ip6-lookup",
+  },
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (nat64_out2in_node, nat64_out2in_node_fn);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat_all_api_h.h b/src/plugins/nat/nat_all_api_h.h
new file mode 100644
index 00000000..acd9ba1c
--- /dev/null
+++ b/src/plugins/nat/nat_all_api_h.h
@@ -0,0 +1,19 @@
+
+/*
+ * nat_all_api_h.h - skeleton vpp engine plug-in api #include file
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include 
diff --git a/src/plugins/nat/nat_api.c b/src/plugins/nat/nat_api.c
new file mode 100644
index 00000000..0a2141f2
--- /dev/null
+++ b/src/plugins/nat/nat_api.c
@@ -0,0 +1,3255 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief NAT plugin API implementation
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include 
+#undef vl_endianfun
+
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+
+#define REPLY_MSG_ID_BASE sm->msg_id_base
+#include 
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+/* Macro to finish up custom dump fns */
+#define FINISH                                  \
+    vec_add1 (s, 0);                            \
+    vl_print (handle, (char *)s);               \
+    vec_free (s);                               \
+    return handle;
+
+static void
+  vl_api_snat_add_address_range_t_handler
+  (vl_api_snat_add_address_range_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_add_address_range_reply_t *rmp;
+  ip4_address_t this_addr;
+  u32 start_host_order, end_host_order;
+  u32 vrf_id;
+  int i, count;
+  int rv = 0;
+  u32 *tmp;
+
+  if (mp->is_ip4 != 1)
+    {
+      rv = VNET_API_ERROR_UNIMPLEMENTED;
+      goto send_reply;
+    }
+
+  if (sm->static_mapping_only)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  tmp = (u32 *) mp->first_ip_address;
+  start_host_order = clib_host_to_net_u32 (tmp[0]);
+  tmp = (u32 *) mp->last_ip_address;
+  end_host_order = clib_host_to_net_u32 (tmp[0]);
+
+  count = (end_host_order - start_host_order) + 1;
+
+  vrf_id = clib_host_to_net_u32 (mp->vrf_id);
+
+  if (count > 1024)
+    clib_warning ("%U - %U, %d addresses...",
+		  format_ip4_address, mp->first_ip_address,
+		  format_ip4_address, mp->last_ip_address, count);
+
+  memcpy (&this_addr.as_u8, mp->first_ip_address, 4);
+
+  for (i = 0; i < count; i++)
+    {
+      if (mp->is_add)
+	snat_add_address (sm, &this_addr, vrf_id);
+      else
+	rv = snat_del_address (sm, this_addr, 0);
+
+      if (rv)
+	goto send_reply;
+
+      increment_v4_address (&this_addr);
+    }
+
+send_reply:
+  REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY);
+}
+
+static void *vl_api_snat_add_address_range_t_print
+  (vl_api_snat_add_address_range_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_add_address_range ");
+  s = format (s, "%U ", format_ip4_address, mp->first_ip_address);
+  if (memcmp (mp->first_ip_address, mp->last_ip_address, 4))
+    {
+      s = format (s, " - %U ", format_ip4_address, mp->last_ip_address);
+    }
+  FINISH;
+}
+
+static void
+  send_snat_address_details
+  (snat_address_t * a, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_address_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_SNAT_ADDRESS_DETAILS + sm->msg_id_base);
+  rmp->is_ip4 = 1;
+  clib_memcpy (rmp->ip_address, &(a->addr), 4);
+  if (a->fib_index != ~0)
+    {
+      fib_table_t *fib = fib_table_get (a->fib_index, FIB_PROTOCOL_IP4);
+      rmp->vrf_id = ntohl (fib->ft_table_id);
+    }
+  else
+    rmp->vrf_id = ~0;
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_snat_address_dump_t_handler (vl_api_snat_address_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_address_t *a;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (a, sm->addresses)
+    send_snat_address_details (a, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_address_dump_t_print
+  (vl_api_snat_address_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_address_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_interface_add_del_feature_t_handler
+  (vl_api_snat_interface_add_del_feature_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_interface_add_del_feature_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_interface_add_del (sw_if_index, mp->is_inside, is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY);
+}
+
+static void *vl_api_snat_interface_add_del_feature_t_print
+  (vl_api_snat_interface_add_del_feature_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_add_del_feature ");
+  s = format (s, "sw_if_index %d %s %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+  send_snat_interface_details
+  (snat_interface_t * i, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_interface_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_SNAT_INTERFACE_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (i->sw_if_index);
+  rmp->is_inside = i->is_inside;
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_snat_interface_dump_t_handler (vl_api_snat_interface_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (i, sm->interfaces,
+  ({
+    send_snat_interface_details(i, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_interface_dump_t_print
+  (vl_api_snat_interface_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_interface_add_del_output_feature_t_handler
+  (vl_api_snat_interface_add_del_output_feature_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_interface_add_del_output_feature_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_interface_add_del_output_feature (sw_if_index, mp->is_inside,
+					      is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SNAT_INTERFACE_ADD_DEL_OUTPUT_FEATURE_REPLY);
+}
+
+static void *vl_api_snat_interface_add_del_output_feature_t_print
+  (vl_api_snat_interface_add_del_output_feature_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_add_del_output_feature ");
+  s = format (s, "sw_if_index %d %s %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+send_snat_interface_output_feature_details (snat_interface_t * i,
+					    unix_shared_memory_queue_t * q,
+					    u32 context)
+{
+  vl_api_snat_interface_output_feature_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_SNAT_INTERFACE_OUTPUT_FEATURE_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (i->sw_if_index);
+  rmp->context = context;
+  rmp->is_inside = i->is_inside;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  vl_api_snat_interface_output_feature_dump_t_handler
+  (vl_api_snat_interface_output_feature_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    send_snat_interface_output_feature_details(i, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_interface_output_feature_dump_t_print
+  (vl_api_snat_interface_output_feature_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_output_feature_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_add_static_mapping_t_handler
+  (vl_api_snat_add_static_mapping_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_add_static_mapping_reply_t *rmp;
+  ip4_address_t local_addr, external_addr;
+  u16 local_port = 0, external_port = 0;
+  u32 vrf_id, external_sw_if_index;
+  int rv = 0;
+  snat_protocol_t proto;
+
+  if (mp->is_ip4 != 1)
+    {
+      rv = VNET_API_ERROR_UNIMPLEMENTED;
+      goto send_reply;
+    }
+
+  memcpy (&local_addr.as_u8, mp->local_ip_address, 4);
+  memcpy (&external_addr.as_u8, mp->external_ip_address, 4);
+  if (mp->addr_only == 0)
+    {
+      local_port = clib_net_to_host_u16 (mp->local_port);
+      external_port = clib_net_to_host_u16 (mp->external_port);
+    }
+  vrf_id = clib_net_to_host_u32 (mp->vrf_id);
+  external_sw_if_index = clib_net_to_host_u32 (mp->external_sw_if_index);
+  proto = ip_proto_to_snat_proto (mp->protocol);
+
+  rv = snat_add_static_mapping (local_addr, external_addr, local_port,
+				external_port, vrf_id, mp->addr_only,
+				external_sw_if_index, proto, mp->is_add);
+
+send_reply:
+  REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY);
+}
+
+static void *vl_api_snat_add_static_mapping_t_print
+  (vl_api_snat_add_static_mapping_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_add_static_mapping ");
+  s = format (s, "protocol %d local_addr %U external_addr %U ",
+	      mp->protocol,
+	      format_ip4_address, mp->local_ip_address,
+	      format_ip4_address, mp->external_ip_address);
+
+  if (mp->addr_only == 0)
+    s = format (s, "local_port %d external_port %d ",
+		clib_net_to_host_u16 (mp->local_port),
+		clib_net_to_host_u16 (mp->external_port));
+
+  if (mp->vrf_id != ~0)
+    s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id));
+
+  if (mp->external_sw_if_index != ~0)
+    s = format (s, "external_sw_if_index %d",
+		clib_net_to_host_u32 (mp->external_sw_if_index));
+  FINISH;
+}
+
+static void
+  send_snat_static_mapping_details
+  (snat_static_mapping_t * m, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_static_mapping_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_SNAT_STATIC_MAPPING_DETAILS + sm->msg_id_base);
+  rmp->is_ip4 = 1;
+  rmp->addr_only = m->addr_only;
+  clib_memcpy (rmp->local_ip_address, &(m->local_addr), 4);
+  clib_memcpy (rmp->external_ip_address, &(m->external_addr), 4);
+  rmp->local_port = htons (m->local_port);
+  rmp->external_port = htons (m->external_port);
+  rmp->external_sw_if_index = ~0;
+  rmp->vrf_id = htonl (m->vrf_id);
+  rmp->protocol = snat_proto_to_ip_proto (m->proto);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  send_snat_static_map_resolve_details
+  (snat_static_map_resolve_t * m, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_static_mapping_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_SNAT_STATIC_MAPPING_DETAILS + sm->msg_id_base);
+  rmp->is_ip4 = 1;
+  rmp->addr_only = m->addr_only;
+  clib_memcpy (rmp->local_ip_address, &(m->l_addr), 4);
+  rmp->local_port = htons (m->l_port);
+  rmp->external_port = htons (m->e_port);
+  rmp->external_sw_if_index = htonl (m->sw_if_index);
+  rmp->vrf_id = htonl (m->vrf_id);
+  rmp->protocol = snat_proto_to_ip_proto (m->proto);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  vl_api_snat_static_mapping_dump_t_handler
+  (vl_api_snat_static_mapping_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_static_mapping_t *m;
+  snat_static_map_resolve_t *rp;
+  int j;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (m, sm->static_mappings,
+  ({
+      send_snat_static_mapping_details (m, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+
+  for (j = 0; j < vec_len (sm->to_resolve); j++)
+    {
+      rp = sm->to_resolve + j;
+      send_snat_static_map_resolve_details (rp, q, mp->context);
+    }
+}
+
+static void *vl_api_snat_static_mapping_dump_t_print
+  (vl_api_snat_static_mapping_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_static_mapping_dump ");
+
+  FINISH;
+}
+
+static void
+vl_api_snat_control_ping_t_handler (vl_api_snat_control_ping_t * mp)
+{
+  vl_api_snat_control_ping_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_SNAT_CONTROL_PING_REPLY,
+  ({
+    rmp->vpe_pid = ntohl (getpid ());
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_control_ping_t_print
+  (vl_api_snat_control_ping_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_control_ping ");
+
+  FINISH;
+}
+
+static void
+vl_api_snat_show_config_t_handler (vl_api_snat_show_config_t * mp)
+{
+  vl_api_snat_show_config_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_SNAT_SHOW_CONFIG_REPLY,
+  ({
+    rmp->translation_buckets = htonl (sm->translation_buckets);
+    rmp->translation_memory_size = htonl (sm->translation_memory_size);
+    rmp->user_buckets = htonl (sm->user_buckets);
+    rmp->user_memory_size = htonl (sm->user_memory_size);
+    rmp->max_translations_per_user = htonl (sm->max_translations_per_user);
+    rmp->outside_vrf_id = htonl (sm->outside_vrf_id);
+    rmp->inside_vrf_id = htonl (sm->inside_vrf_id);
+    rmp->static_mapping_only = sm->static_mapping_only;
+    rmp->static_mapping_connection_tracking =
+      sm->static_mapping_connection_tracking;
+    rmp->deterministic = sm->deterministic;
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_show_config_t_print
+  (vl_api_snat_show_config_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_show_config ");
+
+  FINISH;
+}
+
+static void
+vl_api_snat_set_workers_t_handler (vl_api_snat_set_workers_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_set_workers_reply_t *rmp;
+  int rv = 0;
+  uword *bitmap = 0;
+  u64 mask = clib_net_to_host_u64 (mp->worker_mask);
+
+  if (sm->num_workers < 2)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask));
+  rv = snat_set_workers (bitmap);
+  clib_bitmap_free (bitmap);
+
+send_reply:
+  REPLY_MACRO (VL_API_SNAT_SET_WORKERS_REPLY);
+}
+
+static void *vl_api_snat_set_workers_t_print
+  (vl_api_snat_set_workers_t * mp, void *handle)
+{
+  u8 *s;
+  uword *bitmap = 0;
+  u8 first = 1;
+  int i;
+  u64 mask = clib_net_to_host_u64 (mp->worker_mask);
+
+  s = format (0, "SCRIPT: snat_set_workers ");
+  bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask));
+  /* *INDENT-OFF* */
+  clib_bitmap_foreach (i, bitmap,
+    ({
+      if (first)
+        s = format (s, "%d", i);
+      else
+        s = format (s, ",%d", i);
+      first = 0;
+    }));
+  /* *INDENT-ON* */
+  clib_bitmap_free (bitmap);
+  FINISH;
+}
+
+static void
+  send_snat_worker_details
+  (u32 worker_index, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_worker_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  vlib_worker_thread_t *w =
+    vlib_worker_threads + worker_index + sm->first_worker_index;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_SNAT_WORKER_DETAILS + sm->msg_id_base);
+  rmp->context = context;
+  rmp->worker_index = htonl (worker_index);
+  rmp->lcore_id = htonl (w->lcore_id);
+  strncpy ((char *) rmp->name, (char *) w->name, ARRAY_LEN (rmp->name) - 1);
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_snat_worker_dump_t_handler (vl_api_snat_worker_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  u32 *worker_index;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (worker_index, sm->workers)
+    send_snat_worker_details(*worker_index, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_worker_dump_t_print
+  (vl_api_snat_worker_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_worker_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_add_del_interface_addr_t_handler
+  (vl_api_snat_add_del_interface_addr_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_add_del_interface_addr_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_add_interface_address (sm, sw_if_index, is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SNAT_ADD_DEL_INTERFACE_ADDR_REPLY);
+}
+
+static void *vl_api_snat_add_del_interface_addr_t_print
+  (vl_api_snat_add_del_interface_addr_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_add_del_interface_addr ");
+  s = format (s, "sw_if_index %d %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+  send_snat_interface_addr_details
+  (u32 sw_if_index, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_interface_addr_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_SNAT_INTERFACE_ADDR_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (sw_if_index);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  vl_api_snat_interface_addr_dump_t_handler
+  (vl_api_snat_interface_addr_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  u32 *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (i, sm->auto_add_sw_if_indices)
+    send_snat_interface_addr_details(*i, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_interface_addr_dump_t_print
+  (vl_api_snat_interface_addr_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_addr_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_ipfix_enable_disable_t_handler
+  (vl_api_snat_ipfix_enable_disable_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_ipfix_enable_disable_reply_t *rmp;
+  int rv = 0;
+
+  rv = snat_ipfix_logging_enable_disable (mp->enable,
+					  clib_host_to_net_u32
+					  (mp->domain_id),
+					  clib_host_to_net_u16
+					  (mp->src_port));
+
+  REPLY_MACRO (VL_API_SNAT_IPFIX_ENABLE_DISABLE_REPLY);
+}
+
+static void *vl_api_snat_ipfix_enable_disable_t_print
+  (vl_api_snat_ipfix_enable_disable_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_ipfix_enable_disable ");
+  if (mp->domain_id)
+    s = format (s, "domain %d ", clib_net_to_host_u32 (mp->domain_id));
+  if (mp->src_port)
+    s = format (s, "src_port %d ", clib_net_to_host_u16 (mp->src_port));
+  if (!mp->enable)
+    s = format (s, "disable ");
+
+  FINISH;
+}
+
+static void
+  send_snat_user_details
+  (snat_user_t * u, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_user_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  fib_table_t *fib = fib_table_get (u->fib_index, FIB_PROTOCOL_IP4);
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_SNAT_USER_DETAILS + sm->msg_id_base);
+
+  rmp->vrf_id = ntohl (fib->ft_table_id);
+
+  rmp->is_ip4 = 1;
+  clib_memcpy (rmp->ip_address, &(u->addr), 4);
+  rmp->nsessions = ntohl (u->nsessions);
+  rmp->nstaticsessions = ntohl (u->nstaticsessions);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_snat_user_dump_t_handler (vl_api_snat_user_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_main_per_thread_data_t *tsm;
+  snat_user_t *u;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (tsm, sm->per_thread_data)
+    vec_foreach (u, tsm->users)
+      send_snat_user_details (u, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_user_dump_t_print
+  (vl_api_snat_user_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_user_dump ");
+
+  FINISH;
+}
+
+static void
+  send_snat_user_session_details
+  (snat_session_t * s, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_user_session_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_SNAT_USER_SESSION_DETAILS + sm->msg_id_base);
+  rmp->is_ip4 = 1;
+  clib_memcpy (rmp->outside_ip_address, (&s->out2in.addr), 4);
+  clib_memcpy (rmp->inside_ip_address, (&s->in2out.addr), 4);
+  rmp->is_static = s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING ? 1 : 0;
+  rmp->last_heard = clib_host_to_net_u64 ((u64) s->last_heard);
+  rmp->total_bytes = clib_host_to_net_u64 (s->total_bytes);
+  rmp->total_pkts = ntohl (s->total_pkts);
+  rmp->context = context;
+  if (snat_is_unk_proto_session (s))
+    {
+      rmp->outside_port = 0;
+      rmp->inside_port = 0;
+      rmp->protocol = ntohs (s->in2out.port);
+    }
+  else
+    {
+      rmp->outside_port = s->out2in.port;
+      rmp->inside_port = s->in2out.port;
+      rmp->protocol = ntohs (snat_proto_to_ip_proto (s->in2out.protocol));
+    }
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  vl_api_snat_user_session_dump_t_handler
+  (vl_api_snat_user_session_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_main_per_thread_data_t *tsm;
+  snat_session_t *s;
+  clib_bihash_kv_8_8_t key, value;
+  snat_user_key_t ukey;
+  snat_user_t *u;
+  u32 session_index, head_index, elt_index;
+  dlist_elt_t *head, *elt;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+  if (!mp->is_ip4)
+    return;
+
+  clib_memcpy (&ukey.addr, mp->ip_address, 4);
+  ukey.fib_index = fib_table_find (FIB_PROTOCOL_IP4, ntohl (mp->vrf_id));
+  key.key = ukey.as_u64;
+  if (!clib_bihash_search_8_8 (&sm->worker_by_in, &key, &value))
+    tsm = vec_elt_at_index (sm->per_thread_data, value.value);
+  else
+    tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
+  if (clib_bihash_search_8_8 (&sm->user_hash, &key, &value))
+    return;
+  u = pool_elt_at_index (tsm->users, value.value);
+  if (!u->nsessions && !u->nstaticsessions)
+    return;
+
+  head_index = u->sessions_per_user_list_head_index;
+  head = pool_elt_at_index (tsm->list_pool, head_index);
+  elt_index = head->next;
+  elt = pool_elt_at_index (tsm->list_pool, elt_index);
+  session_index = elt->value;
+  while (session_index != ~0)
+    {
+      s = pool_elt_at_index (tsm->sessions, session_index);
+
+      send_snat_user_session_details (s, q, mp->context);
+
+      elt_index = elt->next;
+      elt = pool_elt_at_index (tsm->list_pool, elt_index);
+      session_index = elt->value;
+    }
+}
+
+static void *vl_api_snat_user_session_dump_t_print
+  (vl_api_snat_user_session_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_user_session_dump ");
+  s = format (s, "ip_address %U vrf_id %d\n",
+	      format_ip4_address, mp->ip_address,
+	      clib_net_to_host_u32 (mp->vrf_id));
+
+  FINISH;
+}
+
+/******************************************************************/
+/*** detrministic NAT/CGN (old, will be deprecated after 17.10) ***/
+/******************************************************************/
+
+static void
+vl_api_snat_add_det_map_t_handler (vl_api_snat_add_det_map_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_add_det_map_reply_t *rmp;
+  int rv = 0;
+  ip4_address_t in_addr, out_addr;
+
+  clib_memcpy (&in_addr, mp->in_addr, 4);
+  clib_memcpy (&out_addr, mp->out_addr, 4);
+  rv = snat_det_add_map (sm, &in_addr, mp->in_plen, &out_addr,
+			 mp->out_plen, mp->is_add);
+
+  REPLY_MACRO (VL_API_SNAT_ADD_DET_MAP_REPLY);
+}
+
+static void *vl_api_snat_add_det_map_t_print
+  (vl_api_snat_add_det_map_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_add_det_map ");
+  s = format (s, "inside address %U/%d outside address %U/%d\n",
+	      format_ip4_address, mp->in_addr, mp->in_plen,
+	      format_ip4_address, mp->out_addr, mp->out_plen);
+
+  FINISH;
+}
+
+static void
+vl_api_snat_det_forward_t_handler (vl_api_snat_det_forward_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_det_forward_reply_t *rmp;
+  int rv = 0;
+  u16 lo_port = 0, hi_port = 0;
+  snat_det_map_t *dm;
+  ip4_address_t in_addr, out_addr;
+
+  out_addr.as_u32 = 0;
+  clib_memcpy (&in_addr, mp->in_addr, 4);
+  dm = snat_det_map_by_user (sm, &in_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+
+  snat_det_forward (dm, &in_addr, &out_addr, &lo_port);
+  hi_port = lo_port + dm->ports_per_host - 1;
+
+send_reply:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_SNAT_DET_FORWARD_REPLY,
+  ({
+    rmp->out_port_lo = ntohs (lo_port);
+    rmp->out_port_hi = ntohs (hi_port);
+    rmp->is_ip4 = 1;
+    memset (rmp->out_addr, 0, 16);
+    clib_memcpy (rmp->out_addr, &out_addr, 4);
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_det_forward_t_print
+  (vl_api_snat_det_forward_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: smat_det_forward_t");
+  s = format (s, "inside ip address %U\n", format_ip4_address, mp->in_addr);
+
+  FINISH;
+}
+
+static void
+vl_api_snat_det_reverse_t_handler (vl_api_snat_det_reverse_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_det_reverse_reply_t *rmp;
+  int rv = 0;
+  ip4_address_t out_addr, in_addr;
+  snat_det_map_t *dm;
+
+  in_addr.as_u32 = 0;
+  clib_memcpy (&out_addr, mp->out_addr, 4);
+  dm = snat_det_map_by_out (sm, &out_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+
+  snat_det_reverse (dm, &out_addr, htons (mp->out_port), &in_addr);
+
+send_reply:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_SNAT_DET_REVERSE_REPLY,
+  ({
+    rmp->is_ip4 = 1;
+    memset (rmp->in_addr, 0, 16);
+    clib_memcpy (rmp->in_addr, &in_addr, 4);
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_det_reverse_t_print
+  (vl_api_snat_det_reverse_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: smat_det_reverse_t");
+  s = format (s, "outside ip address %U outside port %d",
+	      format_ip4_address, mp->out_addr, ntohs (mp->out_port));
+
+  FINISH;
+}
+
+static void
+  sent_snat_det_map_details
+  (snat_det_map_t * m, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_det_map_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_SNAT_DET_MAP_DETAILS + sm->msg_id_base);
+  rmp->is_ip4 = 1;
+  clib_memcpy (rmp->in_addr, &m->in_addr, 4);
+  rmp->in_plen = m->in_plen;
+  clib_memcpy (rmp->out_addr, &m->out_addr, 4);
+  rmp->out_plen = m->out_plen;
+  rmp->sharing_ratio = htonl (m->sharing_ratio);
+  rmp->ports_per_host = htons (m->ports_per_host);
+  rmp->ses_num = htonl (m->ses_num);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_snat_det_map_dump_t_handler (vl_api_snat_det_map_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_det_map_t *m;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach(m, sm->det_maps)
+    sent_snat_det_map_details(m, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_det_map_dump_t_print
+  (vl_api_snat_det_map_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_det_map_dump ");
+
+  FINISH;
+}
+
+static void
+vl_api_snat_det_set_timeouts_t_handler (vl_api_snat_det_set_timeouts_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_det_set_timeouts_reply_t *rmp;
+  int rv = 0;
+
+  sm->udp_timeout = ntohl (mp->udp);
+  sm->tcp_established_timeout = ntohl (mp->tcp_established);
+  sm->tcp_transitory_timeout = ntohl (mp->tcp_transitory);
+  sm->icmp_timeout = ntohl (mp->icmp);
+
+  REPLY_MACRO (VL_API_SNAT_DET_SET_TIMEOUTS_REPLY);
+}
+
+static void *vl_api_snat_det_set_timeouts_t_print
+  (vl_api_snat_det_set_timeouts_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_det_set_timeouts ");
+  s = format (s, "udp %d tcp_established %d tcp_transitory %d icmp %d\n",
+	      ntohl (mp->udp),
+	      ntohl (mp->tcp_established),
+	      ntohl (mp->tcp_transitory), ntohl (mp->icmp));
+
+  FINISH;
+}
+
+static void
+vl_api_snat_det_get_timeouts_t_handler (vl_api_snat_det_get_timeouts_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_det_get_timeouts_reply_t *rmp;
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_SNAT_DET_GET_TIMEOUTS_REPLY,
+  ({
+    rmp->udp = htonl (sm->udp_timeout);
+    rmp->tcp_established = htonl (sm->tcp_established_timeout);
+    rmp->tcp_transitory = htonl (sm->tcp_transitory_timeout);
+    rmp->icmp = htonl (sm->icmp_timeout);
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_snat_det_get_timeouts_t_print
+  (vl_api_snat_det_get_timeouts_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_det_get_timeouts");
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_det_close_session_out_t_handler
+  (vl_api_snat_det_close_session_out_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_det_close_session_out_reply_t *rmp;
+  ip4_address_t out_addr, ext_addr, in_addr;
+  snat_det_out_key_t key;
+  snat_det_map_t *dm;
+  snat_det_session_t *ses;
+  int rv = 0;
+
+  clib_memcpy (&out_addr, mp->out_addr, 4);
+  clib_memcpy (&ext_addr, mp->ext_addr, 4);
+
+  dm = snat_det_map_by_out (sm, &out_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  snat_det_reverse (dm, &ext_addr, ntohs (mp->out_port), &in_addr);
+  key.ext_host_addr = ext_addr;
+  key.ext_host_port = mp->ext_port;
+  key.out_port = mp->out_port;
+  ses = snat_det_get_ses_by_out (dm, &in_addr, key.as_u64);
+  if (!ses)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  snat_det_ses_close (dm, ses);
+
+send_reply:
+  REPLY_MACRO (VL_API_SNAT_DET_CLOSE_SESSION_OUT_REPLY);
+}
+
+static void *vl_api_snat_det_close_session_out_t_print
+  (vl_api_snat_det_close_session_out_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_det_close_session_out ");
+  s = format (s, "out_addr %U out_port %d "
+	      "ext_addr %U ext_port %d\n",
+	      format_ip4_address, mp->out_addr, ntohs (mp->out_port),
+	      format_ip4_address, mp->ext_addr, ntohs (mp->ext_port));
+
+  FINISH;
+}
+
+static void
+  vl_api_snat_det_close_session_in_t_handler
+  (vl_api_snat_det_close_session_in_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_det_close_session_in_reply_t *rmp;
+  ip4_address_t in_addr, ext_addr;
+  snat_det_out_key_t key;
+  snat_det_map_t *dm;
+  snat_det_session_t *ses;
+  int rv = 0;
+
+  clib_memcpy (&in_addr, mp->in_addr, 4);
+  clib_memcpy (&ext_addr, mp->ext_addr, 4);
+
+  dm = snat_det_map_by_user (sm, &in_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  key.ext_host_addr = ext_addr;
+  key.ext_host_port = mp->ext_port;
+  ses = snat_det_find_ses_by_in (dm, &in_addr, mp->in_port, key);
+  if (!ses)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  snat_det_ses_close (dm, ses);
+
+send_reply:
+  REPLY_MACRO (VL_API_SNAT_DET_CLOSE_SESSION_OUT_REPLY);
+}
+
+static void *vl_api_snat_det_close_session_in_t_print
+  (vl_api_snat_det_close_session_in_t * mp, void *handle)
+{
+  u8 *s;
+  s = format (0, "SCRIPT: snat_det_close_session_in ");
+  s = format (s, "in_addr %U in_port %d "
+	      "ext_addr %U ext_port %d\n",
+	      format_ip4_address, mp->in_addr, ntohs (mp->in_port),
+	      format_ip4_address, mp->ext_addr, ntohs (mp->ext_port));
+
+  FINISH;
+}
+
+static void
+  send_snat_det_session_details
+  (snat_det_session_t * s, unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_snat_det_session_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_SNAT_DET_SESSION_DETAILS + sm->msg_id_base);
+  rmp->is_ip4 = 1;
+  rmp->in_port = s->in_port;
+  clib_memcpy (rmp->ext_addr, &s->out.ext_host_addr, 4);
+  rmp->ext_port = s->out.ext_host_port;
+  rmp->out_port = s->out.out_port;
+  rmp->state = s->state;
+  rmp->expire = ntohl (s->expire);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_snat_det_session_dump_t_handler (vl_api_snat_det_session_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  ip4_address_t user_addr;
+  snat_det_map_t *dm;
+  snat_det_session_t *s, empty_ses;
+  u16 i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+  if (!mp->is_ip4)
+    return;
+
+  memset (&empty_ses, 0, sizeof (empty_ses));
+  clib_memcpy (&user_addr, mp->user_addr, 4);
+  dm = snat_det_map_by_user (sm, &user_addr);
+  if (!dm)
+    return;
+
+  s = dm->sessions + snat_det_user_ses_offset (&user_addr, dm->in_plen);
+  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
+    {
+      if (s->out.as_u64)
+	send_snat_det_session_details (s, q, mp->context);
+      s++;
+    }
+}
+
+static void *vl_api_snat_det_session_dump_t_print
+  (vl_api_snat_det_session_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_det_session_dump ");
+  s = format (s, "user_addr %U\n", format_ip4_address, mp->user_addr);
+
+  FINISH;
+}
+
+/******************************/
+/*** Common NAT plugin APIs ***/
+/******************************/
+
+static void
+vl_api_nat_control_ping_t_handler (vl_api_nat_control_ping_t * mp)
+{
+  vl_api_nat_control_ping_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_NAT_CONTROL_PING_REPLY,
+  ({
+    rmp->vpe_pid = ntohl (getpid ());
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_control_ping_t_print (vl_api_nat_control_ping_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_control_ping ");
+
+  FINISH;
+}
+
+static void
+vl_api_nat_show_config_t_handler (vl_api_nat_show_config_t * mp)
+{
+  vl_api_nat_show_config_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_NAT_SHOW_CONFIG_REPLY,
+  ({
+    rmp->translation_buckets = htonl (sm->translation_buckets);
+    rmp->translation_memory_size = htonl (sm->translation_memory_size);
+    rmp->user_buckets = htonl (sm->user_buckets);
+    rmp->user_memory_size = htonl (sm->user_memory_size);
+    rmp->max_translations_per_user = htonl (sm->max_translations_per_user);
+    rmp->outside_vrf_id = htonl (sm->outside_vrf_id);
+    rmp->inside_vrf_id = htonl (sm->inside_vrf_id);
+    rmp->static_mapping_only = sm->static_mapping_only;
+    rmp->static_mapping_connection_tracking =
+      sm->static_mapping_connection_tracking;
+    rmp->deterministic = sm->deterministic;
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_show_config_t_print (vl_api_nat_show_config_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_show_config ");
+
+  FINISH;
+}
+
+static void
+vl_api_nat_set_workers_t_handler (vl_api_nat_set_workers_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_snat_set_workers_reply_t *rmp;
+  int rv = 0;
+  uword *bitmap = 0;
+  u64 mask = clib_net_to_host_u64 (mp->worker_mask);
+
+  if (sm->num_workers < 2)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask));
+  rv = snat_set_workers (bitmap);
+  clib_bitmap_free (bitmap);
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT_SET_WORKERS_REPLY);
+}
+
+static void *
+vl_api_nat_set_workers_t_print (vl_api_nat_set_workers_t * mp, void *handle)
+{
+  u8 *s;
+  uword *bitmap = 0;
+  u8 first = 1;
+  int i;
+  u64 mask = clib_net_to_host_u64 (mp->worker_mask);
+
+  s = format (0, "SCRIPT: nat_set_workers ");
+  bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask));
+  /* *INDENT-OFF* */
+  clib_bitmap_foreach (i, bitmap,
+    ({
+      if (first)
+        s = format (s, "%d", i);
+      else
+        s = format (s, ",%d", i);
+      first = 0;
+    }));
+  /* *INDENT-ON* */
+  clib_bitmap_free (bitmap);
+  FINISH;
+}
+
+static void
+send_nat_worker_details (u32 worker_index, unix_shared_memory_queue_t * q,
+			 u32 context)
+{
+  vl_api_nat_worker_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  vlib_worker_thread_t *w =
+    vlib_worker_threads + worker_index + sm->first_worker_index;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT_WORKER_DETAILS + sm->msg_id_base);
+  rmp->context = context;
+  rmp->worker_index = htonl (worker_index);
+  rmp->lcore_id = htonl (w->lcore_id);
+  strncpy ((char *) rmp->name, (char *) w->name, ARRAY_LEN (rmp->name) - 1);
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat_worker_dump_t_handler (vl_api_nat_worker_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  u32 *worker_index;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (worker_index, sm->workers)
+    send_nat_worker_details(*worker_index, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_worker_dump_t_print (vl_api_nat_worker_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_worker_dump ");
+
+  FINISH;
+}
+
+static void
+vl_api_nat_ipfix_enable_disable_t_handler (vl_api_nat_ipfix_enable_disable_t *
+					   mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_ipfix_enable_disable_reply_t *rmp;
+  int rv = 0;
+
+  rv = snat_ipfix_logging_enable_disable (mp->enable,
+					  clib_host_to_net_u32
+					  (mp->domain_id),
+					  clib_host_to_net_u16
+					  (mp->src_port));
+
+  REPLY_MACRO (VL_API_NAT_IPFIX_ENABLE_DISABLE_REPLY);
+}
+
+static void *
+vl_api_nat_ipfix_enable_disable_t_print (vl_api_nat_ipfix_enable_disable_t *
+					 mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_ipfix_enable_disable ");
+  if (mp->domain_id)
+    s = format (s, "domain %d ", clib_net_to_host_u32 (mp->domain_id));
+  if (mp->src_port)
+    s = format (s, "src_port %d ", clib_net_to_host_u16 (mp->src_port));
+  if (!mp->enable)
+    s = format (s, "disable ");
+
+  FINISH;
+}
+
+/*************/
+/*** NAT44 ***/
+/*************/
+static void
+  vl_api_nat44_add_del_address_range_t_handler
+  (vl_api_nat44_add_del_address_range_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat44_add_del_address_range_reply_t *rmp;
+  ip4_address_t this_addr;
+  u32 start_host_order, end_host_order;
+  u32 vrf_id;
+  int i, count;
+  int rv = 0;
+  u32 *tmp;
+
+  if (sm->static_mapping_only)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  tmp = (u32 *) mp->first_ip_address;
+  start_host_order = clib_host_to_net_u32 (tmp[0]);
+  tmp = (u32 *) mp->last_ip_address;
+  end_host_order = clib_host_to_net_u32 (tmp[0]);
+
+  count = (end_host_order - start_host_order) + 1;
+
+  vrf_id = clib_host_to_net_u32 (mp->vrf_id);
+
+  if (count > 1024)
+    clib_warning ("%U - %U, %d addresses...",
+		  format_ip4_address, mp->first_ip_address,
+		  format_ip4_address, mp->last_ip_address, count);
+
+  memcpy (&this_addr.as_u8, mp->first_ip_address, 4);
+
+  for (i = 0; i < count; i++)
+    {
+      if (mp->is_add)
+	snat_add_address (sm, &this_addr, vrf_id);
+      else
+	rv = snat_del_address (sm, this_addr, 0);
+
+      if (rv)
+	goto send_reply;
+
+      increment_v4_address (&this_addr);
+    }
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT44_ADD_DEL_ADDRESS_RANGE_REPLY);
+}
+
+static void *vl_api_nat44_add_del_address_range_t_print
+  (vl_api_nat44_add_del_address_range_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_add_address_range ");
+  s = format (s, "%U ", format_ip4_address, mp->first_ip_address);
+  if (memcmp (mp->first_ip_address, mp->last_ip_address, 4))
+    {
+      s = format (s, " - %U ", format_ip4_address, mp->last_ip_address);
+    }
+  FINISH;
+}
+
+static void
+send_nat44_address_details (snat_address_t * a,
+			    unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_nat44_address_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT44_ADDRESS_DETAILS + sm->msg_id_base);
+  clib_memcpy (rmp->ip_address, &(a->addr), 4);
+  if (a->fib_index != ~0)
+    {
+      fib_table_t *fib = fib_table_get (a->fib_index, FIB_PROTOCOL_IP4);
+      rmp->vrf_id = ntohl (fib->ft_table_id);
+    }
+  else
+    rmp->vrf_id = ~0;
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat44_address_dump_t_handler (vl_api_nat44_address_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_address_t *a;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (a, sm->addresses)
+    send_nat44_address_details (a, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat44_address_dump_t_print (vl_api_nat44_address_dump_t * mp,
+				   void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_address_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_nat44_interface_add_del_feature_t_handler
+  (vl_api_nat44_interface_add_del_feature_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat44_interface_add_del_feature_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_interface_add_del (sw_if_index, mp->is_inside, is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_NAT44_INTERFACE_ADD_DEL_FEATURE_REPLY);
+}
+
+static void *vl_api_nat44_interface_add_del_feature_t_print
+  (vl_api_nat44_interface_add_del_feature_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_interface_add_del_feature ");
+  s = format (s, "sw_if_index %d %s %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+send_nat44_interface_details (snat_interface_t * i,
+			      unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_nat44_interface_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT44_INTERFACE_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (i->sw_if_index);
+  rmp->is_inside = i->is_inside;
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat44_interface_dump_t_handler (vl_api_nat44_interface_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (i, sm->interfaces,
+  ({
+    send_nat44_interface_details(i, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat44_interface_dump_t_print (vl_api_nat44_interface_dump_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_interface_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_nat44_interface_add_del_output_feature_t_handler
+  (vl_api_nat44_interface_add_del_output_feature_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat44_interface_add_del_output_feature_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_interface_add_del_output_feature (sw_if_index, mp->is_inside,
+					      is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_NAT44_INTERFACE_ADD_DEL_OUTPUT_FEATURE_REPLY);
+}
+
+static void *vl_api_nat44_interface_add_del_output_feature_t_print
+  (vl_api_nat44_interface_add_del_output_feature_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_interface_add_del_output_feature ");
+  s = format (s, "sw_if_index %d %s %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+send_nat44_interface_output_feature_details (snat_interface_t * i,
+					     unix_shared_memory_queue_t * q,
+					     u32 context)
+{
+  vl_api_nat44_interface_output_feature_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_NAT44_INTERFACE_OUTPUT_FEATURE_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (i->sw_if_index);
+  rmp->context = context;
+  rmp->is_inside = i->is_inside;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+  vl_api_nat44_interface_output_feature_dump_t_handler
+  (vl_api_nat44_interface_output_feature_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_interface_t *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (i, sm->output_feature_interfaces,
+  ({
+    send_nat44_interface_output_feature_details(i, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_nat44_interface_output_feature_dump_t_print
+  (vl_api_nat44_interface_output_feature_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_interface_output_feature_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_nat44_add_del_static_mapping_t_handler
+  (vl_api_nat44_add_del_static_mapping_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat44_add_del_static_mapping_reply_t *rmp;
+  ip4_address_t local_addr, external_addr;
+  u16 local_port = 0, external_port = 0;
+  u32 vrf_id, external_sw_if_index;
+  int rv = 0;
+  snat_protocol_t proto;
+
+  memcpy (&local_addr.as_u8, mp->local_ip_address, 4);
+  memcpy (&external_addr.as_u8, mp->external_ip_address, 4);
+  if (mp->addr_only == 0)
+    {
+      local_port = clib_net_to_host_u16 (mp->local_port);
+      external_port = clib_net_to_host_u16 (mp->external_port);
+    }
+  vrf_id = clib_net_to_host_u32 (mp->vrf_id);
+  external_sw_if_index = clib_net_to_host_u32 (mp->external_sw_if_index);
+  proto = ip_proto_to_snat_proto (mp->protocol);
+
+  rv = snat_add_static_mapping (local_addr, external_addr, local_port,
+				external_port, vrf_id, mp->addr_only,
+				external_sw_if_index, proto, mp->is_add);
+
+  REPLY_MACRO (VL_API_NAT44_ADD_DEL_STATIC_MAPPING_REPLY);
+}
+
+static void *vl_api_nat44_add_del_static_mapping_t_print
+  (vl_api_nat44_add_del_static_mapping_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_add_del_static_mapping ");
+  s = format (s, "protocol %d local_addr %U external_addr %U ",
+	      mp->protocol,
+	      format_ip4_address, mp->local_ip_address,
+	      format_ip4_address, mp->external_ip_address);
+
+  if (mp->addr_only == 0)
+    s = format (s, "local_port %d external_port %d ",
+		clib_net_to_host_u16 (mp->local_port),
+		clib_net_to_host_u16 (mp->external_port));
+
+  if (mp->vrf_id != ~0)
+    s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id));
+
+  if (mp->external_sw_if_index != ~0)
+    s = format (s, "external_sw_if_index %d",
+		clib_net_to_host_u32 (mp->external_sw_if_index));
+  FINISH;
+}
+
+static void
+send_nat44_static_mapping_details (snat_static_mapping_t * m,
+				   unix_shared_memory_queue_t * q,
+				   u32 context)
+{
+  vl_api_nat44_static_mapping_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_NAT44_STATIC_MAPPING_DETAILS + sm->msg_id_base);
+  rmp->addr_only = m->addr_only;
+  clib_memcpy (rmp->local_ip_address, &(m->local_addr), 4);
+  clib_memcpy (rmp->external_ip_address, &(m->external_addr), 4);
+  rmp->local_port = htons (m->local_port);
+  rmp->external_port = htons (m->external_port);
+  rmp->external_sw_if_index = ~0;
+  rmp->vrf_id = htonl (m->vrf_id);
+  rmp->protocol = snat_proto_to_ip_proto (m->proto);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+send_nat44_static_map_resolve_details (snat_static_map_resolve_t * m,
+				       unix_shared_memory_queue_t * q,
+				       u32 context)
+{
+  vl_api_nat44_static_mapping_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_NAT44_STATIC_MAPPING_DETAILS + sm->msg_id_base);
+  rmp->addr_only = m->addr_only;
+  clib_memcpy (rmp->local_ip_address, &(m->l_addr), 4);
+  rmp->local_port = htons (m->l_port);
+  rmp->external_port = htons (m->e_port);
+  rmp->external_sw_if_index = htonl (m->sw_if_index);
+  rmp->vrf_id = htonl (m->vrf_id);
+  rmp->protocol = snat_proto_to_ip_proto (m->proto);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat44_static_mapping_dump_t_handler (vl_api_nat44_static_mapping_dump_t
+					    * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_static_mapping_t *m;
+  snat_static_map_resolve_t *rp;
+  int j;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  pool_foreach (m, sm->static_mappings,
+  ({
+      send_nat44_static_mapping_details (m, q, mp->context);
+  }));
+  /* *INDENT-ON* */
+
+  for (j = 0; j < vec_len (sm->to_resolve); j++)
+    {
+      rp = sm->to_resolve + j;
+      send_nat44_static_map_resolve_details (rp, q, mp->context);
+    }
+}
+
+static void *
+vl_api_nat44_static_mapping_dump_t_print (vl_api_nat44_static_mapping_dump_t *
+					  mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_static_mapping_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_nat44_add_del_interface_addr_t_handler
+  (vl_api_nat44_add_del_interface_addr_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat44_add_del_interface_addr_reply_t *rmp;
+  u8 is_del = mp->is_add == 0;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = snat_add_interface_address (sm, sw_if_index, is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_NAT44_ADD_DEL_INTERFACE_ADDR_REPLY);
+}
+
+static void *vl_api_nat44_add_del_interface_addr_t_print
+  (vl_api_nat44_add_del_interface_addr_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_add_del_interface_addr ");
+  s = format (s, "sw_if_index %d %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static void
+send_nat44_interface_addr_details (u32 sw_if_index,
+				   unix_shared_memory_queue_t * q,
+				   u32 context)
+{
+  vl_api_nat44_interface_addr_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_NAT44_INTERFACE_ADDR_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (sw_if_index);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat44_interface_addr_dump_t_handler (vl_api_nat44_interface_addr_dump_t
+					    * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  u32 *i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (i, sm->auto_add_sw_if_indices)
+    send_nat44_interface_addr_details(*i, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat44_interface_addr_dump_t_print (vl_api_nat44_interface_addr_dump_t *
+					  mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_interface_addr_dump ");
+
+  FINISH;
+}
+
+static void
+send_nat44_user_details (snat_user_t * u, unix_shared_memory_queue_t * q,
+			 u32 context)
+{
+  vl_api_nat44_user_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  fib_table_t *fib = fib_table_get (u->fib_index, FIB_PROTOCOL_IP4);
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT44_USER_DETAILS + sm->msg_id_base);
+
+  rmp->vrf_id = ntohl (fib->ft_table_id);
+
+  clib_memcpy (rmp->ip_address, &(u->addr), 4);
+  rmp->nsessions = ntohl (u->nsessions);
+  rmp->nstaticsessions = ntohl (u->nstaticsessions);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat44_user_dump_t_handler (vl_api_nat44_user_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_main_per_thread_data_t *tsm;
+  snat_user_t *u;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach (tsm, sm->per_thread_data)
+    vec_foreach (u, tsm->users)
+      send_nat44_user_details (u, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat44_user_dump_t_print (vl_api_nat44_user_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_user_dump ");
+
+  FINISH;
+}
+
+static void
+send_nat44_user_session_details (snat_session_t * s,
+				 unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_nat44_user_session_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_NAT44_USER_SESSION_DETAILS + sm->msg_id_base);
+  clib_memcpy (rmp->outside_ip_address, (&s->out2in.addr), 4);
+  clib_memcpy (rmp->inside_ip_address, (&s->in2out.addr), 4);
+  rmp->is_static = s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING ? 1 : 0;
+  rmp->last_heard = clib_host_to_net_u64 ((u64) s->last_heard);
+  rmp->total_bytes = clib_host_to_net_u64 (s->total_bytes);
+  rmp->total_pkts = ntohl (s->total_pkts);
+  rmp->context = context;
+  if (snat_is_unk_proto_session (s))
+    {
+      rmp->outside_port = 0;
+      rmp->inside_port = 0;
+      rmp->protocol = ntohs (s->in2out.port);
+    }
+  else
+    {
+      rmp->outside_port = s->out2in.port;
+      rmp->inside_port = s->in2out.port;
+      rmp->protocol = ntohs (snat_proto_to_ip_proto (s->in2out.protocol));
+    }
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat44_user_session_dump_t_handler (vl_api_nat44_user_session_dump_t *
+					  mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_main_per_thread_data_t *tsm;
+  snat_session_t *s;
+  clib_bihash_kv_8_8_t key, value;
+  snat_user_key_t ukey;
+  snat_user_t *u;
+  u32 session_index, head_index, elt_index;
+  dlist_elt_t *head, *elt;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  clib_memcpy (&ukey.addr, mp->ip_address, 4);
+  ukey.fib_index = fib_table_find (FIB_PROTOCOL_IP4, ntohl (mp->vrf_id));
+  key.key = ukey.as_u64;
+  if (!clib_bihash_search_8_8 (&sm->worker_by_in, &key, &value))
+    tsm = vec_elt_at_index (sm->per_thread_data, value.value);
+  else
+    tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
+  if (clib_bihash_search_8_8 (&sm->user_hash, &key, &value))
+    return;
+  u = pool_elt_at_index (tsm->users, value.value);
+  if (!u->nsessions && !u->nstaticsessions)
+    return;
+
+  head_index = u->sessions_per_user_list_head_index;
+  head = pool_elt_at_index (tsm->list_pool, head_index);
+  elt_index = head->next;
+  elt = pool_elt_at_index (tsm->list_pool, elt_index);
+  session_index = elt->value;
+  while (session_index != ~0)
+    {
+      s = pool_elt_at_index (tsm->sessions, session_index);
+
+      send_nat44_user_session_details (s, q, mp->context);
+
+      elt_index = elt->next;
+      elt = pool_elt_at_index (tsm->list_pool, elt_index);
+      session_index = elt->value;
+    }
+}
+
+static void *
+vl_api_nat44_user_session_dump_t_print (vl_api_nat44_user_session_dump_t * mp,
+					void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_user_session_dump ");
+  s = format (s, "ip_address %U vrf_id %d\n",
+	      format_ip4_address, mp->ip_address,
+	      clib_net_to_host_u32 (mp->vrf_id));
+
+  FINISH;
+}
+
+/*******************************/
+/*** Deterministic NAT (CGN) ***/
+/*******************************/
+
+static void
+vl_api_nat_det_add_del_map_t_handler (vl_api_nat_det_add_del_map_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_add_del_map_reply_t *rmp;
+  int rv = 0;
+  ip4_address_t in_addr, out_addr;
+
+  if (!mp->is_nat44)
+    {
+      rv = VNET_API_ERROR_UNIMPLEMENTED;
+      goto send_reply;
+    }
+
+  clib_memcpy (&in_addr, mp->in_addr, 4);
+  clib_memcpy (&out_addr, mp->out_addr, 4);
+  rv = snat_det_add_map (sm, &in_addr, mp->in_plen, &out_addr,
+			 mp->out_plen, mp->is_add);
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT_DET_ADD_DEL_MAP_REPLY);
+}
+
+static void *
+vl_api_nat_det_add_del_map_t_print (vl_api_nat_det_add_del_map_t * mp,
+				    void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_add_del_map ");
+  s = format (s, "inside address %U/%d outside address %U/%d\n",
+	      format_ip4_address, mp->in_addr, mp->in_plen,
+	      format_ip4_address, mp->out_addr, mp->out_plen);
+
+  FINISH;
+}
+
+static void
+vl_api_nat_det_forward_t_handler (vl_api_nat_det_forward_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_forward_reply_t *rmp;
+  int rv = 0;
+  u16 lo_port = 0, hi_port = 0;
+  snat_det_map_t *dm;
+  ip4_address_t in_addr, out_addr;
+
+  if (!mp->is_nat44)
+    {
+      out_addr.as_u32 = 0;
+      rv = VNET_API_ERROR_UNIMPLEMENTED;
+      goto send_reply;
+    }
+
+  out_addr.as_u32 = 0;
+  clib_memcpy (&in_addr, mp->in_addr, 4);
+  dm = snat_det_map_by_user (sm, &in_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+
+  snat_det_forward (dm, &in_addr, &out_addr, &lo_port);
+  hi_port = lo_port + dm->ports_per_host - 1;
+
+send_reply:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_NAT_DET_FORWARD_REPLY,
+  ({
+    rmp->out_port_lo = ntohs (lo_port);
+    rmp->out_port_hi = ntohs (hi_port);
+    clib_memcpy (rmp->out_addr, &out_addr, 4);
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_det_forward_t_print (vl_api_nat_det_forward_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_forward");
+  s = format (s, "inside ip address %U\n", format_ip4_address, mp->in_addr);
+
+  FINISH;
+}
+
+static void
+vl_api_nat_det_reverse_t_handler (vl_api_nat_det_reverse_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_reverse_reply_t *rmp;
+  int rv = 0;
+  ip4_address_t out_addr, in_addr;
+  snat_det_map_t *dm;
+
+  in_addr.as_u32 = 0;
+  clib_memcpy (&out_addr, mp->out_addr, 4);
+  dm = snat_det_map_by_out (sm, &out_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+
+  snat_det_reverse (dm, &out_addr, htons (mp->out_port), &in_addr);
+
+send_reply:
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_NAT_DET_REVERSE_REPLY,
+  ({
+    rmp->is_nat44 = 1;
+    memset (rmp->in_addr, 0, 16);
+    clib_memcpy (rmp->in_addr, &in_addr, 4);
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_det_reverse_t_print (vl_api_nat_det_reverse_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_reverse");
+  s = format (s, "outside ip address %U outside port %d",
+	      format_ip4_address, mp->out_addr, ntohs (mp->out_port));
+
+  FINISH;
+}
+
+static void
+sent_nat_det_map_details (snat_det_map_t * m, unix_shared_memory_queue_t * q,
+			  u32 context)
+{
+  vl_api_nat_det_map_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT_DET_MAP_DETAILS + sm->msg_id_base);
+  rmp->is_nat44 = 1;
+  clib_memcpy (rmp->in_addr, &m->in_addr, 4);
+  rmp->in_plen = m->in_plen;
+  clib_memcpy (rmp->out_addr, &m->out_addr, 4);
+  rmp->out_plen = m->out_plen;
+  rmp->sharing_ratio = htonl (m->sharing_ratio);
+  rmp->ports_per_host = htons (m->ports_per_host);
+  rmp->ses_num = htonl (m->ses_num);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat_det_map_dump_t_handler (vl_api_nat_det_map_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  snat_det_map_t *m;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  /* *INDENT-OFF* */
+  vec_foreach(m, sm->det_maps)
+    sent_nat_det_map_details(m, q, mp->context);
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_det_map_dump_t_print (vl_api_nat_det_map_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_map_dump ");
+
+  FINISH;
+}
+
+static void
+vl_api_nat_det_set_timeouts_t_handler (vl_api_nat_det_set_timeouts_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_set_timeouts_reply_t *rmp;
+  int rv = 0;
+
+  sm->udp_timeout = ntohl (mp->udp);
+  sm->tcp_established_timeout = ntohl (mp->tcp_established);
+  sm->tcp_transitory_timeout = ntohl (mp->tcp_transitory);
+  sm->icmp_timeout = ntohl (mp->icmp);
+
+  REPLY_MACRO (VL_API_NAT_DET_SET_TIMEOUTS_REPLY);
+}
+
+static void *
+vl_api_nat_det_set_timeouts_t_print (vl_api_nat_det_set_timeouts_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_set_timeouts ");
+  s = format (s, "udp %d tcp_established %d tcp_transitory %d icmp %d\n",
+	      ntohl (mp->udp),
+	      ntohl (mp->tcp_established),
+	      ntohl (mp->tcp_transitory), ntohl (mp->icmp));
+
+  FINISH;
+}
+
+static void
+vl_api_nat_det_get_timeouts_t_handler (vl_api_nat_det_get_timeouts_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_get_timeouts_reply_t *rmp;
+  int rv = 0;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_NAT_DET_GET_TIMEOUTS_REPLY,
+  ({
+    rmp->udp = htonl (sm->udp_timeout);
+    rmp->tcp_established = htonl (sm->tcp_established_timeout);
+    rmp->tcp_transitory = htonl (sm->tcp_transitory_timeout);
+    rmp->icmp = htonl (sm->icmp_timeout);
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *
+vl_api_nat_det_get_timeouts_t_print (vl_api_nat_det_get_timeouts_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_get_timeouts");
+
+  FINISH;
+}
+
+static void
+vl_api_nat_det_close_session_out_t_handler (vl_api_nat_det_close_session_out_t
+					    * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_close_session_out_reply_t *rmp;
+  ip4_address_t out_addr, ext_addr, in_addr;
+  snat_det_out_key_t key;
+  snat_det_map_t *dm;
+  snat_det_session_t *ses;
+  int rv = 0;
+
+  clib_memcpy (&out_addr, mp->out_addr, 4);
+  clib_memcpy (&ext_addr, mp->ext_addr, 4);
+
+  dm = snat_det_map_by_out (sm, &out_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  snat_det_reverse (dm, &ext_addr, ntohs (mp->out_port), &in_addr);
+  key.ext_host_addr = ext_addr;
+  key.ext_host_port = mp->ext_port;
+  key.out_port = mp->out_port;
+  ses = snat_det_get_ses_by_out (dm, &in_addr, key.as_u64);
+  if (!ses)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  snat_det_ses_close (dm, ses);
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT_DET_CLOSE_SESSION_OUT_REPLY);
+}
+
+static void *
+vl_api_nat_det_close_session_out_t_print (vl_api_nat_det_close_session_out_t *
+					  mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_close_session_out ");
+  s = format (s, "out_addr %U out_port %d "
+	      "ext_addr %U ext_port %d\n",
+	      format_ip4_address, mp->out_addr, ntohs (mp->out_port),
+	      format_ip4_address, mp->ext_addr, ntohs (mp->ext_port));
+
+  FINISH;
+}
+
+static void
+vl_api_nat_det_close_session_in_t_handler (vl_api_nat_det_close_session_in_t *
+					   mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_det_close_session_in_reply_t *rmp;
+  ip4_address_t in_addr, ext_addr;
+  snat_det_out_key_t key;
+  snat_det_map_t *dm;
+  snat_det_session_t *ses;
+  int rv = 0;
+
+  if (!mp->is_nat44)
+    {
+      rv = VNET_API_ERROR_UNIMPLEMENTED;
+      goto send_reply;
+    }
+
+  clib_memcpy (&in_addr, mp->in_addr, 4);
+  clib_memcpy (&ext_addr, mp->ext_addr, 4);
+
+  dm = snat_det_map_by_user (sm, &in_addr);
+  if (!dm)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  key.ext_host_addr = ext_addr;
+  key.ext_host_port = mp->ext_port;
+  ses = snat_det_find_ses_by_in (dm, &in_addr, mp->in_port, key);
+  if (!ses)
+    {
+      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
+      goto send_reply;
+    }
+  snat_det_ses_close (dm, ses);
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT_DET_CLOSE_SESSION_OUT_REPLY);
+}
+
+static void *
+vl_api_nat_det_close_session_in_t_print (vl_api_nat_det_close_session_in_t *
+					 mp, void *handle)
+{
+  u8 *s;
+  s = format (0, "SCRIPT: nat_det_close_session_in ");
+  s = format (s, "in_addr %U in_port %d ext_addr %U ext_port %d\n",
+	      format_ip4_address, mp->in_addr, ntohs (mp->in_port),
+	      format_ip4_address, mp->ext_addr, ntohs (mp->ext_port));
+
+  FINISH;
+}
+
+static void
+send_nat_det_session_details (snat_det_session_t * s,
+			      unix_shared_memory_queue_t * q, u32 context)
+{
+  vl_api_nat_det_session_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT_DET_SESSION_DETAILS + sm->msg_id_base);
+  rmp->in_port = s->in_port;
+  clib_memcpy (rmp->ext_addr, &s->out.ext_host_addr, 4);
+  rmp->ext_port = s->out.ext_host_port;
+  rmp->out_port = s->out.out_port;
+  rmp->state = s->state;
+  rmp->expire = ntohl (s->expire);
+  rmp->context = context;
+
+  vl_msg_api_send_shmem (q, (u8 *) & rmp);
+}
+
+static void
+vl_api_nat_det_session_dump_t_handler (vl_api_nat_det_session_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  snat_main_t *sm = &snat_main;
+  ip4_address_t user_addr;
+  snat_det_map_t *dm;
+  snat_det_session_t *s, empty_ses;
+  u16 i;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+  if (!mp->is_nat44)
+    return;
+
+  memset (&empty_ses, 0, sizeof (empty_ses));
+  clib_memcpy (&user_addr, mp->user_addr, 4);
+  dm = snat_det_map_by_user (sm, &user_addr);
+  if (!dm)
+    return;
+
+  s = dm->sessions + snat_det_user_ses_offset (&user_addr, dm->in_plen);
+  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
+    {
+      if (s->out.as_u64)
+	send_nat_det_session_details (s, q, mp->context);
+      s++;
+    }
+}
+
+static void *
+vl_api_nat_det_session_dump_t_print (vl_api_nat_det_session_dump_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat_det_session_dump ");
+  s = format (s, "user_addr %U\n", format_ip4_address, mp->user_addr);
+
+  FINISH;
+}
+
+/*************/
+/*** NAT64 ***/
+/*************/
+
+static void
+  vl_api_nat64_add_del_pool_addr_range_t_handler
+  (vl_api_nat64_add_del_pool_addr_range_t * mp)
+{
+  vl_api_nat64_add_del_pool_addr_range_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_main_t *nm = &nat64_main;
+  int rv = 0;
+  ip4_address_t this_addr;
+  u32 start_host_order, end_host_order;
+  u32 vrf_id;
+  int i, count;
+  u32 *tmp;
+
+  if (nm->is_disabled)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  tmp = (u32 *) mp->start_addr;
+  start_host_order = clib_host_to_net_u32 (tmp[0]);
+  tmp = (u32 *) mp->end_addr;
+  end_host_order = clib_host_to_net_u32 (tmp[0]);
+
+  count = (end_host_order - start_host_order) + 1;
+
+  vrf_id = clib_host_to_net_u32 (mp->vrf_id);
+
+  memcpy (&this_addr.as_u8, mp->start_addr, 4);
+
+  for (i = 0; i < count; i++)
+    {
+      if ((rv = nat64_add_del_pool_addr (&this_addr, vrf_id, mp->is_add)))
+	goto send_reply;
+
+      increment_v4_address (&this_addr);
+    }
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT64_ADD_DEL_POOL_ADDR_RANGE_REPLY);
+}
+
+static void *vl_api_nat64_add_del_pool_addr_range_t_print
+  (vl_api_nat64_add_del_pool_addr_range_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_add_del_pool_addr_range ");
+  s = format (s, "%U - %U vrf_id %u %s\n",
+	      format_ip4_address, mp->start_addr,
+	      format_ip4_address, mp->end_addr,
+	      ntohl (mp->vrf_id), mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+typedef struct nat64_api_walk_ctx_t_
+{
+  unix_shared_memory_queue_t *q;
+  u32 context;
+} nat64_api_walk_ctx_t;
+
+static int
+nat64_api_pool_walk (snat_address_t * a, void *arg)
+{
+  vl_api_nat64_pool_addr_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_api_walk_ctx_t *ctx = arg;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT64_POOL_ADDR_DETAILS + sm->msg_id_base);
+  clib_memcpy (rmp->address, &(a->addr), 4);
+  if (a->fib_index != ~0)
+    {
+      fib_table_t *fib = fib_table_get (a->fib_index, FIB_PROTOCOL_IP6);
+      if (!fib)
+	return -1;
+      rmp->vrf_id = ntohl (fib->ft_table_id);
+    }
+  else
+    rmp->vrf_id = ~0;
+  rmp->context = ctx->context;
+
+  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
+
+  return 0;
+}
+
+static void
+vl_api_nat64_pool_addr_dump_t_handler (vl_api_nat64_pool_addr_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  nat64_api_walk_ctx_t ctx = {
+    .q = q,
+    .context = mp->context,
+  };
+
+  nat64_pool_addr_walk (nat64_api_pool_walk, &ctx);
+}
+
+static void *
+vl_api_nat64_pool_addr_dump_t_print (vl_api_nat64_pool_addr_dump_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_pool_addr_dump\n");
+
+  FINISH;
+}
+
+static void
+vl_api_nat64_add_del_interface_t_handler (vl_api_nat64_add_del_interface_t *
+					  mp)
+{
+  snat_main_t *sm = &snat_main;
+  nat64_main_t *nm = &nat64_main;
+  vl_api_nat64_add_del_interface_reply_t *rmp;
+  int rv = 0;
+
+  if (nm->is_disabled)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv =
+    nat64_add_del_interface (ntohl (mp->sw_if_index), mp->is_inside,
+			     mp->is_add);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT64_ADD_DEL_INTERFACE_REPLY);
+}
+
+static void *
+vl_api_nat64_add_del_interface_t_print (vl_api_nat64_add_del_interface_t * mp,
+					void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_add_del_interface ");
+  s = format (s, "sw_if_index %d %s %s",
+	      clib_host_to_net_u32 (mp->sw_if_index),
+	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static int
+nat64_api_interface_walk (snat_interface_t * i, void *arg)
+{
+  vl_api_nat64_interface_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_api_walk_ctx_t *ctx = arg;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT64_INTERFACE_DETAILS + sm->msg_id_base);
+  rmp->sw_if_index = ntohl (i->sw_if_index);
+  rmp->is_inside = i->is_inside;
+  rmp->context = ctx->context;
+
+  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
+
+  return 0;
+}
+
+static void
+vl_api_nat64_interface_dump_t_handler (vl_api_nat64_interface_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  nat64_api_walk_ctx_t ctx = {
+    .q = q,
+    .context = mp->context,
+  };
+
+  nat64_interfaces_walk (nat64_api_interface_walk, &ctx);
+}
+
+static void *
+vl_api_nat64_interface_dump_t_print (vl_api_nat64_interface_dump_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_interface_dump ");
+
+  FINISH;
+}
+
+static void
+  vl_api_nat64_add_del_static_bib_t_handler
+  (vl_api_nat64_add_del_static_bib_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  nat64_main_t *nm = &nat64_main;
+  vl_api_nat64_add_del_static_bib_reply_t *rmp;
+  ip6_address_t in_addr;
+  ip4_address_t out_addr;
+  int rv = 0;
+
+  if (nm->is_disabled)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  memcpy (&in_addr.as_u8, mp->i_addr, 16);
+  memcpy (&out_addr.as_u8, mp->o_addr, 4);
+
+  rv =
+    nat64_add_del_static_bib_entry (&in_addr, &out_addr,
+				    clib_net_to_host_u16 (mp->i_port),
+				    clib_net_to_host_u16 (mp->o_port),
+				    mp->proto,
+				    clib_net_to_host_u32 (mp->vrf_id),
+				    mp->is_add);
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT64_ADD_DEL_STATIC_BIB_REPLY);
+}
+
+static void *vl_api_nat64_add_del_static_bib_t_print
+  (vl_api_nat64_add_del_static_bib_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_add_del_static_bib ");
+  s = format (s, "protocol %d i_addr %U o_addr %U ",
+	      mp->proto,
+	      format_ip6_address, mp->i_addr, format_ip4_address, mp->o_addr);
+
+  if (mp->vrf_id != ~0)
+    s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id));
+
+  FINISH;
+}
+
+static int
+nat64_api_bib_walk (nat64_db_bib_entry_t * bibe, void *arg)
+{
+  vl_api_nat64_bib_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_api_walk_ctx_t *ctx = arg;
+  fib_table_t *fib;
+
+  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
+  if (!fib)
+    return -1;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT64_BIB_DETAILS + sm->msg_id_base);
+  rmp->context = ctx->context;
+  clib_memcpy (rmp->i_addr, &(bibe->in_addr), 16);
+  clib_memcpy (rmp->o_addr, &(bibe->out_addr), 4);
+  rmp->i_port = bibe->in_port;
+  rmp->o_port = bibe->out_port;
+  rmp->vrf_id = ntohl (fib->ft_table_id);
+  rmp->proto = bibe->proto;
+  rmp->is_static = bibe->is_static;
+  rmp->ses_num = ntohl (bibe->ses_num);
+
+  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
+
+  return 0;
+}
+
+static void
+vl_api_nat64_bib_dump_t_handler (vl_api_nat64_bib_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  nat64_api_walk_ctx_t ctx = {
+    .q = q,
+    .context = mp->context,
+  };
+
+  nat64_db_bib_walk (&nm->db, mp->proto, nat64_api_bib_walk, &ctx);
+}
+
+static void *
+vl_api_nat64_bib_dump_t_print (vl_api_nat64_bib_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_bib_dump protocol %d", mp->proto);
+
+  FINISH;
+}
+
+static void
+vl_api_nat64_set_timeouts_t_handler (vl_api_nat64_set_timeouts_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  nat64_main_t *nm = &nat64_main;
+  vl_api_nat64_set_timeouts_reply_t *rmp;
+  int rv = 0;
+
+  if (nm->is_disabled)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  rv = nat64_set_icmp_timeout (ntohl (mp->icmp));
+  if (rv)
+    goto send_reply;
+  rv = nat64_set_udp_timeout (ntohl (mp->udp));
+  if (rv)
+    goto send_reply;
+  rv =
+    nat64_set_tcp_timeouts (ntohl (mp->tcp_trans), ntohl (mp->tcp_est),
+			    ntohl (mp->tcp_incoming_syn));
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT64_SET_TIMEOUTS_REPLY);
+}
+
+static void *vl_api_nat64_set_timeouts_t_print
+  (vl_api_nat64_set_timeouts_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_set_timeouts ");
+  s =
+    format (s,
+	    "udp %d icmp %d, tcp_trans %d, tcp_est %d, tcp_incoming_syn %d\n",
+	    ntohl (mp->udp), ntohl (mp->icmp), ntohl (mp->tcp_trans),
+	    ntohl (mp->tcp_est), ntohl (mp->tcp_incoming_syn));
+
+  FINISH;
+}
+
+static void
+vl_api_nat64_get_timeouts_t_handler (vl_api_nat64_get_timeouts_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  nat64_main_t *nm = &nat64_main;
+  vl_api_nat64_get_timeouts_reply_t *rmp;
+  int rv = 0;
+
+  if (nm->is_disabled)
+    return;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_NAT64_GET_TIMEOUTS_REPLY,
+  ({
+    rmp->udp = htonl (nat64_get_udp_timeout());
+    rmp->icmp = htonl (nat64_get_icmp_timeout());
+    rmp->tcp_trans = htonl (nat64_get_tcp_trans_timeout());
+    rmp->tcp_est = htonl (nat64_get_tcp_est_timeout());
+    rmp->tcp_incoming_syn = htonl (nat64_get_tcp_incoming_syn_timeout());
+  }))
+  /* *INDENT-ON* */
+}
+
+static void *vl_api_nat64_get_timeouts_t_print
+  (vl_api_nat64_get_timeouts_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_get_timeouts");
+
+  FINISH;
+}
+
+static int
+nat64_api_st_walk (nat64_db_st_entry_t * ste, void *arg)
+{
+  vl_api_nat64_st_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_api_walk_ctx_t *ctx = arg;
+  nat64_main_t *nm = &nat64_main;
+  nat64_db_bib_entry_t *bibe;
+  fib_table_t *fib;
+
+  bibe = nat64_db_bib_entry_by_index (&nm->db, ste->proto, ste->bibe_index);
+  if (!bibe)
+    return -1;
+
+  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
+  if (!fib)
+    return -1;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT64_ST_DETAILS + sm->msg_id_base);
+  rmp->context = ctx->context;
+  clib_memcpy (rmp->il_addr, &(bibe->in_addr), 16);
+  clib_memcpy (rmp->ol_addr, &(bibe->out_addr), 4);
+  rmp->il_port = bibe->in_port;
+  rmp->ol_port = bibe->out_port;
+  clib_memcpy (rmp->ir_addr, &(ste->in_r_addr), 16);
+  clib_memcpy (rmp->or_addr, &(ste->out_r_addr), 4);
+  rmp->il_port = ste->r_port;
+  rmp->vrf_id = ntohl (fib->ft_table_id);
+  rmp->proto = ste->proto;
+
+  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
+
+  return 0;
+}
+
+static void
+vl_api_nat64_st_dump_t_handler (vl_api_nat64_st_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  nat64_api_walk_ctx_t ctx = {
+    .q = q,
+    .context = mp->context,
+  };
+
+  nat64_db_st_walk (&nm->db, mp->proto, nat64_api_st_walk, &ctx);
+}
+
+static void *
+vl_api_nat64_st_dump_t_print (vl_api_nat64_st_dump_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: snat_st_dump protocol %d", mp->proto);
+
+  FINISH;
+}
+
+static void
+vl_api_nat64_add_del_prefix_t_handler (vl_api_nat64_add_del_prefix_t * mp)
+{
+  vl_api_nat64_add_del_prefix_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_main_t *nm = &nat64_main;
+  ip6_address_t prefix;
+  int rv = 0;
+
+  if (nm->is_disabled)
+    {
+      rv = VNET_API_ERROR_FEATURE_DISABLED;
+      goto send_reply;
+    }
+
+  memcpy (&prefix.as_u8, mp->prefix, 16);
+
+  rv =
+    nat64_add_del_prefix (&prefix, mp->prefix_len,
+			  clib_net_to_host_u32 (mp->vrf_id), mp->is_add);
+send_reply:
+  REPLY_MACRO (VL_API_NAT64_ADD_DEL_PREFIX_REPLY);
+}
+
+static void *
+vl_api_nat64_add_del_prefix_t_print (vl_api_nat64_add_del_prefix_t * mp,
+				     void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_add_del_prefix %U/%u vrf_id %u %s\n",
+	      format_ip6_address, mp->prefix, mp->prefix_len,
+	      ntohl (mp->vrf_id), mp->is_add ? "" : "del");
+
+  FINISH;
+}
+
+static int
+nat64_api_prefix_walk (nat64_prefix_t * p, void *arg)
+{
+  vl_api_nat64_prefix_details_t *rmp;
+  snat_main_t *sm = &snat_main;
+  nat64_api_walk_ctx_t *ctx = arg;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  memset (rmp, 0, sizeof (*rmp));
+  rmp->_vl_msg_id = ntohs (VL_API_NAT64_PREFIX_DETAILS + sm->msg_id_base);
+  clib_memcpy (rmp->prefix, &(p->prefix), 16);
+  rmp->prefix_len = p->plen;
+  rmp->vrf_id = ntohl (p->vrf_id);
+  rmp->context = ctx->context;
+
+  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
+
+  return 0;
+}
+
+static void
+vl_api_nat64_prefix_dump_t_handler (vl_api_nat64_prefix_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+  nat64_main_t *nm = &nat64_main;
+
+  if (nm->is_disabled)
+    return;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    return;
+
+  nat64_api_walk_ctx_t ctx = {
+    .q = q,
+    .context = mp->context,
+  };
+
+  nat64_prefix_walk (nat64_api_prefix_walk, &ctx);
+}
+
+static void *
+vl_api_nat64_prefix_dump_t_print (vl_api_nat64_prefix_dump_t * mp,
+				  void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat64_prefix_dump\n");
+
+  FINISH;
+}
+
+/* List of message types that this plugin understands */
+#define foreach_snat_plugin_api_msg                                     \
+_(SNAT_ADD_ADDRESS_RANGE, snat_add_address_range)                       \
+_(SNAT_INTERFACE_ADD_DEL_FEATURE, snat_interface_add_del_feature)       \
+_(SNAT_ADD_STATIC_MAPPING, snat_add_static_mapping)                     \
+_(SNAT_CONTROL_PING, snat_control_ping)                                 \
+_(SNAT_STATIC_MAPPING_DUMP, snat_static_mapping_dump)                   \
+_(SNAT_SHOW_CONFIG, snat_show_config)                                   \
+_(SNAT_ADDRESS_DUMP, snat_address_dump)                                 \
+_(SNAT_INTERFACE_DUMP, snat_interface_dump)                             \
+_(SNAT_SET_WORKERS, snat_set_workers)                                   \
+_(SNAT_WORKER_DUMP, snat_worker_dump)                                   \
+_(SNAT_ADD_DEL_INTERFACE_ADDR, snat_add_del_interface_addr)             \
+_(SNAT_INTERFACE_ADDR_DUMP, snat_interface_addr_dump)                   \
+_(SNAT_IPFIX_ENABLE_DISABLE, snat_ipfix_enable_disable)                 \
+_(SNAT_USER_DUMP, snat_user_dump)                                       \
+_(SNAT_USER_SESSION_DUMP, snat_user_session_dump)                       \
+_(SNAT_INTERFACE_ADD_DEL_OUTPUT_FEATURE,                                \
+  snat_interface_add_del_output_feature)                                \
+_(SNAT_INTERFACE_OUTPUT_FEATURE_DUMP,                                   \
+  snat_interface_output_feature_dump)                                   \
+_(SNAT_ADD_DET_MAP, snat_add_det_map)                                   \
+_(SNAT_DET_FORWARD, snat_det_forward)                                   \
+_(SNAT_DET_REVERSE, snat_det_reverse)                                   \
+_(SNAT_DET_MAP_DUMP, snat_det_map_dump)                                 \
+_(SNAT_DET_SET_TIMEOUTS, snat_det_set_timeouts)                         \
+_(SNAT_DET_GET_TIMEOUTS, snat_det_get_timeouts)                         \
+_(SNAT_DET_CLOSE_SESSION_OUT, snat_det_close_session_out)               \
+_(SNAT_DET_CLOSE_SESSION_IN, snat_det_close_session_in)                 \
+_(SNAT_DET_SESSION_DUMP, snat_det_session_dump)                         \
+_(NAT_CONTROL_PING, nat_control_ping)                                   \
+_(NAT_SHOW_CONFIG, nat_show_config)                                     \
+_(NAT_SET_WORKERS, nat_set_workers)                                     \
+_(NAT_WORKER_DUMP, nat_worker_dump)                                     \
+_(NAT_IPFIX_ENABLE_DISABLE, nat_ipfix_enable_disable)                   \
+_(NAT44_ADD_DEL_ADDRESS_RANGE, nat44_add_del_address_range)             \
+_(NAT44_INTERFACE_ADD_DEL_FEATURE, nat44_interface_add_del_feature)     \
+_(NAT44_ADD_DEL_STATIC_MAPPING, nat44_add_del_static_mapping)           \
+_(NAT44_STATIC_MAPPING_DUMP, nat44_static_mapping_dump)                 \
+_(NAT44_ADDRESS_DUMP, nat44_address_dump)                               \
+_(NAT44_INTERFACE_DUMP, nat44_interface_dump)                           \
+_(NAT44_ADD_DEL_INTERFACE_ADDR, nat44_add_del_interface_addr)           \
+_(NAT44_INTERFACE_ADDR_DUMP, nat44_interface_addr_dump)                 \
+_(NAT44_USER_DUMP, nat44_user_dump)                                     \
+_(NAT44_USER_SESSION_DUMP, nat44_user_session_dump)                     \
+_(NAT44_INTERFACE_ADD_DEL_OUTPUT_FEATURE,                               \
+  nat44_interface_add_del_output_feature)                               \
+_(NAT44_INTERFACE_OUTPUT_FEATURE_DUMP,                                  \
+  nat44_interface_output_feature_dump)                                  \
+_(NAT_DET_ADD_DEL_MAP, nat_det_add_del_map)                             \
+_(NAT_DET_FORWARD, nat_det_forward)                                     \
+_(NAT_DET_REVERSE, nat_det_reverse)                                     \
+_(NAT_DET_MAP_DUMP, nat_det_map_dump)                                   \
+_(NAT_DET_SET_TIMEOUTS, nat_det_set_timeouts)                           \
+_(NAT_DET_GET_TIMEOUTS, nat_det_get_timeouts)                           \
+_(NAT_DET_CLOSE_SESSION_OUT, nat_det_close_session_out)                 \
+_(NAT_DET_CLOSE_SESSION_IN, nat_det_close_session_in)                   \
+_(NAT_DET_SESSION_DUMP, nat_det_session_dump)                           \
+_(NAT64_ADD_DEL_POOL_ADDR_RANGE, nat64_add_del_pool_addr_range)         \
+_(NAT64_POOL_ADDR_DUMP, nat64_pool_addr_dump)                           \
+_(NAT64_ADD_DEL_INTERFACE, nat64_add_del_interface)                     \
+_(NAT64_INTERFACE_DUMP, nat64_interface_dump)                           \
+_(NAT64_ADD_DEL_STATIC_BIB, nat64_add_del_static_bib)                   \
+_(NAT64_BIB_DUMP, nat64_bib_dump)                                       \
+_(NAT64_SET_TIMEOUTS, nat64_set_timeouts)                               \
+_(NAT64_GET_TIMEOUTS, nat64_get_timeouts)                               \
+_(NAT64_ST_DUMP, nat64_st_dump)                                         \
+_(NAT64_ADD_DEL_PREFIX, nat64_add_del_prefix)                           \
+_(NAT64_PREFIX_DUMP, nat64_prefix_dump)
+
+/* Set up the API message handling tables */
+static clib_error_t *
+snat_plugin_api_hookup (vlib_main_t * vm)
+{
+  snat_main_t *sm __attribute__ ((unused)) = &snat_main;
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),     \
+                           #n,					\
+                           vl_api_##n##_t_handler,              \
+                           vl_noop_handler,                     \
+                           vl_api_##n##_t_endian,               \
+                           vl_api_##n##_t_print,                \
+                           sizeof(vl_api_##n##_t), 1);
+  foreach_snat_plugin_api_msg;
+#undef _
+
+  return 0;
+}
+
+#define vl_msg_name_crc_list
+#include 
+#undef vl_msg_name_crc_list
+
+static void
+setup_message_id_table (snat_main_t * sm, api_main_t * am)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base);
+  foreach_vl_msg_name_crc_nat;
+#undef _
+}
+
+static void
+plugin_custom_dump_configure (snat_main_t * sm)
+{
+#define _(n,f) sm->api_main->msg_print_handlers \
+  [VL_API_##n + sm->msg_id_base]                \
+    = (void *) vl_api_##f##_t_print;
+  foreach_snat_plugin_api_msg;
+#undef _
+}
+
+clib_error_t *
+snat_api_init (vlib_main_t * vm, snat_main_t * sm)
+{
+  u8 *name;
+  clib_error_t *error = 0;
+
+  name = format (0, "snat_%08x%c", api_version, 0);
+
+  /* Ask for a correctly-sized block of API message decode slots */
+  sm->msg_id_base =
+    vl_msg_api_get_msg_ids ((char *) name, VL_MSG_FIRST_AVAILABLE);
+
+  error = snat_plugin_api_hookup (vm);
+
+  /* Add our API messages to the global name_crc hash table */
+  setup_message_id_table (sm, sm->api_main);
+
+  plugin_custom_dump_configure (sm);
+
+  vec_free (name);
+
+  return error;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat_det.c b/src/plugins/nat/nat_det.c
new file mode 100644
index 00000000..3af6698c
--- /dev/null
+++ b/src/plugins/nat/nat_det.c
@@ -0,0 +1,158 @@
+/*
+ * snat_det.c - deterministic NAT
+ *
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief deterministic NAT
+ */
+
+#include 
+
+
+/**
+ * @brief Add/delete deterministic NAT mapping.
+ *
+ * Create bijective mapping of inside address to outside address and port range
+ * pairs, with the purpose of enabling deterministic NAT to reduce logging in
+ * CGN deployments.
+ *
+ * @param sm       SNAT main.
+ * @param in_addr  Inside network address.
+ * @param in_plen  Inside network prefix length.
+ * @param out_addr Outside network address.
+ * @param out_plen Outside network prefix length.
+ * @param is_add   If 0 delete, otherwise add.
+ */
+int
+snat_det_add_map (snat_main_t * sm, ip4_address_t * in_addr, u8 in_plen,
+		  ip4_address_t * out_addr, u8 out_plen, int is_add)
+{
+  snat_det_map_t *det_map;
+  static snat_det_session_t empty_snat_det_session = { 0 };
+  snat_interface_t *i;
+  ip4_address_t in_cmp, out_cmp;
+  u8 found = 0;
+
+  in_cmp.as_u32 = in_addr->as_u32 & ip4_main.fib_masks[in_plen];
+  out_cmp.as_u32 = out_addr->as_u32 & ip4_main.fib_masks[out_plen];
+  vec_foreach (det_map, sm->det_maps)
+  {
+    /* Checking for overlapping addresses to be added here */
+    if (det_map->in_addr.as_u32 == in_cmp.as_u32 &&
+	det_map->in_plen == in_plen &&
+	det_map->out_addr.as_u32 == out_cmp.as_u32 &&
+	det_map->out_plen == out_plen)
+      {
+	found = 1;
+	break;
+      }
+  }
+
+  /* If found, don't add again */
+  if (found && is_add)
+    return VNET_API_ERROR_VALUE_EXIST;
+
+  /* If not found, don't delete */
+  if (!found && !is_add)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  if (is_add)
+    {
+      pool_get (sm->det_maps, det_map);
+      memset (det_map, 0, sizeof (*det_map));
+      det_map->in_addr.as_u32 = in_cmp.as_u32;
+      det_map->in_plen = in_plen;
+      det_map->out_addr.as_u32 = out_cmp.as_u32;
+      det_map->out_plen = out_plen;
+      det_map->sharing_ratio = (1 << (32 - in_plen)) / (1 << (32 - out_plen));
+      det_map->ports_per_host = (65535 - 1023) / det_map->sharing_ratio;
+
+      vec_validate_init_empty (det_map->sessions,
+			       SNAT_DET_SES_PER_USER * (1 << (32 - in_plen)) -
+			       1, empty_snat_det_session);
+    }
+  else
+    {
+      vec_free (det_map->sessions);
+      vec_del1 (sm->det_maps, det_map - sm->det_maps);
+    }
+
+  /* Add/del external address range to FIB */
+  /* *INDENT-OFF* */
+  pool_foreach (i, sm->interfaces,
+  ({
+    if (i->is_inside)
+      continue;
+
+    snat_add_del_addr_to_fib(out_addr, out_plen, i->sw_if_index, is_add);
+    break;
+  }));
+  /* *INDENT-ON* */
+  return 0;
+}
+
+/**
+ * @brief The 'nat-det-expire-walk' process's main loop.
+ *
+ * Check expire time for active sessions.
+ */
+static uword
+snat_det_expire_walk_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
+			 vlib_frame_t * f)
+{
+  snat_main_t *sm = &snat_main;
+  snat_det_map_t *dm;
+  snat_det_session_t *ses;
+
+  while (sm->deterministic)
+    {
+      vlib_process_wait_for_event_or_clock (vm, 10.0);
+      vlib_process_get_events (vm, NULL);
+      u32 now = (u32) vlib_time_now (vm);
+      /* *INDENT-OFF* */
+      pool_foreach (dm, sm->det_maps,
+      ({
+        vec_foreach(ses, dm->sessions)
+          {
+            /* Delete if session expired */
+            if (ses->in_port && (ses->expire < now))
+              snat_det_ses_close (dm, ses);
+          }
+      }));
+      /* *INDENT-ON* */
+    }
+
+  return 0;
+}
+
+static vlib_node_registration_t snat_det_expire_walk_node;
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (snat_det_expire_walk_node, static) = {
+    .function = snat_det_expire_walk_fn,
+    .type = VLIB_NODE_TYPE_PROCESS,
+    .name =
+    "nat-det-expire-walk",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat_det.h b/src/plugins/nat/nat_det.h
new file mode 100644
index 00000000..2ab7f27e
--- /dev/null
+++ b/src/plugins/nat/nat_det.h
@@ -0,0 +1,196 @@
+/*
+ * snat_det.h - deterministic NAT definitions
+ *
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file
+ * @brief deterministic NAT definitions
+ */
+
+#ifndef __included_nat_det_h__
+#define __included_nat_det_h__
+
+#include 
+#include 
+#include 
+
+
+#define SNAT_DET_SES_PER_USER 1000
+
+
+int snat_det_add_map (snat_main_t * sm, ip4_address_t * in_addr, u8 in_plen,
+		      ip4_address_t * out_addr, u8 out_plen, int is_add);
+
+always_inline int
+is_addr_in_net (ip4_address_t * addr, ip4_address_t * net, u8 plen)
+{
+  if (net->as_u32 == (addr->as_u32 & ip4_main.fib_masks[plen]))
+    return 1;
+  return 0;
+}
+
+always_inline snat_det_map_t *
+snat_det_map_by_user (snat_main_t * sm, ip4_address_t * user_addr)
+{
+  snat_det_map_t *dm;
+
+  /* *INDENT-OFF* */
+  pool_foreach (dm, sm->det_maps,
+  ({
+    if (is_addr_in_net(user_addr, &dm->in_addr, dm->in_plen))
+      return dm;
+  }));
+  /* *INDENT-ON* */
+  return 0;
+}
+
+always_inline snat_det_map_t *
+snat_det_map_by_out (snat_main_t * sm, ip4_address_t * out_addr)
+{
+  snat_det_map_t *dm;
+
+  /* *INDENT-OFF* */
+  pool_foreach (dm, sm->det_maps,
+  ({
+    if (is_addr_in_net(out_addr, &dm->out_addr, dm->out_plen))
+      return dm;
+  }));
+  /* *INDENT-ON* */
+  return 0;
+}
+
+always_inline void
+snat_det_forward (snat_det_map_t * dm, ip4_address_t * in_addr,
+		  ip4_address_t * out_addr, u16 * lo_port)
+{
+  u32 in_offset, out_offset;
+
+  in_offset = clib_net_to_host_u32 (in_addr->as_u32) -
+    clib_net_to_host_u32 (dm->in_addr.as_u32);
+  out_offset = in_offset / dm->sharing_ratio;
+  out_addr->as_u32 =
+    clib_host_to_net_u32 (clib_net_to_host_u32 (dm->out_addr.as_u32) +
+			  out_offset);
+  *lo_port = 1024 + dm->ports_per_host * (in_offset % dm->sharing_ratio);
+}
+
+always_inline void
+snat_det_reverse (snat_det_map_t * dm, ip4_address_t * out_addr, u16 out_port,
+		  ip4_address_t * in_addr)
+{
+  u32 in_offset1, in_offset2, out_offset;
+
+  out_offset = clib_net_to_host_u32 (out_addr->as_u32) -
+    clib_net_to_host_u32 (dm->out_addr.as_u32);
+  in_offset1 = out_offset * dm->sharing_ratio;
+  in_offset2 = (out_port - 1024) / dm->ports_per_host;
+  in_addr->as_u32 =
+    clib_host_to_net_u32 (clib_net_to_host_u32 (dm->in_addr.as_u32) +
+			  in_offset1 + in_offset2);
+}
+
+always_inline u32
+snat_det_user_ses_offset (ip4_address_t * addr, u8 plen)
+{
+  return (clib_net_to_host_u32 (addr->as_u32) & pow2_mask (32 - plen)) *
+    SNAT_DET_SES_PER_USER;
+}
+
+always_inline snat_det_session_t *
+snat_det_get_ses_by_out (snat_det_map_t * dm, ip4_address_t * in_addr,
+			 u64 out_key)
+{
+  u32 user_offset;
+  u16 i;
+
+  user_offset = snat_det_user_ses_offset (in_addr, dm->in_plen);
+  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
+    {
+      if (dm->sessions[i + user_offset].out.as_u64 == out_key)
+	return &dm->sessions[i + user_offset];
+    }
+
+  return 0;
+}
+
+always_inline snat_det_session_t *
+snat_det_find_ses_by_in (snat_det_map_t * dm, ip4_address_t * in_addr,
+			 u16 in_port, snat_det_out_key_t out_key)
+{
+  snat_det_session_t *ses;
+  u32 user_offset;
+  u16 i;
+
+  user_offset = snat_det_user_ses_offset (in_addr, dm->in_plen);
+  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
+    {
+      ses = &dm->sessions[i + user_offset];
+      if (ses->in_port == in_port &&
+	  ses->out.ext_host_addr.as_u32 == out_key.ext_host_addr.as_u32 &&
+	  ses->out.ext_host_port == out_key.ext_host_port)
+	return &dm->sessions[i + user_offset];
+    }
+
+  return 0;
+}
+
+always_inline snat_det_session_t *
+snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
+		     u16 in_port, snat_det_out_key_t * out)
+{
+  u32 user_offset;
+  u16 i;
+
+  user_offset = snat_det_user_ses_offset (in_addr, dm->in_plen);
+
+  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
+    {
+      if (!dm->sessions[i + user_offset].in_port)
+	{
+	  if (__sync_bool_compare_and_swap
+	      (&dm->sessions[i + user_offset].in_port, 0, in_port))
+	    {
+	      dm->sessions[i + user_offset].out.as_u64 = out->as_u64;
+	      dm->sessions[i + user_offset].state = SNAT_SESSION_UNKNOWN;
+	      dm->sessions[i + user_offset].expire = 0;
+	      __sync_add_and_fetch (&dm->ses_num, 1);
+	      return &dm->sessions[i + user_offset];
+	    }
+	}
+    }
+
+  snat_ipfix_logging_max_entries_per_user (in_addr->as_u32);
+  return 0;
+}
+
+always_inline void
+snat_det_ses_close (snat_det_map_t * dm, snat_det_session_t * ses)
+{
+  if (__sync_bool_compare_and_swap (&ses->in_port, ses->in_port, 0))
+    {
+      ses->out.as_u64 = 0;
+      __sync_add_and_fetch (&dm->ses_num, -1);
+    }
+}
+
+#endif /* __included_nat_det_h__ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/nat/nat_ipfix_logging.c b/src/plugins/nat/nat_ipfix_logging.c
new file mode 100644
index 00000000..0cc0f82a
--- /dev/null
+++ b/src/plugins/nat/nat_ipfix_logging.c
@@ -0,0 +1,835 @@
+/*
+ * nat_ipfix_logging.c - NAT Events IPFIX logging
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+snat_ipfix_logging_main_t snat_ipfix_logging_main;
+
+#define NAT44_SESSION_CREATE_LEN 26
+#define NAT_ADDRESSES_EXHAUTED_LEN 13
+#define MAX_ENTRIES_PER_USER_LEN 17
+
+#define NAT44_SESSION_CREATE_FIELD_COUNT 8
+#define NAT_ADDRESSES_EXHAUTED_FIELD_COUNT 3
+#define MAX_ENTRIES_PER_USER_FIELD_COUNT 4
+
+typedef struct {
+  u8 nat_event;
+  u32 src_ip;
+  u32 nat_src_ip;
+  snat_protocol_t snat_proto;
+  u16 src_port;
+  u16 nat_src_port;
+  u32 vrf_id;
+} snat_ipfix_logging_nat44_ses_args_t;
+
+typedef struct {
+  u32 pool_id;
+} snat_ipfix_logging_addr_exhausted_args_t;
+
+typedef struct {
+  u32 src_ip;
+} snat_ipfix_logging_max_entries_per_user_args_t;
+
+/**
+ * @brief Create an IPFIX template packet rewrite string
+ *
+ * @param frm               flow report main
+ * @param fr                flow report
+ * @param collector_address collector address
+ * @param src_address       source address
+ * @param collector_port    collector
+ * @param event             NAT event ID
+ * @param quota_event       NAT quota exceeded event ID
+ *
+ * @returns template packet
+ */
+static inline u8 *
+snat_template_rewrite (flow_report_main_t * frm,
+                       flow_report_t * fr,
+                       ip4_address_t * collector_address,
+                       ip4_address_t * src_address,
+                       u16 collector_port,
+                       nat_event_t event,
+                       quota_exceed_event_t quota_event)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  ip4_header_t *ip;
+  udp_header_t *udp;
+  ipfix_message_header_t *h;
+  ipfix_set_header_t *s;
+  ipfix_template_header_t *t;
+  ipfix_field_specifier_t *f;
+  ipfix_field_specifier_t *first_field;
+  u8 *rewrite = 0;
+  ip4_ipfix_template_packet_t *tp;
+  u32 field_count = 0;
+  flow_report_stream_t *stream;
+
+  stream = &frm->streams[fr->stream_index];
+  silm->stream_index = fr->stream_index;
+
+  if (event == NAT_ADDRESSES_EXHAUTED)
+    {
+      field_count = NAT_ADDRESSES_EXHAUTED_FIELD_COUNT;
+      silm->addr_exhausted_template_id = fr->template_id;
+    }
+  else if (event == NAT44_SESSION_CREATE)
+    {
+      field_count = NAT44_SESSION_CREATE_FIELD_COUNT;
+      silm->nat44_session_template_id = fr->template_id;
+    }
+  else if (event == QUOTA_EXCEEDED)
+    {
+      if (quota_event == MAX_ENTRIES_PER_USER)
+        {
+          field_count = MAX_ENTRIES_PER_USER_FIELD_COUNT;
+          silm->max_entries_per_user_template_id = fr->template_id;
+        }
+    }
+
+  /* allocate rewrite space */
+  vec_validate_aligned (rewrite,
+			sizeof (ip4_ipfix_template_packet_t)
+			+ field_count * sizeof (ipfix_field_specifier_t) - 1,
+			CLIB_CACHE_LINE_BYTES);
+
+  tp = (ip4_ipfix_template_packet_t *) rewrite;
+  ip = (ip4_header_t *) & tp->ip4;
+  udp = (udp_header_t *) (ip + 1);
+  h = (ipfix_message_header_t *) (udp + 1);
+  s = (ipfix_set_header_t *) (h + 1);
+  t = (ipfix_template_header_t *) (s + 1);
+  first_field = f = (ipfix_field_specifier_t *) (t + 1);
+
+  ip->ip_version_and_header_length = 0x45;
+  ip->ttl = 254;
+  ip->protocol = IP_PROTOCOL_UDP;
+  ip->src_address.as_u32 = src_address->as_u32;
+  ip->dst_address.as_u32 = collector_address->as_u32;
+  udp->src_port = clib_host_to_net_u16 (stream->src_port);
+  udp->dst_port = clib_host_to_net_u16 (collector_port);
+  udp->length = clib_host_to_net_u16 (vec_len (rewrite) - sizeof (*ip));
+
+  /* FIXUP: message header export_time */
+  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
+
+  /* Add TLVs to the template */
+  if (event == NAT_ADDRESSES_EXHAUTED)
+    {
+      f->e_id_length = ipfix_e_id_length (0, observationTimeMilliseconds, 8);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, natEvent, 1);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, natPoolId, 4);
+      f++;
+    }
+  else if (event == NAT44_SESSION_CREATE)
+    {
+      f->e_id_length = ipfix_e_id_length (0, observationTimeMilliseconds, 8);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, natEvent, 1);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, sourceIPv4Address, 4);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, postNATSourceIPv4Address, 4);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, protocolIdentifier, 1);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, sourceTransportPort, 2);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, postNAPTSourceTransportPort, 2);
+      f++;
+      f->e_id_length = ipfix_e_id_length (0, ingressVRFID, 4);
+      f++;
+    }
+  else if (event == QUOTA_EXCEEDED)
+    {
+      if (quota_event == MAX_ENTRIES_PER_USER)
+        {
+          f->e_id_length = ipfix_e_id_length (0, observationTimeMilliseconds,
+                                              8);
+          f++;
+          f->e_id_length = ipfix_e_id_length (0, natEvent, 1);
+          f++;
+          f->e_id_length = ipfix_e_id_length (0, natQuotaExceededEvent, 4);
+          f++;
+          f->e_id_length = ipfix_e_id_length (0, sourceIPv4Address, 4);
+          f++;
+        }
+    }
+
+  /* Back to the template packet... */
+  ip = (ip4_header_t *) & tp->ip4;
+  udp = (udp_header_t *) (ip + 1);
+
+  ASSERT (f - first_field);
+  /* Field count in this template */
+  t->id_count = ipfix_id_count (fr->template_id, f - first_field);
+
+  /* set length in octets */
+  s->set_id_length =
+    ipfix_set_id_length (2 /* set_id */ , (u8 *) f - (u8 *) s);
+
+  /* message length in octets */
+  h->version_length = version_length ((u8 *) f - (u8 *) h);
+
+  ip->length = clib_host_to_net_u16 ((u8 *) f - (u8 *) ip);
+  ip->checksum = ip4_header_checksum (ip);
+
+  return rewrite;
+}
+
+u8 *
+snat_template_rewrite_addr_exhausted (flow_report_main_t * frm,
+                                      flow_report_t * fr,
+                                      ip4_address_t * collector_address,
+                                      ip4_address_t * src_address,
+                                      u16 collector_port)
+{
+  return snat_template_rewrite (frm, fr, collector_address, src_address,
+                                collector_port, NAT_ADDRESSES_EXHAUTED, 0);
+}
+
+u8 *
+snat_template_rewrite_nat44_session (flow_report_main_t * frm,
+                                     flow_report_t * fr,
+                                     ip4_address_t * collector_address,
+                                     ip4_address_t * src_address,
+                                     u16 collector_port)
+{
+  return snat_template_rewrite (frm, fr, collector_address, src_address,
+                                collector_port, NAT44_SESSION_CREATE, 0);
+}
+
+u8 *
+snat_template_rewrite_max_entries_per_usr (flow_report_main_t * frm,
+                                           flow_report_t * fr,
+                                           ip4_address_t * collector_address,
+                                           ip4_address_t * src_address,
+                                           u16 collector_port)
+{
+  return snat_template_rewrite (frm, fr, collector_address, src_address,
+                                collector_port, QUOTA_EXCEEDED,
+                                MAX_ENTRIES_PER_USER);
+}
+
+static inline void
+snat_ipfix_header_create (flow_report_main_t * frm,
+                          vlib_buffer_t * b0,
+                          u32 * offset)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  flow_report_stream_t *stream;
+  ip4_ipfix_template_packet_t * tp;
+  ipfix_message_header_t * h = 0;
+  ipfix_set_header_t * s = 0;
+  ip4_header_t * ip;
+  udp_header_t * udp;
+
+  stream = &frm->streams[silm->stream_index];
+
+  b0->current_data = 0;
+  b0->current_length = sizeof (*ip) + sizeof (*udp) + sizeof (*h) +
+                       sizeof (*s);
+  b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_FLOW_REPORT);
+  vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
+  vnet_buffer (b0)->sw_if_index[VLIB_TX] = frm->fib_index;
+  tp = vlib_buffer_get_current (b0);
+  ip = (ip4_header_t *) &tp->ip4;
+  udp = (udp_header_t *) (ip+1);
+  h = (ipfix_message_header_t *)(udp+1);
+  s = (ipfix_set_header_t *)(h+1);
+
+  ip->ip_version_and_header_length = 0x45;
+  ip->ttl = 254;
+  ip->protocol = IP_PROTOCOL_UDP;
+  ip->flags_and_fragment_offset = 0;
+  ip->src_address.as_u32 = frm->src_address.as_u32;
+  ip->dst_address.as_u32 = frm->ipfix_collector.as_u32;
+  udp->src_port = clib_host_to_net_u16 (stream->src_port);
+  udp->dst_port = clib_host_to_net_u16 (frm->collector_port);
+  udp->checksum = 0;
+
+  h->export_time = clib_host_to_net_u32 (
+    (u32) (((f64)frm->unix_time_0) + (vlib_time_now(frm->vlib_main) -
+    frm->vlib_time_0)));
+  h->sequence_number = clib_host_to_net_u32 (stream->sequence_number++);
+  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
+
+  *offset = (u32) (((u8 *)(s+1)) - (u8 *)tp);
+}
+
+static inline void
+snat_ipfix_send (flow_report_main_t * frm,
+                 vlib_frame_t * f,
+                 vlib_buffer_t * b0,
+                 u16 template_id)
+{
+  ip4_ipfix_template_packet_t * tp;
+  ipfix_message_header_t * h = 0;
+  ipfix_set_header_t * s = 0;
+  ip4_header_t * ip;
+  udp_header_t * udp;
+  vlib_main_t * vm = frm->vlib_main;
+
+  tp = vlib_buffer_get_current (b0);
+  ip = (ip4_header_t *) & tp->ip4;
+  udp = (udp_header_t *) (ip + 1);
+  h = (ipfix_message_header_t *) (udp + 1);
+  s = (ipfix_set_header_t *) (h + 1);
+
+  s->set_id_length = ipfix_set_id_length (template_id,
+                                          b0->current_length -
+                                          (sizeof (*ip) + sizeof (*udp) +
+                                           sizeof (*h)));
+  h->version_length = version_length (b0->current_length -
+                                      (sizeof (*ip) + sizeof (*udp)));
+
+  ip->length = clib_host_to_net_u16 (b0->current_length);
+  ip->checksum = ip4_header_checksum (ip);
+  udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
+
+  if (frm->udp_checksum)
+    {
+      udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
+      if (udp->checksum == 0)
+        udp->checksum = 0xffff;
+    }
+
+  ASSERT (ip->checksum == ip4_header_checksum (ip));
+
+  vlib_put_frame_to_node (vm, ip4_lookup_node.index, f);
+}
+
+static void
+snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip,
+                              snat_protocol_t snat_proto, u16 src_port,
+                              u16 nat_src_port, u32 vrf_id, int do_flush)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vlib_frame_t *f;
+  vlib_buffer_t *b0 = 0;
+  u32 bi0 = ~0;
+  u32 offset;
+  vlib_main_t * vm = frm->vlib_main;
+  u64 now;
+  vlib_buffer_free_list_t *fl;
+  u8 proto = ~0;
+
+  if (!silm->enabled)
+    return;
+
+  proto = snat_proto_to_ip_proto (snat_proto);
+
+  now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
+  now += silm->milisecond_time_0;
+
+  b0 = silm->nat44_session_buffer;
+
+  if (PREDICT_FALSE (b0 == 0))
+    {
+      if (do_flush)
+        return;
+
+      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
+        {
+          clib_warning ("can't allocate buffer for NAT IPFIX event");
+          return;
+        }
+
+      b0 = silm->nat44_session_buffer =
+        vlib_get_buffer (vm, bi0);
+      fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+      vlib_buffer_init_for_free_list (b0, fl);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+      offset = 0;
+    }
+  else
+    {
+      bi0 = vlib_get_buffer_index (vm, b0);
+      offset = silm->nat44_session_next_record_offset;
+    }
+
+  f = silm->nat44_session_frame;
+  if (PREDICT_FALSE (f == 0))
+    {
+      u32 * to_next;
+      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
+      silm->nat44_session_frame = f;
+      to_next = vlib_frame_vector_args (f);
+      to_next[0] = bi0;
+      f->n_vectors = 1;
+    }
+
+  if (PREDICT_FALSE (offset == 0))
+    snat_ipfix_header_create (frm, b0, &offset);
+
+  if (PREDICT_TRUE (do_flush == 0))
+    {
+      u64 time_stamp = clib_host_to_net_u64 (now);
+      clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
+      offset += sizeof (time_stamp);
+
+      clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event));
+      offset += sizeof (nat_event);
+
+      clib_memcpy (b0->data + offset, &src_ip, sizeof (src_ip));
+      offset += sizeof (src_ip);
+
+      clib_memcpy (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip));
+      offset += sizeof (nat_src_ip);
+
+      clib_memcpy (b0->data + offset, &proto, sizeof (proto));
+      offset += sizeof (proto);
+
+      clib_memcpy (b0->data + offset, &src_port, sizeof (src_port));
+      offset += sizeof (src_port);
+
+      clib_memcpy (b0->data + offset, &nat_src_port, sizeof (nat_src_port));
+      offset += sizeof (nat_src_port);
+
+      clib_memcpy (b0->data + offset, &vrf_id, sizeof(vrf_id));
+      offset += sizeof (vrf_id);
+
+      b0->current_length += NAT44_SESSION_CREATE_LEN;
+    }
+
+  if (PREDICT_FALSE (do_flush || (offset + NAT44_SESSION_CREATE_LEN) > frm->path_mtu))
+    {
+      snat_ipfix_send (frm, f, b0, silm->nat44_session_template_id);
+      silm->nat44_session_frame = 0;
+      silm->nat44_session_buffer = 0;
+      offset = 0;
+    }
+  silm->nat44_session_next_record_offset = offset;
+ }
+
+static void
+snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vlib_frame_t *f;
+  vlib_buffer_t *b0 = 0;
+  u32 bi0 = ~0;
+  u32 offset;
+  vlib_main_t * vm = frm->vlib_main;
+  u64 now;
+  vlib_buffer_free_list_t *fl;
+  u8 nat_event = NAT_ADDRESSES_EXHAUTED;
+
+  if (!silm->enabled)
+    return;
+
+  now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
+  now += silm->milisecond_time_0;
+
+  b0 = silm->addr_exhausted_buffer;
+
+  if (PREDICT_FALSE (b0 == 0))
+    {
+      if (do_flush)
+        return;
+
+      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
+        {
+          clib_warning ("can't allocate buffer for NAT IPFIX event");
+          return;
+        }
+
+      b0 = silm->addr_exhausted_buffer =
+        vlib_get_buffer (vm, bi0);
+      fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+      vlib_buffer_init_for_free_list (b0, fl);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+      offset = 0;
+    }
+  else
+    {
+      bi0 = vlib_get_buffer_index (vm, b0);
+      offset = silm->addr_exhausted_next_record_offset;
+    }
+
+  f = silm->addr_exhausted_frame;
+  if (PREDICT_FALSE (f == 0))
+    {
+      u32 * to_next;
+      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
+      silm->addr_exhausted_frame = f;
+      to_next = vlib_frame_vector_args (f);
+      to_next[0] = bi0;
+      f->n_vectors = 1;
+    }
+
+  if (PREDICT_FALSE (offset == 0))
+    snat_ipfix_header_create (frm, b0, &offset);
+
+  if (PREDICT_TRUE (do_flush == 0))
+    {
+      u64 time_stamp = clib_host_to_net_u64 (now);
+      clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
+      offset += sizeof (time_stamp);
+
+      clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event));
+      offset += sizeof (nat_event);
+
+      clib_memcpy (b0->data + offset, &pool_id, sizeof(pool_id));
+      offset += sizeof (pool_id);
+
+      b0->current_length += NAT_ADDRESSES_EXHAUTED_LEN;
+    }
+
+  if (PREDICT_FALSE (do_flush || (offset + NAT_ADDRESSES_EXHAUTED_LEN) > frm->path_mtu))
+    {
+      snat_ipfix_send (frm, f, b0, silm->addr_exhausted_template_id);
+      silm->addr_exhausted_frame = 0;
+      silm->addr_exhausted_buffer = 0;
+      offset = 0;
+    }
+  silm->addr_exhausted_next_record_offset = offset;
+}
+
+static void
+snat_ipfix_logging_max_entries_per_usr (u32 src_ip, int do_flush)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vlib_frame_t *f;
+  vlib_buffer_t *b0 = 0;
+  u32 bi0 = ~0;
+  u32 offset;
+  vlib_main_t * vm = frm->vlib_main;
+  u64 now;
+  vlib_buffer_free_list_t *fl;
+  u8 nat_event = QUOTA_EXCEEDED;
+  u32 quota_event = MAX_ENTRIES_PER_USER;
+
+  if (!silm->enabled)
+    return;
+
+  now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
+  now += silm->milisecond_time_0;
+
+  b0 = silm->max_entries_per_user_buffer;
+
+  if (PREDICT_FALSE (b0 == 0))
+    {
+      if (do_flush)
+        return;
+
+      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
+        {
+          clib_warning ("can't allocate buffer for NAT IPFIX event");
+          return;
+        }
+
+      b0 = silm->max_entries_per_user_buffer =
+        vlib_get_buffer (vm, bi0);
+      fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+      vlib_buffer_init_for_free_list (b0, fl);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+      offset = 0;
+    }
+  else
+    {
+      bi0 = vlib_get_buffer_index (vm, b0);
+      offset = silm->max_entries_per_user_next_record_offset;
+    }
+
+  f = silm->max_entries_per_user_frame;
+  if (PREDICT_FALSE (f == 0))
+    {
+      u32 * to_next;
+      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
+      silm->max_entries_per_user_frame = f;
+      to_next = vlib_frame_vector_args (f);
+      to_next[0] = bi0;
+      f->n_vectors = 1;
+    }
+
+  if (PREDICT_FALSE (offset == 0))
+    snat_ipfix_header_create (frm, b0, &offset);
+
+  if (PREDICT_TRUE (do_flush == 0))
+    {
+      u64 time_stamp = clib_host_to_net_u64 (now);
+      clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
+      offset += sizeof (time_stamp);
+
+      clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event));
+      offset += sizeof (nat_event);
+
+      clib_memcpy (b0->data + offset, "a_event, sizeof(quota_event));
+      offset += sizeof (quota_event);
+
+      clib_memcpy (b0->data + offset, &src_ip, sizeof (src_ip));
+      offset += sizeof (src_ip);
+
+      b0->current_length += MAX_ENTRIES_PER_USER_LEN;
+    }
+
+  if (PREDICT_FALSE (do_flush || (offset + MAX_ENTRIES_PER_USER_LEN) > frm->path_mtu))
+    {
+      snat_ipfix_send (frm, f, b0, silm->max_entries_per_user_template_id);
+      silm->max_entries_per_user_frame = 0;
+      silm->max_entries_per_user_buffer = 0;
+      offset = 0;
+    }
+  silm->max_entries_per_user_next_record_offset = offset;
+}
+
+static void
+snat_ipfix_logging_nat44_ses_rpc_cb (snat_ipfix_logging_nat44_ses_args_t *a)
+{
+  snat_ipfix_logging_nat44_ses(a->nat_event, a->src_ip, a->nat_src_ip,
+                               a->snat_proto, a->src_port, a->nat_src_port,
+                               a->vrf_id, 0);
+}
+
+/**
+ * @brief Generate NAT44 session create event
+ *
+ * @param src_ip       source IPv4 address
+ * @param nat_src_ip   transaltes source IPv4 address
+ * @param snat_proto   NAT transport protocol
+ * @param src_port     source port
+ * @param nat_src_port translated source port
+ * @param vrf_id       VRF ID
+ */
+void
+snat_ipfix_logging_nat44_ses_create (u32 src_ip,
+                                     u32 nat_src_ip,
+                                     snat_protocol_t snat_proto,
+                                     u16 src_port,
+                                     u16 nat_src_port,
+                                     u32 vrf_id)
+{
+  snat_ipfix_logging_nat44_ses_args_t a;
+
+  a.nat_event = NAT44_SESSION_CREATE;
+  a.src_ip = src_ip;
+  a.nat_src_ip = nat_src_ip;
+  a.snat_proto = snat_proto;
+  a.src_port = src_port;
+  a.nat_src_port = nat_src_port;
+  a.vrf_id = vrf_id;
+
+  vl_api_rpc_call_main_thread (snat_ipfix_logging_nat44_ses_rpc_cb, (u8 *) &a,
+                               sizeof (a));
+}
+
+/**
+ * @brief Generate NAT44 session delete event
+ *
+ * @param src_ip       source IPv4 address
+ * @param nat_src_ip   transaltes source IPv4 address
+ * @param snat_proto   NAT transport protocol
+ * @param src_port     source port
+ * @param nat_src_port translated source port
+ * @param vrf_id       VRF ID
+ */
+void
+snat_ipfix_logging_nat44_ses_delete (u32 src_ip,
+                                     u32 nat_src_ip,
+                                     snat_protocol_t snat_proto,
+                                     u16 src_port,
+                                     u16 nat_src_port,
+                                     u32 vrf_id)
+{
+  snat_ipfix_logging_nat44_ses_args_t a;
+
+  a.nat_event = NAT44_SESSION_DELETE;
+  a.src_ip = src_ip;
+  a.nat_src_ip = nat_src_ip;
+  a.snat_proto = snat_proto;
+  a.src_port = src_port;
+  a.nat_src_port = nat_src_port;
+  a.vrf_id = vrf_id;
+
+  vl_api_rpc_call_main_thread (snat_ipfix_logging_nat44_ses_rpc_cb, (u8 *) &a,
+                               sizeof (a));
+}
+
+vlib_frame_t *
+snat_data_callback_nat44_session (flow_report_main_t * frm,
+                                  flow_report_t * fr,
+                                  vlib_frame_t * f,
+                                  u32 * to_next,
+                                  u32 node_index)
+{
+  snat_ipfix_logging_nat44_ses(0, 0, 0, 0, 0, 0, 0, 1);
+  return f;
+}
+
+static void
+snat_ipfix_logging_addr_exhausted_rpc_cb
+ (snat_ipfix_logging_addr_exhausted_args_t * a)
+{
+  snat_ipfix_logging_addr_exhausted(a->pool_id, 0);
+}
+
+/**
+ * @brief Generate NAT addresses exhausted event
+ *
+ * @param pool_id NAT pool ID
+ */
+void
+snat_ipfix_logging_addresses_exhausted(u32 pool_id)
+{
+  //TODO: This event SHOULD be rate limited
+  snat_ipfix_logging_addr_exhausted_args_t a;
+
+  a.pool_id = pool_id;
+
+  vl_api_rpc_call_main_thread (snat_ipfix_logging_addr_exhausted_rpc_cb,
+                               (u8 *) &a, sizeof (a));
+}
+
+vlib_frame_t *
+snat_data_callback_addr_exhausted (flow_report_main_t * frm,
+                                   flow_report_t * fr,
+                                   vlib_frame_t * f,
+                                   u32 * to_next,
+                                   u32 node_index)
+{
+  snat_ipfix_logging_addr_exhausted(0, 1);
+  return f;
+}
+
+static void
+snat_ipfix_logging_max_entries_per_usr_rpc_cb
+ (snat_ipfix_logging_max_entries_per_user_args_t * a)
+{
+  snat_ipfix_logging_max_entries_per_usr(a->src_ip, 0);
+}
+
+/**
+ * @brief Generate maximum entries per user exceeded event
+ *
+ * @param src_ip source IPv4 address
+ */
+void
+snat_ipfix_logging_max_entries_per_user(u32 src_ip)
+{
+  //TODO: This event SHOULD be rate limited
+  snat_ipfix_logging_max_entries_per_user_args_t a;
+
+  a.src_ip = src_ip;
+
+  vl_api_rpc_call_main_thread (snat_ipfix_logging_max_entries_per_usr_rpc_cb,
+                               (u8 *) &a, sizeof (a));
+}
+
+vlib_frame_t *
+snat_data_callback_max_entries_per_usr (flow_report_main_t * frm,
+                                        flow_report_t * fr,
+                                        vlib_frame_t * f,
+                                        u32 * to_next,
+                                        u32 node_index)
+{
+  snat_ipfix_logging_max_entries_per_usr(0, 1);
+  return f;
+}
+
+/**
+ * @brief Enable/disable NAT plugin IPFIX logging
+ *
+ * @param enable    1 if enable, 0 if disable
+ * @param domain_id observation domain ID
+ * @param src_port  source port number
+ *
+ * @returns 0 if success
+ */
+int
+snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
+{
+  snat_main_t * sm = &snat_main;
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+  flow_report_main_t *frm = &flow_report_main;
+  vnet_flow_report_add_del_args_t a;
+  int rv;
+  u8 e = enable ? 1 : 0;
+
+  if (silm->enabled == e)
+    return 0;
+
+  silm->enabled = e;
+
+  memset (&a, 0, sizeof (a));
+  a.is_add = enable;
+  a.domain_id = domain_id ? domain_id : 1;
+  a.src_port = src_port ? src_port : UDP_DST_PORT_ipfix;
+
+  if (sm->deterministic)
+    {
+      a.rewrite_callback = snat_template_rewrite_max_entries_per_usr;
+      a.flow_data_callback = snat_data_callback_max_entries_per_usr;
+
+      rv = vnet_flow_report_add_del (frm, &a, NULL);
+      if (rv)
+        {
+          clib_warning ("vnet_flow_report_add_del returned %d", rv);
+          return -1;
+        }
+    }
+  else
+    {
+      a.rewrite_callback = snat_template_rewrite_nat44_session;
+      a.flow_data_callback = snat_data_callback_nat44_session;
+
+      rv = vnet_flow_report_add_del (frm, &a, NULL);
+      if (rv)
+        {
+          clib_warning ("vnet_flow_report_add_del returned %d", rv);
+          return -1;
+        }
+
+      a.rewrite_callback = snat_template_rewrite_addr_exhausted;
+      a.flow_data_callback = snat_data_callback_addr_exhausted;
+
+      rv = vnet_flow_report_add_del (frm, &a, NULL);
+      if (rv)
+        {
+          clib_warning ("vnet_flow_report_add_del returned %d", rv);
+          return -1;
+        }
+    }
+
+  return 0;
+}
+
+/**
+ * @brief Initialize NAT plugin IPFIX logging
+ *
+ * @param vm vlib main
+ */
+void
+snat_ipfix_logging_init (vlib_main_t * vm)
+{
+  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
+
+  silm->enabled = 0;
+
+  /* Set up time reference pair */
+  silm->vlib_time_0 = vlib_time_now (vm);
+  silm->milisecond_time_0 = unix_time_now_nsec () * 1e-6;
+}
diff --git a/src/plugins/nat/nat_ipfix_logging.h b/src/plugins/nat/nat_ipfix_logging.h
new file mode 100644
index 00000000..6dbf6627
--- /dev/null
+++ b/src/plugins/nat/nat_ipfix_logging.h
@@ -0,0 +1,79 @@
+/*
+ * nat_ipfix_logging.h - NAT Events IPFIX logging
+ *
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_nat_ipfix_logging_h__
+#define __included_nat_ipfix_logging_h__
+
+typedef enum {
+  NAT_ADDRESSES_EXHAUTED = 3,
+  NAT44_SESSION_CREATE = 4,
+  NAT44_SESSION_DELETE = 5,
+  NAT_PORTS_EXHAUSTED = 12,
+  QUOTA_EXCEEDED = 13,
+} nat_event_t;
+
+typedef enum {
+  MAX_ENTRIES_PER_USER = 3,
+} quota_exceed_event_t;
+
+typedef struct {
+  /** NAT plugin IPFIX logging enabled */
+  u8 enabled;
+
+  /** ipfix buffers under construction */
+  vlib_buffer_t *nat44_session_buffer;
+  vlib_buffer_t *addr_exhausted_buffer;
+  vlib_buffer_t *max_entries_per_user_buffer;
+
+  /** frames containing ipfix buffers */
+  vlib_frame_t *nat44_session_frame;
+  vlib_frame_t *addr_exhausted_frame;
+  vlib_frame_t *max_entries_per_user_frame;
+
+  /** next record offset */
+  u32 nat44_session_next_record_offset;
+  u32 addr_exhausted_next_record_offset;
+  u32 max_entries_per_user_next_record_offset;
+
+  /** Time reference pair */
+  u64 milisecond_time_0;
+  f64 vlib_time_0;
+
+  /** template IDs */
+  u16 nat44_session_template_id;
+  u16 addr_exhausted_template_id;
+  u16 max_entries_per_user_template_id;
+
+  /** stream index */
+  u32 stream_index;
+} snat_ipfix_logging_main_t;
+
+extern snat_ipfix_logging_main_t snat_ipfix_logging_main;
+
+void snat_ipfix_logging_init (vlib_main_t * vm);
+int snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port);
+void snat_ipfix_logging_nat44_ses_create (u32 src_ip, u32 nat_src_ip,
+                                          snat_protocol_t snat_proto,
+                                          u16 src_port, u16 nat_src_port,
+                                          u32 vrf_id);
+void snat_ipfix_logging_nat44_ses_delete (u32 src_ip, u32 nat_src_ip,
+                                          snat_protocol_t snat_proto,
+                                          u16 src_port, u16 nat_src_port,
+                                          u32 vrf_id);
+void snat_ipfix_logging_addresses_exhausted(u32 pool_id);
+void snat_ipfix_logging_max_entries_per_user(u32 src_ip);
+
+#endif /* __included_nat_ipfix_logging_h__ */
diff --git a/src/plugins/nat/nat_msg_enum.h b/src/plugins/nat/nat_msg_enum.h
new file mode 100644
index 00000000..710b631c
--- /dev/null
+++ b/src/plugins/nat/nat_msg_enum.h
@@ -0,0 +1,31 @@
+
+/*
+ * nat_msg_enum.h - skeleton vpp engine plug-in message enumeration
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_nat_msg_enum_h
+#define included_nat_msg_enum_h
+
+#include 
+
+#define vl_msg_id(n,h) n,
+typedef enum {
+#include 
+    /* We'll want to know how many messages IDs we need... */
+    VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif /* included_nat_msg_enum_h */
diff --git a/src/plugins/nat/nat_test.c b/src/plugins/nat/nat_test.c
new file mode 100644
index 00000000..b653b77e
--- /dev/null
+++ b/src/plugins/nat/nat_test.c
@@ -0,0 +1,1167 @@
+
+/*
+ * nat.c - skeleton vpp-api-test plug-in
+ *
+ * Copyright (c)  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define __plugin_msg_base snat_test_main.msg_id_base
+#include 
+
+uword unformat_sw_if_index (unformat_input_t * input, va_list * args);
+
+/* Declare message IDs */
+#include 
+
+/* define message structures */
+#define vl_typedefs
+#include 
+#undef vl_typedefs
+
+/* declare message handlers for each api */
+
+#define vl_endianfun             /* define message structures */
+#include 
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...)
+#define vl_printfun
+#include 
+#undef vl_printfun
+
+/* Get the API version number. */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include 
+#undef vl_api_version
+
+typedef struct {
+    /* API message ID base */
+    u16 msg_id_base;
+    vat_main_t *vat_main;
+} snat_test_main_t;
+
+snat_test_main_t snat_test_main;
+
+#define foreach_standard_reply_retval_handler   \
+_(snat_add_address_range_reply)                 \
+_(snat_interface_add_del_feature_reply)         \
+_(snat_add_static_mapping_reply)                \
+_(snat_set_workers_reply)                       \
+_(snat_add_del_interface_addr_reply)            \
+_(snat_ipfix_enable_disable_reply)              \
+_(snat_add_det_map_reply)                       \
+_(snat_det_set_timeouts_reply)                  \
+_(snat_det_close_session_out_reply)             \
+_(snat_det_close_session_in_reply)
+
+#define _(n)                                            \
+    static void vl_api_##n##_t_handler                  \
+    (vl_api_##n##_t * mp)                               \
+    {                                                   \
+        vat_main_t * vam = snat_test_main.vat_main;   \
+        i32 retval = ntohl(mp->retval);                 \
+        if (vam->async_mode) {                          \
+            vam->async_errors += (retval < 0);          \
+        } else {                                        \
+            vam->retval = retval;                       \
+            vam->result_ready = 1;                      \
+        }                                               \
+    }
+foreach_standard_reply_retval_handler;
+#undef _
+
+/* 
+ * Table of message reply handlers, must include boilerplate handlers
+ * we just generated
+ */
+#define foreach_vpe_api_reply_msg                               \
+_(SNAT_ADD_ADDRESS_RANGE_REPLY, snat_add_address_range_reply)   \
+_(SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY,                         \
+  snat_interface_add_del_feature_reply)                         \
+_(SNAT_ADD_STATIC_MAPPING_REPLY, snat_add_static_mapping_reply) \
+_(SNAT_CONTROL_PING_REPLY, snat_control_ping_reply)             \
+_(SNAT_STATIC_MAPPING_DETAILS, snat_static_mapping_details)     \
+_(SNAT_SHOW_CONFIG_REPLY, snat_show_config_reply)               \
+_(SNAT_ADDRESS_DETAILS, snat_address_details)                   \
+_(SNAT_INTERFACE_DETAILS, snat_interface_details)               \
+_(SNAT_SET_WORKERS_REPLY, snat_set_workers_reply)               \
+_(SNAT_WORKER_DETAILS, snat_worker_details)                     \
+_(SNAT_ADD_DEL_INTERFACE_ADDR_REPLY,                            \
+  snat_add_del_interface_addr_reply)                            \
+_(SNAT_INTERFACE_ADDR_DETAILS, snat_interface_addr_details)     \
+_(SNAT_IPFIX_ENABLE_DISABLE_REPLY,                              \
+  snat_ipfix_enable_disable_reply)                              \
+_(SNAT_USER_DETAILS, snat_user_details)                         \
+_(SNAT_USER_SESSION_DETAILS, snat_user_session_details)         \
+_(SNAT_ADD_DET_MAP_REPLY, snat_add_det_map_reply)               \
+_(SNAT_DET_FORWARD_REPLY, snat_det_forward_reply)               \
+_(SNAT_DET_REVERSE_REPLY, snat_det_reverse_reply)               \
+_(SNAT_DET_MAP_DETAILS, snat_det_map_details)                   \
+_(SNAT_DET_SET_TIMEOUTS_REPLY, snat_det_set_timeouts_reply)     \
+_(SNAT_DET_GET_TIMEOUTS_REPLY, snat_det_get_timeouts_reply)     \
+_(SNAT_DET_CLOSE_SESSION_OUT_REPLY,                             \
+  snat_det_close_session_out_reply)                             \
+_(SNAT_DET_CLOSE_SESSION_IN_REPLY,                              \
+  snat_det_close_session_in_reply)                              \
+_(SNAT_DET_SESSION_DETAILS, snat_det_session_details)
+
+static int api_snat_add_address_range (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  ip4_address_t start_addr, end_addr;
+  u32 start_host_order, end_host_order;
+  vl_api_snat_add_address_range_t * mp;
+  u8 is_add = 1;
+  int count;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U - %U",
+                    unformat_ip4_address, &start_addr,
+                    unformat_ip4_address, &end_addr))
+        ;
+      else if (unformat (i, "%U", unformat_ip4_address, &start_addr))
+        end_addr = start_addr;
+      else if (unformat (i, "del"))
+        is_add = 0;
+      else
+        {
+          clib_warning("unknown input '%U'", format_unformat_error, i);
+          return -99;
+        }
+    }
+
+  start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
+  end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
+  
+  if (end_host_order < start_host_order)
+    {
+      errmsg ("end address less than start address\n");
+      return -99;
+    }
+
+  count = (end_host_order - start_host_order) + 1;
+
+  if (count > 1024)
+    {
+    errmsg ("%U - %U, %d addresses...\n",
+           format_ip4_address, &start_addr,
+           format_ip4_address, &end_addr,
+           count);
+    }
+  
+  M(SNAT_ADD_ADDRESS_RANGE, mp);
+
+  memcpy (mp->first_ip_address, &start_addr, 4);
+  memcpy (mp->last_ip_address, &end_addr, 4);
+  mp->is_ip4 = 1;
+  mp->is_add = is_add;
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static int api_snat_interface_add_del_feature (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_interface_add_del_feature_t * mp;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  u8 is_inside = 1; 
+  u8 is_add = 1;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
+        sw_if_index_set = 1;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+        sw_if_index_set = 1;
+      else if (unformat (i, "out"))
+        is_inside = 0;
+      else if (unformat (i, "in"))
+        is_inside = 1;
+      else if (unformat (i, "del"))
+        is_add = 0;
+      else
+        {
+          clib_warning("unknown input '%U'", format_unformat_error, i);
+          return -99;
+        }
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      errmsg ("interface / sw_if_index required\n");
+      return -99;
+    }
+
+  M(SNAT_INTERFACE_ADD_DEL_FEATURE, mp);
+  mp->sw_if_index = ntohl(sw_if_index);
+  mp->is_add = is_add;
+  mp->is_inside = is_inside;
+  
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static int api_snat_add_static_mapping(vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_add_static_mapping_t * mp;
+  u8 external_addr_set = 0;
+  u8 local_addr_set = 0;
+  u8 is_add = 1;
+  u8 addr_only = 1;
+  ip4_address_t local_addr, external_addr;
+  u32 local_port = 0, external_port = 0, vrf_id = ~0;
+  u32 sw_if_index = ~0;
+  u8 sw_if_index_set = 0;
+  u32 proto = ~0;
+  u8 proto_set = 0;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "local_addr %U", unformat_ip4_address, &local_addr))
+        local_addr_set = 1;
+      else if (unformat (i, "external_addr %U", unformat_ip4_address,
+                         &external_addr))
+        external_addr_set = 1;
+      else if (unformat (i, "local_port %u", &local_port))
+        addr_only = 0;
+      else if (unformat (i, "external_port %u", &external_port))
+        addr_only = 0;
+      else if (unformat (i, "external_if %U", unformat_sw_if_index, vam,
+                         &sw_if_index))
+        sw_if_index_set = 1;
+      else if (unformat (i, "external_sw_if_index %d", &sw_if_index))
+        sw_if_index_set = 1;
+      else if (unformat (i, "vrf %u", &vrf_id))
+        ;
+      else if (unformat (i, "protocol %u", &proto))
+        proto_set = 1;
+      else if (unformat (i, "del"))
+        is_add = 0;
+      else
+        {
+          clib_warning("unknown input '%U'", format_unformat_error, i);
+          return -99;
+        }
+    }
+
+  if (!addr_only && !proto_set)
+    {
+      errmsg ("protocol required\n");
+      return -99;
+    }
+
+  if (!local_addr_set)
+    {
+      errmsg ("local addr required\n");
+      return -99;
+    }
+  if (!external_addr_set && !sw_if_index_set)
+    {
+      errmsg ("external addr or interface required\n");
+      return -99;
+    }
+
+  M(SNAT_ADD_STATIC_MAPPING, mp);
+  mp->is_add = is_add;
+  mp->is_ip4 = 1;
+  mp->addr_only = addr_only;
+  mp->local_port = ntohs ((u16) local_port);
+  mp->external_port = ntohs ((u16) external_port);
+  mp->external_sw_if_index = ntohl (sw_if_index);
+  mp->vrf_id = ntohl (vrf_id);
+  mp->protocol = (u8) proto;
+  memcpy (mp->local_ip_address, &local_addr, 4);
+  memcpy (mp->external_ip_address, &external_addr, 4);
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_control_ping_reply_t_handler
+  (vl_api_snat_control_ping_reply_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+  i32 retval = ntohl (mp->retval);
+  if (vam->async_mode)
+    {
+      vam->async_errors += (retval < 0);
+    }
+  else
+    {
+      vam->retval = retval;
+      vam->result_ready = 1;
+    }
+}
+
+static void vl_api_snat_static_mapping_details_t_handler
+  (vl_api_snat_static_mapping_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  if (mp->addr_only && mp->external_sw_if_index != ~0)
+      fformat (vam->ofp, "%15U%6s%15d%6s%11d%6d\n",
+               format_ip4_address, &mp->local_ip_address, "",
+               ntohl (mp->external_sw_if_index), "",
+               ntohl (mp->vrf_id),
+               mp->protocol);
+  else if (mp->addr_only && mp->external_sw_if_index == ~0)
+      fformat (vam->ofp, "%15U%6s%15U%6s%11d%6d\n",
+               format_ip4_address, &mp->local_ip_address, "",
+               format_ip4_address, &mp->external_ip_address, "",
+               ntohl (mp->vrf_id),
+               mp->protocol);
+  else if (!mp->addr_only && mp->external_sw_if_index != ~0)
+      fformat (vam->ofp, "%15U%6d%15d%6d%11d%6d\n",
+               format_ip4_address, &mp->local_ip_address,
+               ntohs (mp->local_port),
+               ntohl (mp->external_sw_if_index),
+               ntohs (mp->external_port),
+               ntohl (mp->vrf_id),
+               mp->protocol);
+  else
+      fformat (vam->ofp, "%15U%6d%15U%6d%11d%6d\n",
+               format_ip4_address, &mp->local_ip_address,
+               ntohs (mp->local_port),
+               format_ip4_address, &mp->external_ip_address,
+               ntohs (mp->external_port),
+               ntohl (mp->vrf_id),
+               mp->protocol);
+
+}
+
+static int api_snat_static_mapping_dump(vat_main_t * vam)
+{
+  vl_api_snat_static_mapping_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_static_mapping_dump");
+      return -99;
+    }
+
+  fformat (vam->ofp, "%21s%21s\n", "local", "external");
+  fformat (vam->ofp, "%15s%6s%15s%6s%11s%6s\n", "address", "port",
+           "address/if_idx", "port", "vrf", "proto");
+
+  M(SNAT_STATIC_MAPPING_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_show_config_reply_t_handler
+  (vl_api_snat_show_config_reply_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+  i32 retval = ntohl (mp->retval);
+
+  if (retval >= 0)
+    {
+      fformat (vam->ofp, "translation hash buckets %d\n",
+               ntohl (mp->translation_buckets));
+      fformat (vam->ofp, "translation hash memory %d\n",
+               ntohl (mp->translation_memory_size));
+      fformat (vam->ofp, "user hash buckets %d\n", ntohl (mp->user_buckets));
+      fformat (vam->ofp, "user hash memory %d\n", ntohl (mp->user_memory_size));
+      fformat (vam->ofp, "max translations per user %d\n",
+               ntohl (mp->max_translations_per_user));
+      fformat (vam->ofp, "outside VRF id %d\n", ntohl (mp->outside_vrf_id));
+      fformat (vam->ofp, "inside VRF id %d\n", ntohl (mp->inside_vrf_id));
+      if (mp->static_mapping_only)
+        {
+          fformat (vam->ofp, "static mapping only");
+          if (mp->static_mapping_connection_tracking)
+            fformat (vam->ofp, " connection tracking");
+          fformat (vam->ofp, "\n");
+        }
+    }
+  vam->retval = retval;
+  vam->result_ready = 1;
+}
+
+static int api_snat_show_config(vat_main_t * vam)
+{
+  vl_api_snat_show_config_t * mp;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_show_config");
+      return -99;
+    }
+
+  M(SNAT_SHOW_CONFIG, mp);
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_address_details_t_handler
+  (vl_api_snat_address_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat (vam->ofp, "%U\n", format_ip4_address, &mp->ip_address);
+}
+
+static int api_snat_address_dump(vat_main_t * vam)
+{
+  vl_api_snat_address_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_address_dump");
+      return -99;
+    }
+
+  M(SNAT_ADDRESS_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_interface_details_t_handler
+  (vl_api_snat_interface_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat (vam->ofp, "sw_if_index %d %s\n", ntohl (mp->sw_if_index),
+           mp->is_inside ? "in" : "out");
+}
+
+static int api_snat_interface_dump(vat_main_t * vam)
+{
+  vl_api_snat_interface_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_address_dump");
+      return -99;
+    }
+
+  M(SNAT_INTERFACE_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static int api_snat_set_workers (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_set_workers_t * mp;
+  uword *bitmap;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", unformat_bitmap_list, &bitmap))
+        ;
+      else
+        {
+          clib_warning("unknown input '%U'", format_unformat_error, i);
+          return -99;
+        }
+    }
+
+  M(SNAT_SET_WORKERS, mp);
+  mp->worker_mask = clib_host_to_net_u64 (bitmap[0]);
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_worker_details_t_handler
+  (vl_api_snat_worker_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat (vam->ofp, "worker_index %d (%s at lcore %u)\n",
+           ntohl (mp->worker_index), mp->name, ntohl (mp->lcore_id));
+}
+
+static int api_snat_worker_dump(vat_main_t * vam)
+{
+  vl_api_snat_worker_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_address_dump");
+      return -99;
+    }
+
+  M(SNAT_WORKER_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static int api_snat_add_del_interface_addr (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_add_del_interface_addr_t * mp;
+  u32 sw_if_index;
+  u8 sw_if_index_set = 0;
+  u8 is_add = 1;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
+        sw_if_index_set = 1;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+        sw_if_index_set = 1;
+      else if (unformat (i, "del"))
+        is_add = 0;
+      else
+        {
+          clib_warning("unknown input '%U'", format_unformat_error, i);
+          return -99;
+        }
+    }
+
+  if (sw_if_index_set == 0)
+    {
+      errmsg ("interface / sw_if_index required\n");
+      return -99;
+    }
+
+  M(SNAT_ADD_DEL_INTERFACE_ADDR, mp);
+  mp->sw_if_index = ntohl(sw_if_index);
+  mp->is_add = is_add;
+  
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_interface_addr_details_t_handler
+  (vl_api_snat_interface_addr_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat (vam->ofp, "sw_if_index %d\n", ntohl (mp->sw_if_index));
+}
+
+static int api_snat_interface_addr_dump(vat_main_t * vam)
+{
+  vl_api_snat_interface_addr_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_address_dump");
+      return -99;
+    }
+
+  M(SNAT_INTERFACE_ADDR_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static int api_snat_ipfix_enable_disable (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_ipfix_enable_disable_t * mp;
+  u32 domain_id = 0;
+  u32 src_port = 0;
+  u8 enable = 1;
+  int ret;
+
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "domain %d", &domain_id))
+        ;
+      else if (unformat (i, "src_port %d", &src_port))
+        ;
+      else if (unformat (i, "disable"))
+        enable = 0;
+      else
+        {
+          clib_warning("unknown input '%U'", format_unformat_error, i);
+          return -99;
+        }
+    }
+
+  M(SNAT_IPFIX_ENABLE_DISABLE, mp);
+  mp->domain_id = htonl(domain_id);
+  mp->src_port = htons((u16) src_port);
+  mp->enable = enable;
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_user_session_details_t_handler
+  (vl_api_snat_user_session_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat(vam->ofp, "%s session %U:%d to %U:%d protocol id %d "
+                    "total packets %d total bytes %d\n",
+          mp->is_static ? "static" : "dynamic",
+          format_ip4_address, mp->inside_ip_address, ntohl(mp->inside_port),
+          format_ip4_address, mp->outside_ip_address, ntohl(mp->outside_port),
+          ntohl(mp->protocol), ntohl(mp->total_pkts), ntohl(mp->total_bytes));
+}
+
+static int api_snat_user_session_dump(vat_main_t * vam)
+{
+  unformat_input_t* i = vam->input;
+  vl_api_snat_user_session_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  ip4_address_t addr;
+  u32 vrf_id = ~0;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_address_dump");
+      return -99;
+    }
+
+  if (unformat (i, "ip_address %U vrf_id %d",
+                unformat_ip4_address, &addr, &vrf_id))
+    ;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_USER_SESSION_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  memset(mp->ip_address, 0, 16);
+  clib_memcpy(mp->ip_address, &addr, 4);
+  mp->vrf_id = htonl(vrf_id);
+  mp->is_ip4 = 1;
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_user_details_t_handler
+  (vl_api_snat_user_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat(vam->ofp, "user with ip %U with vrf_id %d "
+                    "with %d sessions and %d static sessions\n",
+          format_ip4_address, mp->ip_address, ntohl(mp->vrf_id),
+          ntohl(mp->nsessions), ntohl(mp->nstaticsessions));
+}
+
+static int api_snat_user_dump(vat_main_t * vam)
+{
+  vl_api_snat_user_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_address_dump");
+      return -99;
+    }
+
+  M(SNAT_USER_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static int api_snat_add_det_map (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_add_det_map_t * mp;
+  ip4_address_t in_addr, out_addr;
+  u32 in_plen, out_plen;
+  u8 is_add = 1;
+  int ret;
+
+  if (unformat (i, "in %U/%d out %U/%d",
+                unformat_ip4_address, &in_addr, &in_plen,
+                unformat_ip4_address, &out_addr, &out_plen))
+    ;
+  else if (unformat (i, "del"))
+    is_add = 0;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_ADD_DET_MAP, mp);
+  clib_memcpy(mp->in_addr, &in_addr, 4);
+  mp->in_plen = in_plen;
+  clib_memcpy(mp->out_addr, &out_addr, 4);
+  mp->out_plen = out_plen;
+  mp->is_add = is_add;
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_det_forward_reply_t_handler
+  (vl_api_snat_det_forward_reply_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+  i32 retval = ntohl(mp->retval);
+
+  if (retval >= 0)
+  {
+    fformat (vam->ofp, "outside address %U", format_ip4_address, &mp->out_addr);
+    fformat (vam->ofp, " outside port range start %d", ntohs(mp->out_port_lo));
+    fformat (vam->ofp, " outside port range end %d\n", ntohs(mp->out_port_hi));
+  }
+
+  vam->retval = retval;
+  vam->result_ready = 1;
+}
+
+static int api_snat_det_forward (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_det_forward_t * mp;
+  ip4_address_t in_addr;
+  int ret;
+
+  if (unformat (i, "%U", unformat_ip4_address, &in_addr))
+    ;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_DET_FORWARD, mp);
+  clib_memcpy(mp->in_addr, &in_addr, 4);
+
+  S(mp);
+  W(ret);
+  return ret;
+}
+
+static void vl_api_snat_det_reverse_reply_t_handler
+  (vl_api_snat_det_reverse_reply_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+  i32 retval = ntohl(mp->retval);
+
+  if (retval >= 0)
+  {
+    fformat (vam->ofp, "inside address %U\n", format_ip4_address, &mp->in_addr);
+  }
+
+  vam->retval = retval;
+  vam->result_ready = 1;
+}
+
+static int api_snat_det_reverse (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_det_reverse_t * mp;
+  ip4_address_t out_addr;
+  u16 out_port;
+  int ret;
+
+  if (unformat (i, "%U %d", unformat_ip4_address, &out_addr, &out_port))
+    ;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_DET_REVERSE, mp);
+  clib_memcpy(mp->out_addr, &out_addr, 4);
+  mp->out_port = htons(out_port);
+
+  S(mp);
+  W(ret);
+  return ret;
+}
+
+static void vl_api_snat_det_map_details_t_handler
+  (vl_api_snat_det_map_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat (vam->ofp, "Deterministic S-NAT mapping in %U/%d out %U/%d "
+                     "ports per host %d sharing ratio %d "
+                     "number of sessions %d",
+           format_ip4_address, mp->in_addr, mp->in_plen,
+           format_ip4_address, mp->out_addr, mp->out_plen,
+           ntohs(mp->ports_per_host), ntohl(mp->sharing_ratio),
+           ntohl(mp->ses_num));
+}
+
+static int api_snat_det_map_dump(vat_main_t * vam)
+{
+  vl_api_snat_det_map_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_det_map_dump");
+      return -99;
+    }
+
+  M(SNAT_DET_MAP_DUMP, mp);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static int api_snat_det_set_timeouts (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_det_set_timeouts_t * mp;
+  u32 udp = SNAT_UDP_TIMEOUT;
+  u32 tcp_established = SNAT_TCP_ESTABLISHED_TIMEOUT;
+  u32 tcp_transitory = SNAT_TCP_TRANSITORY_TIMEOUT;
+  u32 icmp = SNAT_ICMP_TIMEOUT;
+  int ret;
+
+  if (unformat (i, "udp %d", &udp))
+    ;
+  else if (unformat (i, "tcp_established %d", &tcp_established))
+    ;
+  else if (unformat (i, "tcp_transitory %d", &tcp_transitory))
+    ;
+  else if (unformat (i, "icmp %d", &icmp))
+    ;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_DET_SET_TIMEOUTS, mp);
+  mp->udp = htonl(udp);
+  mp->tcp_established = htonl(tcp_established);
+  mp->tcp_transitory = htonl(tcp_transitory);
+  mp->icmp = htonl(icmp);
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_det_get_timeouts_reply_t_handler
+  (vl_api_snat_det_get_timeouts_reply_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+  i32 retval = ntohl (mp->retval);
+
+  if (retval >= 0)
+    {
+      fformat (vam->ofp, "udp timeout: %dsec\n", ntohl (mp->udp));
+      fformat (vam->ofp, "tcp-established timeout: %dsec",
+               ntohl (mp->tcp_established));
+      fformat (vam->ofp, "tcp-transitory timeout: %dsec",
+               ntohl (mp->tcp_transitory));
+      fformat (vam->ofp, "icmp timeout: %dsec", ntohl (mp->icmp));
+    }
+  vam->retval = retval;
+  vam->result_ready = 1;
+}
+
+static int api_snat_det_get_timeouts(vat_main_t * vam)
+{
+  vl_api_snat_det_get_timeouts_t * mp;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_show_config");
+      return -99;
+    }
+
+  M(SNAT_DET_GET_TIMEOUTS, mp);
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static int api_snat_det_close_session_out (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_det_close_session_out_t * mp;
+  ip4_address_t out_addr, ext_addr;
+  u16 out_port, ext_port;
+  int ret;
+
+  if (unformat (i, "%U:%d %U:%d",
+                unformat_ip4_address, &out_addr, &out_port,
+                unformat_ip4_address, &ext_addr, &ext_port))
+    ;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_DET_CLOSE_SESSION_OUT, mp);
+  clib_memcpy(mp->out_addr, &out_addr, 4);
+  mp->out_port = ntohs(out_port);
+  clib_memcpy(mp->ext_addr, &ext_addr, 4);
+  mp->ext_port = ntohs(ext_port);
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static int api_snat_det_close_session_in (vat_main_t * vam)
+{
+  unformat_input_t * i = vam->input;
+  vl_api_snat_det_close_session_in_t * mp;
+  ip4_address_t in_addr, ext_addr;
+  u16 in_port, ext_port;
+  int ret;
+
+  if (unformat (i, "%U:%d %U:%d",
+                unformat_ip4_address, &in_addr, &in_port,
+                unformat_ip4_address, &ext_addr, &ext_port))
+    ;
+  else
+    {
+      clib_warning("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_DET_CLOSE_SESSION_IN, mp);
+  clib_memcpy(mp->in_addr, &in_addr, 4);
+  mp->in_port = ntohs(in_port);
+  clib_memcpy(mp->ext_addr, &ext_addr, 4);
+  mp->ext_port = ntohs(ext_port);
+
+  S(mp);
+  W (ret);
+  return ret;
+}
+
+static void vl_api_snat_det_session_details_t_handler
+  (vl_api_snat_det_session_details_t *mp)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  vat_main_t *vam = sm->vat_main;
+
+  fformat(vam->ofp, "deterministic session, external host address %U, "
+                    "external host port %d, outer port %d, inside port %d",
+          format_ip4_address, mp->ext_addr, mp->ext_port,
+          mp->out_port, mp->in_port);
+}
+
+static int api_snat_det_session_dump(vat_main_t * vam)
+{
+  unformat_input_t* i = vam->input;
+  vl_api_snat_det_session_dump_t * mp;
+  vl_api_snat_control_ping_t *mp_ping;
+  ip4_address_t user_addr;
+  int ret;
+
+  if (vam->json_output)
+    {
+      clib_warning ("JSON output not supported for snat_det_session_dump");
+      return -99;
+    }
+
+  if (unformat (i, "user_addr %U", unformat_ip4_address, &user_addr))
+    ;
+  else
+    {
+      clib_warning ("unknown input '%U'", format_unformat_error, i);
+      return -99;
+    }
+
+  M(SNAT_DET_SESSION_DUMP, mp);
+  clib_memcpy (&mp->user_addr, &user_addr, 4);
+  S(mp);
+
+  /* Use a control ping for synchronization */
+  M(SNAT_CONTROL_PING, mp_ping);
+  S(mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+/* 
+ * List of messages that the api test plugin sends,
+ * and that the data plane plugin processes
+ */
+#define foreach_vpe_api_msg                                      \
+_(snat_add_address_range, " [-  | sw_if_index  [in] [out] [del]")                 \
+_(snat_add_static_mapping, "local_addr  (external_addr " \
+  " | external_if  | external_sw_if_ndex ) "          \
+  "[local_port ] [external_port ] [vrf ] [del] " \
+  "protocol ")                                                \
+_(snat_set_workers, "")                          \
+_(snat_static_mapping_dump, "")                                  \
+_(snat_show_config, "")                                          \
+_(snat_address_dump, "")                                         \
+_(snat_interface_dump, "")                                       \
+_(snat_worker_dump, "")                                          \
+_(snat_add_del_interface_addr,                                   \
+  " | sw_if_index  [del]")                            \
+_(snat_interface_addr_dump, "")                                  \
+_(snat_ipfix_enable_disable, "[domain ] [src_port ] "     \
+  "[disable]")                                                   \
+_(snat_user_dump, "")                                            \
+_(snat_user_session_dump, "ip_address  vrf_id ")   \
+_(snat_add_det_map, "in / out "                \
+  "/ [del]")                                 \
+_(snat_det_forward, "")                                 \
+_(snat_det_reverse, " ")                     \
+_(snat_det_map_dump, "")                                         \
+_(snat_det_set_timeouts, "[udp  | tcp_established  | " \
+  "tcp_transitory  | icmp ]")                          \
+_(snat_det_get_timeouts, "")                                     \
+_(snat_det_close_session_out, ": "           \
+  ":")                                       \
+_(snat_det_close_session_in, ": "              \
+  ":")                                       \
+_(snat_det_session_dump, "ip_address ")
+
+static void 
+snat_vat_api_hookup (vat_main_t *vam)
+{
+  snat_test_main_t * sm __attribute__((unused)) = &snat_test_main;
+  /* Hook up handlers for replies from the data plane plug-in */
+#define _(N,n)                                                  \
+  vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),       \
+                          #n,                                   \
+                          vl_api_##n##_t_handler,               \
+                          vl_noop_handler,                      \
+                          vl_api_##n##_t_endian,                \
+                          vl_api_##n##_t_print,                 \
+                          sizeof(vl_api_##n##_t), 1); 
+  foreach_vpe_api_reply_msg;
+#undef _
+
+  /* API messages we can send */
+#define _(n,h)                                          \
+  hash_set_mem (vam->function_by_name, #n, api_##n);
+  foreach_vpe_api_msg;
+#undef _    
+    
+  /* Help strings */
+#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
+  foreach_vpe_api_msg;
+#undef _
+}
+
+clib_error_t * vat_plugin_register (vat_main_t *vam)
+{
+  snat_test_main_t * sm = &snat_test_main;
+  u8 * name;
+
+  sm->vat_main = vam;
+
+  /* Ask the vpp engine for the first assigned message-id */
+  name = format (0, "snat_%08x%c", api_version, 0);
+  sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
+
+  if (sm->msg_id_base != (u16) ~0)
+    snat_vat_api_hookup (vam);
+  
+  vec_free(name);
+  
+  return 0;
+}
diff --git a/src/plugins/nat/out2in.c b/src/plugins/nat/out2in.c
new file mode 100644
index 00000000..67950066
--- /dev/null
+++ b/src/plugins/nat/out2in.c
@@ -0,0 +1,2294 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+typedef struct {
+  u32 sw_if_index;
+  u32 next_index;
+  u32 session_index;
+} snat_out2in_trace_t;
+
+typedef struct {
+  u32 next_worker_index;
+  u8 do_handoff;
+} snat_out2in_worker_handoff_trace_t;
+
+/* packet trace format function */
+static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
+
+  s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
+              t->sw_if_index, t->next_index, t->session_index);
+  return s;
+}
+
+static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
+
+  s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
+              t->sw_if_index, t->next_index);
+  return s;
+}
+
+static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  snat_out2in_worker_handoff_trace_t * t =
+    va_arg (*args, snat_out2in_worker_handoff_trace_t *);
+  char * m;
+
+  m = t->do_handoff ? "next worker" : "same worker";
+  s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
+
+  return s;
+}
+
+vlib_node_registration_t snat_out2in_node;
+vlib_node_registration_t snat_out2in_fast_node;
+vlib_node_registration_t snat_out2in_worker_handoff_node;
+vlib_node_registration_t snat_det_out2in_node;
+
+#define foreach_snat_out2in_error                       \
+_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
+_(OUT2IN_PACKETS, "Good out2in packets processed")      \
+_(BAD_ICMP_TYPE, "unsupported ICMP type")               \
+_(NO_TRANSLATION, "No translation")
+
+typedef enum {
+#define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
+  foreach_snat_out2in_error
+#undef _
+  SNAT_OUT2IN_N_ERROR,
+} snat_out2in_error_t;
+
+static char * snat_out2in_error_strings[] = {
+#define _(sym,string) string,
+  foreach_snat_out2in_error
+#undef _
+};
+
+typedef enum {
+  SNAT_OUT2IN_NEXT_DROP,
+  SNAT_OUT2IN_NEXT_LOOKUP,
+  SNAT_OUT2IN_NEXT_ICMP_ERROR,
+  SNAT_OUT2IN_N_NEXT,
+} snat_out2in_next_t;
+
+/**
+ * @brief Create session for static mapping.
+ *
+ * Create NAT session initiated by host from external network with static
+ * mapping.
+ *
+ * @param sm     NAT main.
+ * @param b0     Vlib buffer.
+ * @param in2out In2out NAT44 session key.
+ * @param out2in Out2in NAT44 session key.
+ * @param node   Vlib node.
+ *
+ * @returns SNAT session if successfully created otherwise 0.
+ */
+static inline snat_session_t *
+create_session_for_static_mapping (snat_main_t *sm,
+                                   vlib_buffer_t *b0,
+                                   snat_session_key_t in2out,
+                                   snat_session_key_t out2in,
+                                   vlib_node_runtime_t * node,
+                                   u32 thread_index)
+{
+  snat_user_t *u;
+  snat_user_key_t user_key;
+  snat_session_t *s;
+  clib_bihash_kv_8_8_t kv0, value0;
+  dlist_elt_t * per_user_translation_list_elt;
+  dlist_elt_t * per_user_list_head_elt;
+  ip4_header_t *ip0;
+
+  ip0 = vlib_buffer_get_current (b0);
+
+  user_key.addr = in2out.addr;
+  user_key.fib_index = in2out.fib_index;
+  kv0.key = user_key.as_u64;
+
+  /* Ever heard of the "user" = inside ip4 address before? */
+  if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
+    {
+      /* no, make a new one */
+      pool_get (sm->per_thread_data[thread_index].users, u);
+      memset (u, 0, sizeof (*u));
+      u->addr = in2out.addr;
+      u->fib_index = in2out.fib_index;
+
+      pool_get (sm->per_thread_data[thread_index].list_pool,
+                per_user_list_head_elt);
+
+      u->sessions_per_user_list_head_index = per_user_list_head_elt -
+        sm->per_thread_data[thread_index].list_pool;
+
+      clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
+                       u->sessions_per_user_list_head_index);
+
+      kv0.value = u - sm->per_thread_data[thread_index].users;
+
+      /* add user */
+      clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
+
+      /* add non-traslated packets worker lookup */
+      kv0.value = thread_index;
+      clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
+    }
+  else
+    {
+      u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
+                             value0.value);
+    }
+
+  pool_get (sm->per_thread_data[thread_index].sessions, s);
+  memset (s, 0, sizeof (*s));
+
+  s->outside_address_index = ~0;
+  s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
+  u->nstaticsessions++;
+
+  /* Create list elts */
+  pool_get (sm->per_thread_data[thread_index].list_pool,
+            per_user_translation_list_elt);
+  clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
+                   per_user_translation_list_elt -
+                   sm->per_thread_data[thread_index].list_pool);
+
+  per_user_translation_list_elt->value =
+    s - sm->per_thread_data[thread_index].sessions;
+  s->per_user_index =
+    per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
+  s->per_user_list_head_index = u->sessions_per_user_list_head_index;
+
+  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                      s->per_user_list_head_index,
+                      per_user_translation_list_elt -
+                      sm->per_thread_data[thread_index].list_pool);
+
+  s->in2out = in2out;
+  s->out2in = out2in;
+  s->in2out.protocol = out2in.protocol;
+
+  /* Add to translation hashes */
+  kv0.key = s->in2out.as_u64;
+  kv0.value = s - sm->per_thread_data[thread_index].sessions;
+  if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
+      clib_warning ("in2out key add failed");
+
+  kv0.key = s->out2in.as_u64;
+  kv0.value = s - sm->per_thread_data[thread_index].sessions;
+
+  if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
+      clib_warning ("out2in key add failed");
+
+  /* log NAT event */
+  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
+                                      s->out2in.addr.as_u32,
+                                      s->in2out.protocol,
+                                      s->in2out.port,
+                                      s->out2in.port,
+                                      s->in2out.fib_index);
+   return s;
+}
+
+static_always_inline
+snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
+                                 snat_session_key_t *p_key0)
+{
+  icmp46_header_t *icmp0;
+  snat_session_key_t key0;
+  icmp_echo_header_t *echo0, *inner_echo0 = 0;
+  ip4_header_t *inner_ip0;
+  void *l4_header = 0;
+  icmp46_header_t *inner_icmp0;
+
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  echo0 = (icmp_echo_header_t *)(icmp0+1);
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      key0.protocol = SNAT_PROTOCOL_ICMP;
+      key0.addr = ip0->dst_address;
+      key0.port = echo0->identifier;
+    }
+  else
+    {
+      inner_ip0 = (ip4_header_t *)(echo0+1);
+      l4_header = ip4_next_header (inner_ip0);
+      key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
+      key0.addr = inner_ip0->src_address;
+      switch (key0.protocol)
+        {
+        case SNAT_PROTOCOL_ICMP:
+          inner_icmp0 = (icmp46_header_t*)l4_header;
+          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
+          key0.port = inner_echo0->identifier;
+          break;
+        case SNAT_PROTOCOL_UDP:
+        case SNAT_PROTOCOL_TCP:
+          key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
+          break;
+        default:
+          return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
+        }
+    }
+  *p_key0 = key0;
+  return -1; /* success */
+}
+
+/**
+ * Get address and port values to be used for ICMP packet translation
+ * and create session if needed
+ *
+ * @param[in,out] sm             NAT main
+ * @param[in,out] node           NAT node runtime
+ * @param[in] thread_index       thread index
+ * @param[in,out] b0             buffer containing packet to be translated
+ * @param[out] p_proto           protocol used for matching
+ * @param[out] p_value           address and port after NAT translation
+ * @param[out] p_dont_translate  if packet should not be translated
+ * @param d                      optional parameter
+ * @param e                      optional parameter
+ */
+u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e)
+{
+  ip4_header_t *ip0;
+  icmp46_header_t *icmp0;
+  u32 sw_if_index0;
+  u32 rx_fib_index0;
+  snat_session_key_t key0;
+  snat_session_key_t sm0;
+  snat_session_t *s0 = 0;
+  u8 dont_translate = 0;
+  clib_bihash_kv_8_8_t kv0, value0;
+  u8 is_addr_only;
+  u32 next0 = ~0;
+  int err;
+
+  ip0 = vlib_buffer_get_current (b0);
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+
+  key0.protocol = 0;
+
+  err = icmp_get_key (ip0, &key0);
+  if (err != -1)
+    {
+      b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out;
+    }
+  key0.fib_index = rx_fib_index0;
+
+  kv0.key = key0.as_u64;
+
+  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
+    {
+      /* Try to match static mapping by external address and port,
+         destination address and port in packet */
+      if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
+        {
+          /* Don't NAT packet aimed at the intfc address */
+          if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
+                                              ip0->dst_address.as_u32)))
+            {
+              dont_translate = 1;
+              goto out;
+            }
+          b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+          next0 = SNAT_OUT2IN_NEXT_DROP;
+          goto out;
+        }
+
+      if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
+                        (icmp0->type != ICMP4_echo_request || !is_addr_only)))
+        {
+          b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
+          next0 = SNAT_OUT2IN_NEXT_DROP;
+          goto out;
+        }
+
+      /* Create session initiated by host from external network */
+      s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
+                                             node, thread_index);
+
+      if (!s0)
+        {
+          next0 = SNAT_OUT2IN_NEXT_DROP;
+          goto out;
+        }
+    }
+  else
+    {
+      if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
+                        icmp0->type != ICMP4_echo_request &&
+                        !icmp_is_error_message (icmp0)))
+        {
+          b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
+          next0 = SNAT_OUT2IN_NEXT_DROP;
+          goto out;
+        }
+
+      s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                              value0.value);
+    }
+
+out:
+  *p_proto = key0.protocol;
+  if (s0)
+    *p_value = s0->in2out;
+  *p_dont_translate = dont_translate;
+  if (d)
+    *(snat_session_t**)d = s0;
+  return next0;
+}
+
+/**
+ * Get address and port values to be used for ICMP packet translation
+ *
+ * @param[in] sm                 NAT main
+ * @param[in,out] node           NAT node runtime
+ * @param[in] thread_index       thread index
+ * @param[in,out] b0             buffer containing packet to be translated
+ * @param[out] p_proto           protocol used for matching
+ * @param[out] p_value           address and port after NAT translation
+ * @param[out] p_dont_translate  if packet should not be translated
+ * @param d                      optional parameter
+ * @param e                      optional parameter
+ */
+u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
+                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                           snat_session_key_t *p_value,
+                           u8 *p_dont_translate, void *d, void *e)
+{
+  ip4_header_t *ip0;
+  icmp46_header_t *icmp0;
+  u32 sw_if_index0;
+  u32 rx_fib_index0;
+  snat_session_key_t key0;
+  snat_session_key_t sm0;
+  u8 dont_translate = 0;
+  u8 is_addr_only;
+  u32 next0 = ~0;
+  int err;
+
+  ip0 = vlib_buffer_get_current (b0);
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+
+  err = icmp_get_key (ip0, &key0);
+  if (err != -1)
+    {
+      b0->error = node->errors[err];
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out2;
+    }
+  key0.fib_index = rx_fib_index0;
+
+  if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
+    {
+      /* Don't NAT packet aimed at the intfc address */
+      if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out;
+    }
+
+  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
+                    (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
+                    !icmp_is_error_message (icmp0)))
+    {
+      b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out;
+    }
+
+out:
+  *p_value = sm0;
+out2:
+  *p_proto = key0.protocol;
+  *p_dont_translate = dont_translate;
+  return next0;
+}
+
+static inline u32 icmp_out2in (snat_main_t *sm,
+                               vlib_buffer_t * b0,
+                               ip4_header_t * ip0,
+                               icmp46_header_t * icmp0,
+                               u32 sw_if_index0,
+                               u32 rx_fib_index0,
+                               vlib_node_runtime_t * node,
+                               u32 next0,
+                               u32 thread_index,
+                               void *d,
+                               void *e)
+{
+  snat_session_key_t sm0;
+  u8 protocol;
+  icmp_echo_header_t *echo0, *inner_echo0 = 0;
+  ip4_header_t *inner_ip0 = 0;
+  void *l4_header = 0;
+  icmp46_header_t *inner_icmp0;
+  u8 dont_translate;
+  u32 new_addr0, old_addr0;
+  u16 old_id0, new_id0;
+  ip_csum_t sum0;
+  u16 checksum0;
+  u32 next0_tmp;
+
+  echo0 = (icmp_echo_header_t *)(icmp0+1);
+
+  next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
+                                       &protocol, &sm0, &dont_translate, d, e);
+  if (next0_tmp != ~0)
+    next0 = next0_tmp;
+  if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
+    goto out;
+
+  sum0 = ip_incremental_checksum (0, icmp0,
+                                  ntohs(ip0->length) - ip4_header_bytes (ip0));
+  checksum0 = ~ip_csum_fold (sum0);
+  if (checksum0 != 0 && checksum0 != 0xffff)
+    {
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out;
+    }
+
+  old_addr0 = ip0->dst_address.as_u32;
+  new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
+  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+
+  sum0 = ip0->checksum;
+  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
+                         dst_address /* changed member */);
+  ip0->checksum = ip_csum_fold (sum0);
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      new_id0 = sm0.port;
+      if (PREDICT_FALSE(new_id0 != echo0->identifier))
+        {
+          old_id0 = echo0->identifier;
+          new_id0 = sm0.port;
+          echo0->identifier = new_id0;
+
+          sum0 = icmp0->checksum;
+          sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
+                                 identifier /* changed member */);
+          icmp0->checksum = ip_csum_fold (sum0);
+        }
+    }
+  else
+    {
+      inner_ip0 = (ip4_header_t *)(echo0+1);
+      l4_header = ip4_next_header (inner_ip0);
+
+      if (!ip4_header_checksum_is_valid (inner_ip0))
+        {
+          next0 = SNAT_OUT2IN_NEXT_DROP;
+          goto out;
+        }
+
+      old_addr0 = inner_ip0->src_address.as_u32;
+      inner_ip0->src_address = sm0.addr;
+      new_addr0 = inner_ip0->src_address.as_u32;
+
+      sum0 = icmp0->checksum;
+      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
+                             src_address /* changed member */);
+      icmp0->checksum = ip_csum_fold (sum0);
+
+      switch (protocol)
+        {
+        case SNAT_PROTOCOL_ICMP:
+          inner_icmp0 = (icmp46_header_t*)l4_header;
+          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
+
+          old_id0 = inner_echo0->identifier;
+          new_id0 = sm0.port;
+          inner_echo0->identifier = new_id0;
+
+          sum0 = icmp0->checksum;
+          sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
+                                 identifier);
+          icmp0->checksum = ip_csum_fold (sum0);
+          break;
+        case SNAT_PROTOCOL_UDP:
+        case SNAT_PROTOCOL_TCP:
+          old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
+          new_id0 = sm0.port;
+          ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
+
+          sum0 = icmp0->checksum;
+          sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
+                                 src_port);
+          icmp0->checksum = ip_csum_fold (sum0);
+          break;
+        default:
+          ASSERT(0);
+        }
+    }
+
+out:
+  return next0;
+}
+
+
+static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
+                                         vlib_buffer_t * b0,
+                                         ip4_header_t * ip0,
+                                         icmp46_header_t * icmp0,
+                                         u32 sw_if_index0,
+                                         u32 rx_fib_index0,
+                                         vlib_node_runtime_t * node,
+                                         u32 next0, f64 now,
+                                         u32 thread_index,
+                                         snat_session_t ** p_s0)
+{
+  next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                      next0, thread_index, p_s0, 0);
+  snat_session_t * s0 = *p_s0;
+  if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
+    {
+      /* Accounting */
+      s0->last_heard = now;
+      s0->total_pkts++;
+      s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
+      /* Per-user LRU list maintenance for dynamic translation */
+      if (!snat_is_session_static (s0))
+        {
+          clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                             s0->per_user_index);
+          clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                              s0->per_user_list_head_index,
+                              s0->per_user_index);
+        }
+    }
+  return next0;
+}
+
+static void
+snat_out2in_unknown_proto (snat_main_t *sm,
+                           vlib_buffer_t * b,
+                           ip4_header_t * ip,
+                           u32 rx_fib_index,
+                           u32 thread_index,
+                           f64 now,
+                           vlib_main_t * vm)
+{
+  clib_bihash_kv_8_8_t kv, value;
+  clib_bihash_kv_16_8_t s_kv, s_value;
+  snat_static_mapping_t *m;
+  snat_session_key_t m_key;
+  u32 old_addr, new_addr;
+  ip_csum_t sum;
+  snat_unk_proto_ses_key_t key;
+  snat_session_t * s;
+  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
+  snat_user_key_t u_key;
+  snat_user_t *u;
+  dlist_elt_t *head, *elt;
+
+  old_addr = ip->dst_address.as_u32;
+
+  key.l_addr = ip->dst_address;
+  key.r_addr = ip->src_address;
+  key.fib_index = rx_fib_index;
+  key.proto = ip->protocol;
+  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
+  s_kv.key[0] = key.as_u64[0];
+  s_kv.key[1] = key.as_u64[1];
+
+  if (!clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+    {
+      s = pool_elt_at_index (tsm->sessions, s_value.value);
+      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
+    }
+  else
+    {
+      m_key.addr = ip->dst_address;
+      m_key.port = 0;
+      m_key.protocol = 0;
+      m_key.fib_index = rx_fib_index;
+      kv.key = m_key.as_u64;
+      if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+        return;
+
+      m = pool_elt_at_index (sm->static_mappings, value.value);
+
+      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
+
+      u_key.addr = ip->src_address;
+      u_key.fib_index = m->fib_index;
+      kv.key = u_key.as_u64;
+
+      /* Ever heard of the "user" = src ip4 address before? */
+      if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+        {
+          /* no, make a new one */
+          pool_get (tsm->users, u);
+          memset (u, 0, sizeof (*u));
+          u->addr = ip->src_address;
+          u->fib_index = rx_fib_index;
+
+          pool_get (tsm->list_pool, head);
+          u->sessions_per_user_list_head_index = head - tsm->list_pool;
+
+          clib_dlist_init (tsm->list_pool,
+                           u->sessions_per_user_list_head_index);
+
+          kv.value = u - tsm->users;
+
+          /* add user */
+          clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
+        }
+      else
+        {
+          u = pool_elt_at_index (tsm->users, value.value);
+        }
+
+      /* Create a new session */
+      pool_get (tsm->sessions, s);
+      memset (s, 0, sizeof (*s));
+
+      s->ext_host_addr.as_u32 = ip->src_address.as_u32;
+      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
+      s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+      s->outside_address_index = ~0;
+      s->out2in.addr.as_u32 = old_addr;
+      s->out2in.fib_index = rx_fib_index;
+      s->in2out.addr.as_u32 = new_addr;
+      s->in2out.fib_index = m->fib_index;
+      s->in2out.port = s->out2in.port = ip->protocol;
+      u->nstaticsessions++;
+
+      /* Create list elts */
+      pool_get (tsm->list_pool, elt);
+      clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
+      elt->value = s - tsm->sessions;
+      s->per_user_index = elt - tsm->list_pool;
+      s->per_user_list_head_index = u->sessions_per_user_list_head_index;
+      clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                          s->per_user_index);
+
+      /* Add to lookup tables */
+      s_kv.value = s - tsm->sessions;
+      if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
+        clib_warning ("out2in key add failed");
+
+      key.l_addr = ip->dst_address;
+      key.fib_index = m->fib_index;
+      s_kv.key[0] = key.as_u64[0];
+      s_kv.key[1] = key.as_u64[1];
+      if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
+        clib_warning ("in2out key add failed");
+   }
+
+  /* Update IP checksum */
+  sum = ip->checksum;
+  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
+  ip->checksum = ip_csum_fold (sum);
+
+  vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
+
+  /* Accounting */
+  s->last_heard = now;
+  s->total_pkts++;
+  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
+  /* Per-user LRU list maintenance */
+  clib_dlist_remove (tsm->list_pool, s->per_user_index);
+  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                      s->per_user_index);
+}
+
+static uword
+snat_out2in_node_fn (vlib_main_t * vm,
+		  vlib_node_runtime_t * node,
+		  vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_out2in_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+  f64 now = vlib_time_now (vm);
+  u32 thread_index = vlib_get_thread_index ();
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
+          u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
+          u32 sw_if_index0, sw_if_index1;
+          ip4_header_t * ip0, *ip1;
+          ip_csum_t sum0, sum1;
+          u32 new_addr0, old_addr0;
+          u16 new_port0, old_port0;
+          u32 new_addr1, old_addr1;
+          u16 new_port1, old_port1;
+          udp_header_t * udp0, * udp1;
+          tcp_header_t * tcp0, * tcp1;
+          icmp46_header_t * icmp0, * icmp1;
+          snat_session_key_t key0, key1, sm0, sm1;
+          u32 rx_fib_index0, rx_fib_index1;
+          u32 proto0, proto1;
+          snat_session_t * s0 = 0, * s1 = 0;
+          clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+          /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          vnet_buffer (b0)->snat.flags = 0;
+          vnet_buffer (b1)->snat.flags = 0;
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+          icmp0 = (icmp46_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
+                                   sw_if_index0);
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace0;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE (proto0 == ~0))
+            {
+              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
+                                        thread_index, now, vm);
+              goto trace0;
+            }
+
+          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              next0 = icmp_out2in_slow_path
+                (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                 next0, now, thread_index, &s0);
+              goto trace0;
+            }
+
+          key0.addr = ip0->dst_address;
+          key0.port = udp0->dst_port;
+          key0.protocol = proto0;
+          key0.fib_index = rx_fib_index0;
+
+          kv0.key = key0.as_u64;
+
+          if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
+            {
+              /* Try to match static mapping by external address and port,
+                 destination address and port in packet */
+              if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
+                {
+                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+                  /*
+                   * Send DHCP packets to the ipv4 stack, or we won't
+                   * be able to use dhcp client on the outside interface
+                   */
+                  if (proto0 != SNAT_PROTOCOL_UDP
+                      || (udp0->dst_port
+                          != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
+                    next0 = SNAT_OUT2IN_NEXT_DROP;
+                  goto trace0;
+                }
+
+              /* Create session initiated by host from external network */
+              s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
+                                                     thread_index);
+              if (!s0)
+                {
+                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+                  next0 = SNAT_OUT2IN_NEXT_DROP;
+                  goto trace0;
+                }
+            }
+          else
+            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                    value0.value);
+
+          old_addr0 = ip0->dst_address.as_u32;
+          ip0->dst_address = s0->in2out.addr;
+          new_addr0 = ip0->dst_address.as_u32;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              old_port0 = tcp0->dst_port;
+              tcp0->dst_port = s0->in2out.port;
+              new_port0 = tcp0->dst_port;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              old_port0 = udp0->dst_port;
+              udp0->dst_port = s0->in2out.port;
+              udp0->checksum = 0;
+            }
+
+          /* Accounting */
+          s0->last_heard = now;
+          s0->total_pkts++;
+          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
+          /* Per-user LRU list maintenance for dynamic translation */
+          if (!snat_is_session_static (s0))
+            {
+              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                                 s0->per_user_index);
+              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                                  s0->per_user_list_head_index,
+                                  s0->per_user_index);
+            }
+        trace0:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+              t->session_index = ~0;
+              if (s0)
+                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
+            }
+
+          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+
+
+          ip1 = vlib_buffer_get_current (b1);
+          udp1 = ip4_next_header (ip1);
+          tcp1 = (tcp_header_t *) udp1;
+          icmp1 = (icmp46_header_t *) udp1;
+
+          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
+	  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
+                                   sw_if_index1);
+
+          if (PREDICT_FALSE(ip1->ttl == 1))
+            {
+              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace1;
+            }
+
+          proto1 = ip_proto_to_snat_proto (ip1->protocol);
+
+          if (PREDICT_FALSE (proto1 == ~0))
+            {
+              snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
+                                        thread_index, now, vm);
+              goto trace1;
+            }
+
+          if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
+            {
+              next1 = icmp_out2in_slow_path
+                (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
+                 next1, now, thread_index, &s1);
+              goto trace1;
+            }
+
+          key1.addr = ip1->dst_address;
+          key1.port = udp1->dst_port;
+          key1.protocol = proto1;
+          key1.fib_index = rx_fib_index1;
+
+          kv1.key = key1.as_u64;
+
+          if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
+            {
+              /* Try to match static mapping by external address and port,
+                 destination address and port in packet */
+              if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
+                {
+                  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+                  /*
+                   * Send DHCP packets to the ipv4 stack, or we won't
+                   * be able to use dhcp client on the outside interface
+                   */
+                  if (proto1 != SNAT_PROTOCOL_UDP
+                      || (udp1->dst_port
+                          != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
+                    next1 = SNAT_OUT2IN_NEXT_DROP;
+                  goto trace1;
+                }
+
+              /* Create session initiated by host from external network */
+              s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
+                                                     thread_index);
+              if (!s1)
+                {
+                  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+                  next1 = SNAT_OUT2IN_NEXT_DROP;
+                  goto trace1;
+                }
+            }
+          else
+            s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                    value1.value);
+
+          old_addr1 = ip1->dst_address.as_u32;
+          ip1->dst_address = s1->in2out.addr;
+          new_addr1 = ip1->dst_address.as_u32;
+          vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
+
+          sum1 = ip1->checksum;
+          sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip1->checksum = ip_csum_fold (sum1);
+
+          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
+            {
+              old_port1 = tcp1->dst_port;
+              tcp1->dst_port = s1->in2out.port;
+              new_port1 = tcp1->dst_port;
+
+              sum1 = tcp1->checksum;
+              sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+
+              sum1 = ip_csum_update (sum1, old_port1, new_port1,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp1->checksum = ip_csum_fold(sum1);
+            }
+          else
+            {
+              old_port1 = udp1->dst_port;
+              udp1->dst_port = s1->in2out.port;
+              udp1->checksum = 0;
+            }
+
+          /* Accounting */
+          s1->last_heard = now;
+          s1->total_pkts++;
+          s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
+          /* Per-user LRU list maintenance for dynamic translation */
+          if (!snat_is_session_static (s1))
+            {
+              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                                 s1->per_user_index);
+              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                                  s1->per_user_list_head_index,
+                                  s1->per_user_index);
+            }
+        trace1:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b1->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b1, sizeof (*t));
+              t->sw_if_index = sw_if_index1;
+              t->next_index = next1;
+              t->session_index = ~0;
+              if (s1)
+                t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
+            }
+
+          pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
+
+          /* verify speculative enqueues, maybe switch current next frame */
+          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+                                           to_next, n_left_to_next,
+                                           bi0, bi1, next0, next1);
+        }
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
+          u32 sw_if_index0;
+          ip4_header_t * ip0;
+          ip_csum_t sum0;
+          u32 new_addr0, old_addr0;
+          u16 new_port0, old_port0;
+          udp_header_t * udp0;
+          tcp_header_t * tcp0;
+          icmp46_header_t * icmp0;
+          snat_session_key_t key0, sm0;
+          u32 rx_fib_index0;
+          u32 proto0;
+          snat_session_t * s0 = 0;
+          clib_bihash_kv_8_8_t kv0, value0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+          vnet_buffer (b0)->snat.flags = 0;
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+          icmp0 = (icmp46_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
+                                   sw_if_index0);
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE (proto0 == ~0))
+            {
+              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
+                                        thread_index, now, vm);
+              goto trace00;
+            }
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace00;
+            }
+
+          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              next0 = icmp_out2in_slow_path
+                (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
+                 next0, now, thread_index, &s0);
+              goto trace00;
+            }
+
+          key0.addr = ip0->dst_address;
+          key0.port = udp0->dst_port;
+          key0.protocol = proto0;
+          key0.fib_index = rx_fib_index0;
+
+          kv0.key = key0.as_u64;
+
+          if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
+            {
+              /* Try to match static mapping by external address and port,
+                 destination address and port in packet */
+              if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
+                {
+                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+                  /*
+                   * Send DHCP packets to the ipv4 stack, or we won't
+                   * be able to use dhcp client on the outside interface
+                   */
+                  if (proto0 != SNAT_PROTOCOL_UDP
+                      || (udp0->dst_port
+                          != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
+
+                    next0 = SNAT_OUT2IN_NEXT_DROP;
+                  goto trace00;
+                }
+
+              /* Create session initiated by host from external network */
+              s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
+                                                     thread_index);
+              if (!s0)
+                {
+                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+                    next0 = SNAT_OUT2IN_NEXT_DROP;
+                  goto trace00;
+                }
+            }
+          else
+            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
+                                    value0.value);
+
+          old_addr0 = ip0->dst_address.as_u32;
+          ip0->dst_address = s0->in2out.addr;
+          new_addr0 = ip0->dst_address.as_u32;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              old_port0 = tcp0->dst_port;
+              tcp0->dst_port = s0->in2out.port;
+              new_port0 = tcp0->dst_port;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              old_port0 = udp0->dst_port;
+              udp0->dst_port = s0->in2out.port;
+              udp0->checksum = 0;
+            }
+
+          /* Accounting */
+          s0->last_heard = now;
+          s0->total_pkts++;
+          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
+          /* Per-user LRU list maintenance for dynamic translation */
+          if (!snat_is_session_static (s0))
+            {
+              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
+                                 s0->per_user_index);
+              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
+                                  s0->per_user_list_head_index,
+                                  s0->per_user_index);
+            }
+        trace00:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+              t->session_index = ~0;
+              if (s0)
+                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
+            }
+
+          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, snat_out2in_node.index,
+                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_out2in_node) = {
+  .function = snat_out2in_node_fn,
+  .name = "nat44-out2in",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_out2in_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
+  .error_strings = snat_out2in_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
+    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
+
+/**************************/
+/*** deterministic mode ***/
+/**************************/
+static uword
+snat_det_out2in_node_fn (vlib_main_t * vm,
+                         vlib_node_runtime_t * node,
+                         vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_out2in_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+  u32 thread_index = vlib_get_thread_index ();
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+        {
+          u32 bi0, bi1;
+	  vlib_buffer_t * b0, * b1;
+          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
+          u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
+          u32 sw_if_index0, sw_if_index1;
+          ip4_header_t * ip0, * ip1;
+          ip_csum_t sum0, sum1;
+          ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
+          u16 new_port0, old_port0, old_port1, new_port1;
+          udp_header_t * udp0, * udp1;
+          tcp_header_t * tcp0, * tcp1;
+          u32 proto0, proto1;
+          snat_det_out_key_t key0, key1;
+          snat_det_map_t * dm0, * dm1;
+          snat_det_session_t * ses0 = 0, * ses1 = 0;
+          u32 rx_fib_index0, rx_fib_index1;
+          icmp46_header_t * icmp0, * icmp1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t * p2, * p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+          /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace0;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+              icmp0 = (icmp46_header_t *) udp0;
+
+              next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
+                                  rx_fib_index0, node, next0, thread_index,
+                                  &ses0, &dm0);
+              goto trace0;
+            }
+
+          key0.ext_host_addr = ip0->src_address;
+          key0.ext_host_port = tcp0->src;
+          key0.out_port = tcp0->dst;
+
+          dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
+          if (PREDICT_FALSE(!dm0))
+            {
+              clib_warning("unknown dst address:  %U",
+                           format_ip4_address, &ip0->dst_address);
+              next0 = SNAT_OUT2IN_NEXT_DROP;
+              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace0;
+            }
+
+          snat_det_reverse(dm0, &ip0->dst_address,
+                           clib_net_to_host_u16(tcp0->dst), &new_addr0);
+
+          ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
+          if (PREDICT_FALSE(!ses0))
+            {
+              clib_warning("no match src %U:%d dst %U:%d for user %U",
+                           format_ip4_address, &ip0->src_address,
+                           clib_net_to_host_u16 (tcp0->src),
+                           format_ip4_address, &ip0->dst_address,
+                           clib_net_to_host_u16 (tcp0->dst),
+                           format_ip4_address, &new_addr0);
+              next0 = SNAT_OUT2IN_NEXT_DROP;
+              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace0;
+            }
+          new_port0 = ses0->in_port;
+
+          old_addr0 = ip0->dst_address;
+          ip0->dst_address = new_addr0;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
+                ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
+              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
+                snat_det_ses_close(dm0, ses0);
+
+              old_port0 = tcp0->dst;
+              tcp0->dst = new_port0;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              old_port0 = udp0->dst_port;
+              udp0->dst_port = new_port0;
+              udp0->checksum = 0;
+            }
+
+        trace0:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+              t->session_index = ~0;
+              if (ses0)
+                t->session_index = ses0 - dm0->sessions;
+            }
+
+          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+
+	  b1 = vlib_get_buffer (vm, bi1);
+
+          ip1 = vlib_buffer_get_current (b1);
+          udp1 = ip4_next_header (ip1);
+          tcp1 = (tcp_header_t *) udp1;
+
+          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE(ip1->ttl == 1))
+            {
+              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace1;
+            }
+
+          proto1 = ip_proto_to_snat_proto (ip1->protocol);
+
+          if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
+            {
+              rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
+              icmp1 = (icmp46_header_t *) udp1;
+
+              next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
+                                  rx_fib_index1, node, next1, thread_index,
+                                  &ses1, &dm1);
+              goto trace1;
+            }
+
+          key1.ext_host_addr = ip1->src_address;
+          key1.ext_host_port = tcp1->src;
+          key1.out_port = tcp1->dst;
+
+          dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
+          if (PREDICT_FALSE(!dm1))
+            {
+              clib_warning("unknown dst address:  %U",
+                           format_ip4_address, &ip1->dst_address);
+              next1 = SNAT_OUT2IN_NEXT_DROP;
+              b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace1;
+            }
+
+          snat_det_reverse(dm1, &ip1->dst_address,
+                           clib_net_to_host_u16(tcp1->dst), &new_addr1);
+
+          ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
+          if (PREDICT_FALSE(!ses1))
+            {
+              clib_warning("no match src %U:%d dst %U:%d for user %U",
+                           format_ip4_address, &ip1->src_address,
+                           clib_net_to_host_u16 (tcp1->src),
+                           format_ip4_address, &ip1->dst_address,
+                           clib_net_to_host_u16 (tcp1->dst),
+                           format_ip4_address, &new_addr1);
+              next1 = SNAT_OUT2IN_NEXT_DROP;
+              b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace1;
+            }
+          new_port1 = ses1->in_port;
+
+          old_addr1 = ip1->dst_address;
+          ip1->dst_address = new_addr1;
+          vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
+
+          sum1 = ip1->checksum;
+          sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip1->checksum = ip_csum_fold (sum1);
+
+          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
+            {
+              if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
+                ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
+              else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
+                snat_det_ses_close(dm1, ses1);
+
+              old_port1 = tcp1->dst;
+              tcp1->dst = new_port1;
+
+              sum1 = tcp1->checksum;
+              sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+
+              sum1 = ip_csum_update (sum1, old_port1, new_port1,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp1->checksum = ip_csum_fold(sum1);
+            }
+          else
+            {
+              old_port1 = udp1->dst_port;
+              udp1->dst_port = new_port1;
+              udp1->checksum = 0;
+            }
+
+        trace1:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b1->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b1, sizeof (*t));
+              t->sw_if_index = sw_if_index1;
+              t->next_index = next1;
+              t->session_index = ~0;
+              if (ses1)
+                t->session_index = ses1 - dm1->sessions;
+            }
+
+          pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
+
+          /* verify speculative enqueues, maybe switch current next frame */
+          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+                                           to_next, n_left_to_next,
+                                           bi0, bi1, next0, next1);
+         }
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
+          u32 sw_if_index0;
+          ip4_header_t * ip0;
+          ip_csum_t sum0;
+          ip4_address_t new_addr0, old_addr0;
+          u16 new_port0, old_port0;
+          udp_header_t * udp0;
+          tcp_header_t * tcp0;
+          u32 proto0;
+          snat_det_out_key_t key0;
+          snat_det_map_t * dm0;
+          snat_det_session_t * ses0 = 0;
+          u32 rx_fib_index0;
+          icmp46_header_t * icmp0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace00;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+              icmp0 = (icmp46_header_t *) udp0;
+
+              next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
+                                  rx_fib_index0, node, next0, thread_index,
+                                  &ses0, &dm0);
+              goto trace00;
+            }
+
+          key0.ext_host_addr = ip0->src_address;
+          key0.ext_host_port = tcp0->src;
+          key0.out_port = tcp0->dst;
+
+          dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
+          if (PREDICT_FALSE(!dm0))
+            {
+              clib_warning("unknown dst address:  %U",
+                           format_ip4_address, &ip0->dst_address);
+              next0 = SNAT_OUT2IN_NEXT_DROP;
+              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace00;
+            }
+
+          snat_det_reverse(dm0, &ip0->dst_address,
+                           clib_net_to_host_u16(tcp0->dst), &new_addr0);
+
+          ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
+          if (PREDICT_FALSE(!ses0))
+            {
+              clib_warning("no match src %U:%d dst %U:%d for user %U",
+                           format_ip4_address, &ip0->src_address,
+                           clib_net_to_host_u16 (tcp0->src),
+                           format_ip4_address, &ip0->dst_address,
+                           clib_net_to_host_u16 (tcp0->dst),
+                           format_ip4_address, &new_addr0);
+              next0 = SNAT_OUT2IN_NEXT_DROP;
+              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace00;
+            }
+          new_port0 = ses0->in_port;
+
+          old_addr0 = ip0->dst_address;
+          ip0->dst_address = new_addr0;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+            {
+              if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
+                ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
+              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
+                snat_det_ses_close(dm0, ses0);
+
+              old_port0 = tcp0->dst;
+              tcp0->dst = new_port0;
+
+              sum0 = tcp0->checksum;
+              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
+                                     ip4_header_t,
+                                     dst_address /* changed member */);
+
+              sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                     ip4_header_t /* cheat */,
+                                     length /* changed member */);
+              tcp0->checksum = ip_csum_fold(sum0);
+            }
+          else
+            {
+              old_port0 = udp0->dst_port;
+              udp0->dst_port = new_port0;
+              udp0->checksum = 0;
+            }
+
+        trace00:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+              t->session_index = ~0;
+              if (ses0)
+                t->session_index = ses0 - dm0->sessions;
+            }
+
+          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, snat_det_out2in_node.index,
+                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_det_out2in_node) = {
+  .function = snat_det_out2in_node_fn,
+  .name = "nat44-det-out2in",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_out2in_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
+  .error_strings = snat_out2in_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
+    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
+
+/**
+ * Get address and port values to be used for ICMP packet translation
+ * and create session if needed
+ *
+ * @param[in,out] sm             NAT main
+ * @param[in,out] node           NAT node runtime
+ * @param[in] thread_index       thread index
+ * @param[in,out] b0             buffer containing packet to be translated
+ * @param[out] p_proto           protocol used for matching
+ * @param[out] p_value           address and port after NAT translation
+ * @param[out] p_dont_translate  if packet should not be translated
+ * @param d                      optional parameter
+ * @param e                      optional parameter
+ */
+u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
+                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
+                          snat_session_key_t *p_value,
+                          u8 *p_dont_translate, void *d, void *e)
+{
+  ip4_header_t *ip0;
+  icmp46_header_t *icmp0;
+  u32 sw_if_index0;
+  u8 protocol;
+  snat_det_out_key_t key0;
+  u8 dont_translate = 0;
+  u32 next0 = ~0;
+  icmp_echo_header_t *echo0, *inner_echo0 = 0;
+  ip4_header_t *inner_ip0;
+  void *l4_header = 0;
+  icmp46_header_t *inner_icmp0;
+  snat_det_map_t * dm0 = 0;
+  ip4_address_t new_addr0 = {{0}};
+  snat_det_session_t * ses0 = 0;
+  ip4_address_t out_addr;
+
+  ip0 = vlib_buffer_get_current (b0);
+  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
+  echo0 = (icmp_echo_header_t *)(icmp0+1);
+  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+
+  if (!icmp_is_error_message (icmp0))
+    {
+      protocol = SNAT_PROTOCOL_ICMP;
+      key0.ext_host_addr = ip0->src_address;
+      key0.ext_host_port = 0;
+      key0.out_port = echo0->identifier;
+      out_addr = ip0->dst_address;
+    }
+  else
+    {
+      inner_ip0 = (ip4_header_t *)(echo0+1);
+      l4_header = ip4_next_header (inner_ip0);
+      protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
+      key0.ext_host_addr = inner_ip0->dst_address;
+      out_addr = inner_ip0->src_address;
+      switch (protocol)
+        {
+        case SNAT_PROTOCOL_ICMP:
+          inner_icmp0 = (icmp46_header_t*)l4_header;
+          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
+          key0.ext_host_port = 0;
+          key0.out_port = inner_echo0->identifier;
+          break;
+        case SNAT_PROTOCOL_UDP:
+        case SNAT_PROTOCOL_TCP:
+          key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
+          key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
+          break;
+        default:
+          b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
+          next0 = SNAT_OUT2IN_NEXT_DROP;
+          goto out;
+        }
+    }
+
+  dm0 = snat_det_map_by_out(sm, &out_addr);
+  if (PREDICT_FALSE(!dm0))
+    {
+      /* Don't NAT packet aimed at the intfc address */
+      if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
+                                          ip0->dst_address.as_u32)))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+      clib_warning("unknown dst address:  %U",
+                   format_ip4_address, &ip0->dst_address);
+      goto out;
+    }
+
+  snat_det_reverse(dm0, &ip0->dst_address,
+                   clib_net_to_host_u16(key0.out_port), &new_addr0);
+
+  ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
+  if (PREDICT_FALSE(!ses0))
+    {
+      /* Don't NAT packet aimed at the intfc address */
+      if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
+                                          ip0->dst_address.as_u32)))
+        {
+          dont_translate = 1;
+          goto out;
+        }
+      clib_warning("no match src %U:%d dst %U:%d for user %U",
+                   format_ip4_address, &key0.ext_host_addr,
+                   clib_net_to_host_u16 (key0.ext_host_port),
+                   format_ip4_address, &out_addr,
+                   clib_net_to_host_u16 (key0.out_port),
+                   format_ip4_address, &new_addr0);
+      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out;
+    }
+
+  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
+                    !icmp_is_error_message (icmp0)))
+    {
+      b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
+      next0 = SNAT_OUT2IN_NEXT_DROP;
+      goto out;
+    }
+
+  goto out;
+
+out:
+  *p_proto = protocol;
+  if (ses0)
+    {
+      p_value->addr = new_addr0;
+      p_value->fib_index = sm->inside_fib_index;
+      p_value->port = ses0->in_port;
+    }
+  *p_dont_translate = dont_translate;
+  if (d)
+    *(snat_det_session_t**)d = ses0;
+  if (e)
+    *(snat_det_map_t**)e = dm0;
+  return next0;
+}
+
+/**********************/
+/*** worker handoff ***/
+/**********************/
+static uword
+snat_out2in_worker_handoff_fn (vlib_main_t * vm,
+                               vlib_node_runtime_t * node,
+                               vlib_frame_t * frame)
+{
+  snat_main_t *sm = &snat_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  u32 n_left_from, *from, *to_next = 0;
+  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
+  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
+    = 0;
+  vlib_frame_queue_elt_t *hf = 0;
+  vlib_frame_t *f = 0;
+  int i;
+  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
+  u32 next_worker_index = 0;
+  u32 current_worker_index = ~0;
+  u32 thread_index = vlib_get_thread_index ();
+
+  ASSERT (vec_len (sm->workers));
+
+  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
+    {
+      vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
+
+      vec_validate_init_empty (congested_handoff_queue_by_worker_index,
+			       sm->first_worker_index + sm->num_workers - 1,
+			       (vlib_frame_queue_t *) (~0));
+    }
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+
+  while (n_left_from > 0)
+    {
+      u32 bi0;
+      vlib_buffer_t *b0;
+      u32 sw_if_index0;
+      u32 rx_fib_index0;
+      ip4_header_t * ip0;
+      u8 do_handoff;
+
+      bi0 = from[0];
+      from += 1;
+      n_left_from -= 1;
+
+      b0 = vlib_get_buffer (vm, bi0);
+
+      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
+      rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+
+      ip0 = vlib_buffer_get_current (b0);
+
+      next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
+
+      if (PREDICT_FALSE (next_worker_index != thread_index))
+        {
+          do_handoff = 1;
+
+          if (next_worker_index != current_worker_index)
+            {
+              if (hf)
+                hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
+
+              hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
+                                                      next_worker_index,
+                                                      handoff_queue_elt_by_worker_index);
+
+              n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
+              to_next_worker = &hf->buffer_index[hf->n_vectors];
+              current_worker_index = next_worker_index;
+            }
+
+          /* enqueue to correct worker thread */
+          to_next_worker[0] = bi0;
+          to_next_worker++;
+          n_left_to_next_worker--;
+
+          if (n_left_to_next_worker == 0)
+            {
+              hf->n_vectors = VLIB_FRAME_SIZE;
+              vlib_put_frame_queue_elt (hf);
+              current_worker_index = ~0;
+              handoff_queue_elt_by_worker_index[next_worker_index] = 0;
+              hf = 0;
+            }
+        }
+      else
+        {
+          do_handoff = 0;
+          /* if this is 1st frame */
+          if (!f)
+            {
+              f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
+              to_next = vlib_frame_vector_args (f);
+            }
+
+          to_next[0] = bi0;
+          to_next += 1;
+          f->n_vectors++;
+        }
+
+      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	{
+          snat_out2in_worker_handoff_trace_t *t =
+            vlib_add_trace (vm, node, b0, sizeof (*t));
+          t->next_worker_index = next_worker_index;
+          t->do_handoff = do_handoff;
+        }
+    }
+
+  if (f)
+    vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
+
+  if (hf)
+    hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
+
+  /* Ship frames to the worker nodes */
+  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
+    {
+      if (handoff_queue_elt_by_worker_index[i])
+	{
+	  hf = handoff_queue_elt_by_worker_index[i];
+	  /*
+	   * It works better to let the handoff node
+	   * rate-adapt, always ship the handoff queue element.
+	   */
+	  if (1 || hf->n_vectors == hf->last_n_vectors)
+	    {
+	      vlib_put_frame_queue_elt (hf);
+	      handoff_queue_elt_by_worker_index[i] = 0;
+	    }
+	  else
+	    hf->last_n_vectors = hf->n_vectors;
+	}
+      congested_handoff_queue_by_worker_index[i] =
+	(vlib_frame_queue_t *) (~0);
+    }
+  hf = 0;
+  current_worker_index = ~0;
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
+  .function = snat_out2in_worker_handoff_fn,
+  .name = "nat44-out2in-worker-handoff",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_out2in_worker_handoff_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_next_nodes = 1,
+
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
+
+static uword
+snat_out2in_fast_node_fn (vlib_main_t * vm,
+                          vlib_node_runtime_t * node,
+		          vlib_frame_t * frame)
+{
+  u32 n_left_from, * from, * to_next;
+  snat_out2in_next_t next_index;
+  u32 pkts_processed = 0;
+  snat_main_t * sm = &snat_main;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index,
+			   to_next, n_left_to_next);
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+          u32 bi0;
+	  vlib_buffer_t * b0;
+          u32 next0 = SNAT_OUT2IN_NEXT_DROP;
+          u32 sw_if_index0;
+          ip4_header_t * ip0;
+          ip_csum_t sum0;
+          u32 new_addr0, old_addr0;
+          u16 new_port0, old_port0;
+          udp_header_t * udp0;
+          tcp_header_t * tcp0;
+          icmp46_header_t * icmp0;
+          snat_session_key_t key0, sm0;
+          u32 proto0;
+          u32 rx_fib_index0;
+
+          /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+
+          ip0 = vlib_buffer_get_current (b0);
+          udp0 = ip4_next_header (ip0);
+          tcp0 = (tcp_header_t *) udp0;
+          icmp0 = (icmp46_header_t *) udp0;
+
+          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+	  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
+
+	  vnet_feature_next (sw_if_index0, &next0, b0);
+
+          if (PREDICT_FALSE(ip0->ttl == 1))
+            {
+              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
+                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
+                                           0);
+              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
+              goto trace00;
+            }
+
+          proto0 = ip_proto_to_snat_proto (ip0->protocol);
+
+          if (PREDICT_FALSE (proto0 == ~0))
+              goto trace00;
+
+          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
+            {
+              next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
+                                  rx_fib_index0, node, next0, ~0, 0, 0);
+              goto trace00;
+            }
+
+          key0.addr = ip0->dst_address;
+          key0.port = udp0->dst_port;
+          key0.fib_index = rx_fib_index0;
+
+          if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
+            {
+              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
+              goto trace00;
+            }
+
+          new_addr0 = sm0.addr.as_u32;
+          new_port0 = sm0.port;
+          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
+          old_addr0 = ip0->dst_address.as_u32;
+          ip0->dst_address.as_u32 = new_addr0;
+
+          sum0 = ip0->checksum;
+          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                 ip4_header_t,
+                                 dst_address /* changed member */);
+          ip0->checksum = ip_csum_fold (sum0);
+
+          if (PREDICT_FALSE(new_port0 != udp0->dst_port))
+            {
+               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+                {
+                  old_port0 = tcp0->dst_port;
+                  tcp0->dst_port = new_port0;
+
+                  sum0 = tcp0->checksum;
+                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                         ip4_header_t,
+                                         dst_address /* changed member */);
+
+                  sum0 = ip_csum_update (sum0, old_port0, new_port0,
+                                         ip4_header_t /* cheat */,
+                                         length /* changed member */);
+                  tcp0->checksum = ip_csum_fold(sum0);
+                }
+              else
+                {
+                  old_port0 = udp0->dst_port;
+                  udp0->dst_port = new_port0;
+                  udp0->checksum = 0;
+                }
+            }
+          else
+            {
+              if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
+                {
+                  sum0 = tcp0->checksum;
+                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
+                                         ip4_header_t,
+                                         dst_address /* changed member */);
+
+                  tcp0->checksum = ip_csum_fold(sum0);
+                }
+            }
+
+        trace00:
+
+          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
+                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+            {
+              snat_out2in_trace_t *t =
+                 vlib_add_trace (vm, node, b0, sizeof (*t));
+              t->sw_if_index = sw_if_index0;
+              t->next_index = next0;
+            }
+
+          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
+
+          /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
+                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
+                               pkts_processed);
+  return frame->n_vectors;
+}
+
+VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
+  .function = snat_out2in_fast_node_fn,
+  .name = "nat44-out2in-fast",
+  .vector_size = sizeof (u32),
+  .format_trace = format_snat_out2in_fast_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
+  .error_strings = snat_out2in_error_strings,
+
+  .runtime_data_bytes = sizeof (snat_runtime_t),
+
+  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
+    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
+    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);
diff --git a/src/plugins/snat.am b/src/plugins/snat.am
deleted file mode 100644
index 6b75f3d2..00000000
--- a/src/plugins/snat.am
+++ /dev/null
@@ -1,41 +0,0 @@
-
-# Copyright (c)  
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-vppapitestplugins_LTLIBRARIES += snat_test_plugin.la
-vppplugins_LTLIBRARIES += snat_plugin.la
-
-snat_plugin_la_SOURCES = snat/snat.c		\
-        snat/snat_api.c				\
-        snat/in2out.c				\
-        snat/out2in.c				\
-	snat/snat_plugin.api.h			\
-        snat/snat_ipfix_logging.c		\
-        snat/snat_det.c               		\
-        snat/nat64.c                  		\
-        snat/nat64_cli.c              		\
-        snat/nat64_in2out.c           		\
-        snat/nat64_out2in.c           		\
-        snat/nat64_db.c
-
-API_FILES += snat/snat.api
-
-nobase_apiinclude_HEADERS +=		\
-  snat/snat_all_api_h.h			\
-  snat/snat_msg_enum.h			\
-  snat/snat.api.h
-
-snat_test_plugin_la_SOURCES = \
-  snat/snat_test.c snat/snat_plugin.api.h
-
-# vi:syntax=automake
diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c
deleted file mode 100644
index abe0d9db..00000000
--- a/src/plugins/snat/in2out.c
+++ /dev/null
@@ -1,3454 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-typedef struct {
-  u32 sw_if_index;
-  u32 next_index;
-  u32 session_index;
-  u32 is_slow_path;
-} snat_in2out_trace_t;
-
-typedef struct {
-  u32 next_worker_index;
-  u8 do_handoff;
-} snat_in2out_worker_handoff_trace_t;
-
-/* packet trace format function */
-static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
-  char * tag;
-
-  tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
-  
-  s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
-              t->sw_if_index, t->next_index, t->session_index);
-
-  return s;
-}
-
-static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
-
-  s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d", 
-              t->sw_if_index, t->next_index);
-
-  return s;
-}
-
-static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  snat_in2out_worker_handoff_trace_t * t =
-    va_arg (*args, snat_in2out_worker_handoff_trace_t *);
-  char * m;
-
-  m = t->do_handoff ? "next worker" : "same worker";
-  s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
-
-  return s;
-}
-
-vlib_node_registration_t snat_in2out_node;
-vlib_node_registration_t snat_in2out_slowpath_node;
-vlib_node_registration_t snat_in2out_fast_node;
-vlib_node_registration_t snat_in2out_worker_handoff_node;
-vlib_node_registration_t snat_det_in2out_node;
-vlib_node_registration_t snat_in2out_output_node;
-vlib_node_registration_t snat_in2out_output_slowpath_node;
-vlib_node_registration_t snat_in2out_output_worker_handoff_node;
-vlib_node_registration_t snat_hairpin_dst_node;
-vlib_node_registration_t snat_hairpin_src_node;
-
-
-#define foreach_snat_in2out_error                       \
-_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
-_(IN2OUT_PACKETS, "Good in2out packets processed")      \
-_(OUT_OF_PORTS, "Out of ports")                         \
-_(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
-_(BAD_ICMP_TYPE, "unsupported ICMP type")               \
-_(NO_TRANSLATION, "No translation")
-  
-typedef enum {
-#define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
-  foreach_snat_in2out_error
-#undef _
-  SNAT_IN2OUT_N_ERROR,
-} snat_in2out_error_t;
-
-static char * snat_in2out_error_strings[] = {
-#define _(sym,string) string,
-  foreach_snat_in2out_error
-#undef _
-};
-
-typedef enum {
-  SNAT_IN2OUT_NEXT_LOOKUP,
-  SNAT_IN2OUT_NEXT_DROP,
-  SNAT_IN2OUT_NEXT_ICMP_ERROR,
-  SNAT_IN2OUT_NEXT_SLOW_PATH,
-  SNAT_IN2OUT_N_NEXT,
-} snat_in2out_next_t;
-
-typedef enum {
-  SNAT_HAIRPIN_SRC_NEXT_DROP,
-  SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
-  SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
-  SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
-  SNAT_HAIRPIN_SRC_N_NEXT,
-} snat_hairpin_next_t;
-
-/**
- * @brief Check if packet should be translated
- *
- * Packets aimed at outside interface and external addresss with active session
- * should be translated.
- *
- * @param sm            SNAT main
- * @param rt            SNAT runtime data
- * @param sw_if_index0  index of the inside interface
- * @param ip0           IPv4 header
- * @param proto0        SNAT protocol
- * @param rx_fib_index0 RX FIB index
- *
- * @returns 0 if packet should be translated otherwise 1
- */
-static inline int
-snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
-                         u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
-                         u32 rx_fib_index0)
-{
-  fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
-  fib_prefix_t pfx = {
-    .fp_proto = FIB_PROTOCOL_IP4,
-    .fp_len = 32,
-    .fp_addr = {
-        .ip4.as_u32 = ip0->dst_address.as_u32,
-    },
-  };
-
-  /* Don't NAT packet aimed at the intfc address */
-  if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
-                                      ip0->dst_address.as_u32)))
-    return 1;
-
-  fei = fib_table_lookup (rx_fib_index0, &pfx);
-  if (FIB_NODE_INDEX_INVALID != fei)
-    {
-      u32 sw_if_index = fib_entry_get_resolving_interface (fei);
-      if (sw_if_index == ~0)
-        {
-          fei = fib_table_lookup (sm->outside_fib_index, &pfx);
-          if (FIB_NODE_INDEX_INVALID != fei)
-            sw_if_index = fib_entry_get_resolving_interface (fei);
-        }
-      snat_interface_t *i;
-      pool_foreach (i, sm->interfaces,
-      ({
-        /* NAT packet aimed at outside interface */
-        if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
-          return 0;
-      }));
-    }
-
-  return 1;
-}
-
-static inline int
-snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
-                    u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
-                    u32 rx_fib_index0)
-{
-  udp_header_t * udp0 = ip4_next_header (ip0);
-  snat_session_key_t key0, sm0;
-  clib_bihash_kv_8_8_t kv0, value0;
-
-  key0.addr = ip0->dst_address;
-  key0.port = udp0->dst_port;
-  key0.protocol = proto0;
-  key0.fib_index = sm->outside_fib_index;
-  kv0.key = key0.as_u64;
-
-  /* NAT packet aimed at external address if */
-  /* has active sessions */
-  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
-    {
-      /* or is static mappings */
-      if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
-        return 0;
-    }
-  else
-    return 0;
-
-  return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
-                                 rx_fib_index0);
-}
-
-static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
-                      ip4_header_t * ip0,
-                      u32 rx_fib_index0,
-                      snat_session_key_t * key0,
-                      snat_session_t ** sessionp,
-                      vlib_node_runtime_t * node,
-                      u32 next0,
-                      u32 thread_index)
-{
-  snat_user_t *u;
-  snat_user_key_t user_key;
-  snat_session_t *s;
-  clib_bihash_kv_8_8_t kv0, value0;
-  u32 oldest_per_user_translation_list_index;
-  dlist_elt_t * oldest_per_user_translation_list_elt;
-  dlist_elt_t * per_user_translation_list_elt;
-  dlist_elt_t * per_user_list_head_elt;
-  u32 session_index;
-  snat_session_key_t key1;
-  u32 address_index = ~0;
-  u32 outside_fib_index;
-  uword * p;
-  snat_worker_key_t worker_by_out_key;
-
-  p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
-  if (! p)
-    {
-      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
-      return SNAT_IN2OUT_NEXT_DROP;
-    }
-  outside_fib_index = p[0];
-
-  key1.protocol = key0->protocol;
-  user_key.addr = ip0->src_address;
-  user_key.fib_index = rx_fib_index0;
-  kv0.key = user_key.as_u64;
-  
-  /* Ever heard of the "user" = src ip4 address before? */
-  if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
-    {
-      /* no, make a new one */
-      pool_get (sm->per_thread_data[thread_index].users, u);
-      memset (u, 0, sizeof (*u));
-      u->addr = ip0->src_address;
-      u->fib_index = rx_fib_index0;
-
-      pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
-
-      u->sessions_per_user_list_head_index = per_user_list_head_elt -
-        sm->per_thread_data[thread_index].list_pool;
-
-      clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
-                       u->sessions_per_user_list_head_index);
-
-      kv0.value = u - sm->per_thread_data[thread_index].users;
-
-      /* add user */
-      clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
-    }
-  else
-    {
-      u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
-                             value0.value);
-    }
-
-  /* Over quota? Recycle the least recently used dynamic translation */
-  if (u->nsessions >= sm->max_translations_per_user)
-    {
-      /* Remove the oldest dynamic translation */
-      do {
-          oldest_per_user_translation_list_index =
-            clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
-                                    u->sessions_per_user_list_head_index);
-
-          ASSERT (oldest_per_user_translation_list_index != ~0);
-
-          /* add it back to the end of the LRU list */
-          clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                              u->sessions_per_user_list_head_index,
-                              oldest_per_user_translation_list_index);
-          /* Get the list element */
-          oldest_per_user_translation_list_elt =
-            pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
-                               oldest_per_user_translation_list_index);
-
-          /* Get the session index from the list element */
-          session_index = oldest_per_user_translation_list_elt->value;
-
-          /* Get the session */
-          s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                 session_index);
-      } while (snat_is_session_static (s));
-
-      if (snat_is_unk_proto_session (s))
-        {
-          clib_bihash_kv_16_8_t up_kv;
-          snat_unk_proto_ses_key_t key;
-
-          /* Remove from lookup tables */
-          key.l_addr = s->in2out.addr;
-          key.r_addr = s->ext_host_addr;
-          key.fib_index = s->in2out.fib_index;
-          key.proto = s->in2out.port;
-          up_kv.key[0] = key.as_u64[0];
-          up_kv.key[1] = key.as_u64[1];
-          if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &up_kv, 0))
-            clib_warning ("in2out key del failed");
-
-          key.l_addr = s->out2in.addr;
-          key.fib_index = s->out2in.fib_index;
-          up_kv.key[0] = key.as_u64[0];
-          up_kv.key[1] = key.as_u64[1];
-          if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &up_kv, 0))
-            clib_warning ("out2in key del failed");
-        }
-      else
-        {
-          /* Remove in2out, out2in keys */
-          kv0.key = s->in2out.as_u64;
-          if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
-              clib_warning ("in2out key delete failed");
-          kv0.key = s->out2in.as_u64;
-          if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
-              clib_warning ("out2in key delete failed");
-
-          /* log NAT event */
-          snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
-                                              s->out2in.addr.as_u32,
-                                              s->in2out.protocol,
-                                              s->in2out.port,
-                                              s->out2in.port,
-                                              s->in2out.fib_index);
-
-          snat_free_outside_address_and_port
-            (sm, &s->out2in, s->outside_address_index);
-        }
-      s->outside_address_index = ~0;
-
-      if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, thread_index,
-                                               &key1, &address_index))
-        {
-          ASSERT(0);
-
-          b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
-          return SNAT_IN2OUT_NEXT_DROP;
-        }
-      s->outside_address_index = address_index;
-    }
-  else
-    {
-      u8 static_mapping = 1;
-
-      /* First try to match static mapping by local address and port */
-      if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
-        {
-          static_mapping = 0;
-          /* Try to create dynamic translation */
-          if (snat_alloc_outside_address_and_port (sm, rx_fib_index0,
-                                                   thread_index, &key1,
-                                                   &address_index))
-            {
-              b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
-              return SNAT_IN2OUT_NEXT_DROP;
-            }
-        }
-
-      /* Create a new session */
-      pool_get (sm->per_thread_data[thread_index].sessions, s);
-      memset (s, 0, sizeof (*s));
-      
-      s->outside_address_index = address_index;
-
-      if (static_mapping)
-        {
-          u->nstaticsessions++;
-          s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
-        }
-      else
-        {
-          u->nsessions++;
-        }
-
-      /* Create list elts */
-      pool_get (sm->per_thread_data[thread_index].list_pool,
-                per_user_translation_list_elt);
-      clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
-                       per_user_translation_list_elt -
-                       sm->per_thread_data[thread_index].list_pool);
-
-      per_user_translation_list_elt->value =
-        s - sm->per_thread_data[thread_index].sessions;
-      s->per_user_index = per_user_translation_list_elt -
-                          sm->per_thread_data[thread_index].list_pool;
-      s->per_user_list_head_index = u->sessions_per_user_list_head_index;
-
-      clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                          s->per_user_list_head_index,
-                          per_user_translation_list_elt -
-                          sm->per_thread_data[thread_index].list_pool);
-   }
-  
-  s->in2out = *key0;
-  s->out2in = key1;
-  s->out2in.protocol = key0->protocol;
-  s->out2in.fib_index = outside_fib_index;
-  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
-  *sessionp = s;
-
-  /* Add to translation hashes */
-  kv0.key = s->in2out.as_u64;
-  kv0.value = s - sm->per_thread_data[thread_index].sessions;
-  if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
-      clib_warning ("in2out key add failed");
-  
-  kv0.key = s->out2in.as_u64;
-  kv0.value = s - sm->per_thread_data[thread_index].sessions;
-  
-  if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
-      clib_warning ("out2in key add failed");
-
-  /* Add to translated packets worker lookup */
-  worker_by_out_key.addr = s->out2in.addr;
-  worker_by_out_key.port = s->out2in.port;
-  worker_by_out_key.fib_index = s->out2in.fib_index;
-  kv0.key = worker_by_out_key.as_u64;
-  kv0.value = thread_index;
-  clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
-
-  /* log NAT event */
-  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
-                                      s->out2in.addr.as_u32,
-                                      s->in2out.protocol,
-                                      s->in2out.port,
-                                      s->out2in.port,
-                                      s->in2out.fib_index);
-  return next0;
-}
-
-static_always_inline
-snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
-                                 snat_session_key_t *p_key0)
-{
-  icmp46_header_t *icmp0;
-  snat_session_key_t key0;
-  icmp_echo_header_t *echo0, *inner_echo0 = 0;
-  ip4_header_t *inner_ip0 = 0;
-  void *l4_header = 0;
-  icmp46_header_t *inner_icmp0;
-
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  echo0 = (icmp_echo_header_t *)(icmp0+1);
-
-  if (!icmp_is_error_message (icmp0))
-    {
-      key0.protocol = SNAT_PROTOCOL_ICMP;
-      key0.addr = ip0->src_address;
-      key0.port = echo0->identifier;
-    }
-  else
-    {
-      inner_ip0 = (ip4_header_t *)(echo0+1);
-      l4_header = ip4_next_header (inner_ip0);
-      key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
-      key0.addr = inner_ip0->dst_address;
-      switch (key0.protocol)
-        {
-        case SNAT_PROTOCOL_ICMP:
-          inner_icmp0 = (icmp46_header_t*)l4_header;
-          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
-          key0.port = inner_echo0->identifier;
-          break;
-        case SNAT_PROTOCOL_UDP:
-        case SNAT_PROTOCOL_TCP:
-          key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
-          break;
-        default:
-          return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
-        }
-    }
-  *p_key0 = key0;
-  return -1; /* success */
-}
-
-/**
- * Get address and port values to be used for packet SNAT translation
- * and create session if needed
- *
- * @param[in,out] sm             SNAT main
- * @param[in,out] node           SNAT node runtime
- * @param[in] thread_index       thread index
- * @param[in,out] b0             buffer containing packet to be translated
- * @param[out] p_proto           protocol used for matching
- * @param[out] p_value           address and port after NAT translation
- * @param[out] p_dont_translate  if packet should not be translated
- * @param d                      optional parameter
- * @param e                      optional parameter
- */
-u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e)
-{
-  ip4_header_t *ip0;
-  icmp46_header_t *icmp0;
-  u32 sw_if_index0;
-  u32 rx_fib_index0;
-  snat_session_key_t key0;
-  snat_session_t *s0 = 0;
-  u8 dont_translate = 0;
-  clib_bihash_kv_8_8_t kv0, value0;
-  u32 next0 = ~0;
-  int err;
-  u32 iph_offset0 = 0;
-
-  if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
-    {
-      iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
-    }
-  ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
-
-  err = icmp_get_key (ip0, &key0);
-  if (err != -1)
-    {
-      b0->error = node->errors[err];
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      goto out;
-    }
-  key0.fib_index = rx_fib_index0;
-
-  kv0.key = key0.as_u64;
-
-  if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
-    {
-      if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
-          IP_PROTOCOL_ICMP, rx_fib_index0) &&
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-
-      if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
-        {
-          b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          goto out;
-        }
-
-      next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
-                         &s0, node, next0, thread_index);
-
-      if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
-        goto out;
-    }
-  else
-    {
-      if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
-                        icmp0->type != ICMP4_echo_reply &&
-                        !icmp_is_error_message (icmp0)))
-        {
-          b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          goto out;
-        }
-
-      s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                              value0.value);
-    }
-
-out:
-  *p_proto = key0.protocol;
-  if (s0)
-    *p_value = s0->out2in;
-  *p_dont_translate = dont_translate;
-  if (d)
-    *(snat_session_t**)d = s0;
-  return next0;
-}
-
-/**
- * Get address and port values to be used for packet SNAT translation
- *
- * @param[in] sm                 SNAT main
- * @param[in,out] node           SNAT node runtime
- * @param[in] thread_index       thread index
- * @param[in,out] b0             buffer containing packet to be translated
- * @param[out] p_proto           protocol used for matching
- * @param[out] p_value           address and port after NAT translation
- * @param[out] p_dont_translate  if packet should not be translated
- * @param d                      optional parameter
- * @param e                      optional parameter
- */
-u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e)
-{
-  ip4_header_t *ip0;
-  icmp46_header_t *icmp0;
-  u32 sw_if_index0;
-  u32 rx_fib_index0;
-  snat_session_key_t key0;
-  snat_session_key_t sm0;
-  u8 dont_translate = 0;
-  u8 is_addr_only;
-  u32 next0 = ~0;
-  int err;
-
-  ip0 = vlib_buffer_get_current (b0);
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
-
-  err = icmp_get_key (ip0, &key0);
-  if (err != -1)
-    {
-      b0->error = node->errors[err];
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      goto out2;
-    }
-  key0.fib_index = rx_fib_index0;
-
-  if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
-    {
-      if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
-          IP_PROTOCOL_ICMP, rx_fib_index0)))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-
-      if (icmp_is_error_message (icmp0))
-        {
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          goto out;
-        }
-
-      b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      goto out;
-    }
-
-  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
-                    (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
-                    !icmp_is_error_message (icmp0)))
-    {
-      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      goto out;
-    }
-
-out:
-  *p_value = sm0;
-out2:
-  *p_proto = key0.protocol;
-  *p_dont_translate = dont_translate;
-  return next0;
-}
-
-static inline u32 icmp_in2out (snat_main_t *sm,
-                               vlib_buffer_t * b0,
-                               ip4_header_t * ip0,
-                               icmp46_header_t * icmp0,
-                               u32 sw_if_index0,
-                               u32 rx_fib_index0,
-                               vlib_node_runtime_t * node,
-                               u32 next0,
-                               u32 thread_index,
-                               void *d,
-                               void *e)
-{
-  snat_session_key_t sm0;
-  u8 protocol;
-  icmp_echo_header_t *echo0, *inner_echo0 = 0;
-  ip4_header_t *inner_ip0;
-  void *l4_header = 0;
-  icmp46_header_t *inner_icmp0;
-  u8 dont_translate;
-  u32 new_addr0, old_addr0;
-  u16 old_id0, new_id0;
-  ip_csum_t sum0;
-  u16 checksum0;
-  u32 next0_tmp;
-
-  echo0 = (icmp_echo_header_t *)(icmp0+1);
-
-  next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
-                                       &protocol, &sm0, &dont_translate, d, e);
-  if (next0_tmp != ~0)
-    next0 = next0_tmp;
-  if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
-    goto out;
-
-  sum0 = ip_incremental_checksum (0, icmp0,
-                                  ntohs(ip0->length) - ip4_header_bytes (ip0));
-  checksum0 = ~ip_csum_fold (sum0);
-  if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
-    {
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      goto out;
-    }
-
-  old_addr0 = ip0->src_address.as_u32;
-  new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
-  if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
-    vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
-
-  sum0 = ip0->checksum;
-  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
-                         src_address /* changed member */);
-  ip0->checksum = ip_csum_fold (sum0);
-  
-  if (!icmp_is_error_message (icmp0))
-    {
-      new_id0 = sm0.port;
-      if (PREDICT_FALSE(new_id0 != echo0->identifier))
-        {
-          old_id0 = echo0->identifier;
-          new_id0 = sm0.port;
-          echo0->identifier = new_id0;
-
-          sum0 = icmp0->checksum;
-          sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
-                                 identifier);
-          icmp0->checksum = ip_csum_fold (sum0);
-        }
-    }
-  else
-    {
-      inner_ip0 = (ip4_header_t *)(echo0+1);
-      l4_header = ip4_next_header (inner_ip0);
-
-      if (!ip4_header_checksum_is_valid (inner_ip0))
-        {
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          goto out;
-        }
-
-      old_addr0 = inner_ip0->dst_address.as_u32;
-      inner_ip0->dst_address = sm0.addr;
-      new_addr0 = inner_ip0->dst_address.as_u32;
-
-      sum0 = icmp0->checksum;
-      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
-                             dst_address /* changed member */);
-      icmp0->checksum = ip_csum_fold (sum0);
-
-      switch (protocol)
-        {
-          case SNAT_PROTOCOL_ICMP:
-            inner_icmp0 = (icmp46_header_t*)l4_header;
-            inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
-
-            old_id0 = inner_echo0->identifier;
-            new_id0 = sm0.port;
-            inner_echo0->identifier = new_id0;
-
-            sum0 = icmp0->checksum;
-            sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
-                                   identifier);
-            icmp0->checksum = ip_csum_fold (sum0);
-            break;
-          case SNAT_PROTOCOL_UDP:
-          case SNAT_PROTOCOL_TCP:
-            old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
-            new_id0 = sm0.port;
-            ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
-
-            sum0 = icmp0->checksum;
-            sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
-                                   dst_port);
-            icmp0->checksum = ip_csum_fold (sum0);
-            break;
-          default:
-            ASSERT(0);
-        }
-    }
-
-out:
-  return next0;
-}
-
-/**
- * @brief Hairpinning
- *
- * Hairpinning allows two endpoints on the internal side of the NAT to
- * communicate even if they only use each other's external IP addresses
- * and ports.
- *
- * @param sm     SNAT main.
- * @param b0     Vlib buffer.
- * @param ip0    IP header.
- * @param udp0   UDP header.
- * @param tcp0   TCP header.
- * @param proto0 SNAT protocol.
- */
-static inline void
-snat_hairpinning (snat_main_t *sm,
-                  vlib_buffer_t * b0,
-                  ip4_header_t * ip0,
-                  udp_header_t * udp0,
-                  tcp_header_t * tcp0,
-                  u32 proto0)
-{
-  snat_session_key_t key0, sm0;
-  snat_worker_key_t k0;
-  snat_session_t * s0;
-  clib_bihash_kv_8_8_t kv0, value0;
-  ip_csum_t sum0;
-  u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
-  u16 new_dst_port0, old_dst_port0;
-
-  key0.addr = ip0->dst_address;
-  key0.port = udp0->dst_port;
-  key0.protocol = proto0;
-  key0.fib_index = sm->outside_fib_index;
-  kv0.key = key0.as_u64;
-
-  /* Check if destination is in active sessions */
-  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
-    {
-      /* or static mappings */
-      if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
-        {
-          new_dst_addr0 = sm0.addr.as_u32;
-          new_dst_port0 = sm0.port;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
-        }
-    }
-  else
-    {
-      si = value0.value;
-      if (sm->num_workers > 1)
-        {
-          k0.addr = ip0->dst_address;
-          k0.port = udp0->dst_port;
-          k0.fib_index = sm->outside_fib_index;
-          kv0.key = k0.as_u64;
-          if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
-            ASSERT(0);
-          else
-            ti = value0.value;
-        }
-      else
-        ti = sm->num_workers;
-
-      s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
-      new_dst_addr0 = s0->in2out.addr.as_u32;
-      new_dst_port0 = s0->in2out.port;
-      vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
-    }
-
-  /* Destination is behind the same NAT, use internal address and port */
-  if (new_dst_addr0)
-    {
-      old_dst_addr0 = ip0->dst_address.as_u32;
-      ip0->dst_address.as_u32 = new_dst_addr0;
-      sum0 = ip0->checksum;
-      sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
-                             ip4_header_t, dst_address);
-      ip0->checksum = ip_csum_fold (sum0);
-
-      old_dst_port0 = tcp0->dst;
-      if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
-        {
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              tcp0->dst = new_dst_port0;
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
-                                     ip4_header_t, dst_address);
-              sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
-                                     ip4_header_t /* cheat */, length);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              udp0->dst_port = new_dst_port0;
-              udp0->checksum = 0;
-            }
-        }
-      else
-        {
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
-                                     ip4_header_t, dst_address);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-        }
-    }
-}
-
-static inline void
-snat_icmp_hairpinning (snat_main_t *sm,
-                       vlib_buffer_t * b0,
-                       ip4_header_t * ip0,
-                       icmp46_header_t * icmp0)
-{
-  snat_session_key_t key0, sm0;
-  clib_bihash_kv_8_8_t kv0, value0;
-  snat_worker_key_t k0;
-  u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
-  ip_csum_t sum0;
-  snat_session_t *s0;
-
-  if (!icmp_is_error_message (icmp0))
-    {
-      icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
-      u16 icmp_id0 = echo0->identifier;
-      key0.addr = ip0->dst_address;
-      key0.port = icmp_id0;
-      key0.protocol = SNAT_PROTOCOL_ICMP;
-      key0.fib_index = sm->outside_fib_index;
-      kv0.key = key0.as_u64;
-
-      /* Check if destination is in active sessions */
-      if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
-        {
-          /* or static mappings */
-          if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
-            {
-              new_dst_addr0 = sm0.addr.as_u32;
-              vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
-            }
-        }
-      else
-        {
-          si = value0.value;
-          if (sm->num_workers > 1)
-            {
-              k0.addr = ip0->dst_address;
-              k0.port = icmp_id0;
-              k0.fib_index = sm->outside_fib_index;
-              kv0.key = k0.as_u64;
-              if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
-                ASSERT(0);
-              else
-                ti = value0.value;
-            }
-          else
-            ti = sm->num_workers;
-
-          s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
-          new_dst_addr0 = s0->in2out.addr.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
-          echo0->identifier = s0->in2out.port;
-          sum0 = icmp0->checksum;
-          sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
-                                 icmp_echo_header_t, identifier);
-          icmp0->checksum = ip_csum_fold (sum0);
-        }
-
-      /* Destination is behind the same NAT, use internal address and port */
-      if (new_dst_addr0)
-        {
-          old_dst_addr0 = ip0->dst_address.as_u32;
-          ip0->dst_address.as_u32 = new_dst_addr0;
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
-                                 ip4_header_t, dst_address);
-          ip0->checksum = ip_csum_fold (sum0);
-        }
-    }
-
-}
-
-static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
-                                         vlib_buffer_t * b0,
-                                         ip4_header_t * ip0,
-                                         icmp46_header_t * icmp0,
-                                         u32 sw_if_index0,
-                                         u32 rx_fib_index0,
-                                         vlib_node_runtime_t * node,
-                                         u32 next0,
-                                         f64 now,
-                                         u32 thread_index,
-                                         snat_session_t ** p_s0)
-{
-  next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
-                      next0, thread_index, p_s0, 0);
-  snat_session_t * s0 = *p_s0;
-  if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
-    {
-      /* Hairpinning */
-      if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
-        snat_icmp_hairpinning(sm, b0, ip0, icmp0);
-      /* Accounting */
-      s0->last_heard = now;
-      s0->total_pkts++;
-      s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
-      /* Per-user LRU list maintenance for dynamic translations */
-      if (!snat_is_session_static (s0))
-        {
-          clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                             s0->per_user_index);
-          clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                              s0->per_user_list_head_index,
-                              s0->per_user_index);
-        }
-    }
-  return next0;
-}
-static inline void
-snat_hairpinning_unknown_proto (snat_main_t *sm,
-                                vlib_buffer_t * b,
-                                ip4_header_t * ip)
-{
-  u32 old_addr, new_addr = 0, ti = 0;
-  clib_bihash_kv_8_8_t kv, value;
-  clib_bihash_kv_16_8_t s_kv, s_value;
-  snat_unk_proto_ses_key_t key;
-  snat_session_key_t m_key;
-  snat_worker_key_t w_key;
-  snat_static_mapping_t *m;
-  ip_csum_t sum;
-  snat_session_t *s;
-
-  old_addr = ip->dst_address.as_u32;
-  key.l_addr.as_u32 = ip->dst_address.as_u32;
-  key.r_addr.as_u32 = ip->src_address.as_u32;
-  key.fib_index = sm->outside_fib_index;
-  key.proto = ip->protocol;
-  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
-  s_kv.key[0] = key.as_u64[0];
-  s_kv.key[1] = key.as_u64[1];
-  if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
-    {
-      m_key.addr = ip->dst_address;
-      m_key.fib_index = sm->outside_fib_index;
-      m_key.port = 0;
-      m_key.protocol = 0;
-      kv.key = m_key.as_u64;
-      if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
-        return;
-
-      m = pool_elt_at_index (sm->static_mappings, value.value);
-      if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
-        vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
-      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
-    }
-  else
-    {
-      if (sm->num_workers > 1)
-        {
-          w_key.addr = ip->dst_address;
-          w_key.port = 0;
-          w_key.fib_index = sm->outside_fib_index;
-          kv.key = w_key.as_u64;
-          if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv, &value))
-            return;
-          else
-            ti = value.value;
-        }
-      else
-        ti = sm->num_workers;
-
-      s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
-      if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
-        vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
-      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
-    }
-  sum = ip->checksum;
-  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
-  ip->checksum = ip_csum_fold (sum);
-}
-
-static void
-snat_in2out_unknown_proto (snat_main_t *sm,
-                           vlib_buffer_t * b,
-                           ip4_header_t * ip,
-                           u32 rx_fib_index,
-                           u32 thread_index,
-                           f64 now,
-                           vlib_main_t * vm)
-{
-  clib_bihash_kv_8_8_t kv, value;
-  clib_bihash_kv_16_8_t s_kv, s_value;
-  snat_static_mapping_t *m;
-  snat_session_key_t m_key;
-  u32 old_addr, new_addr = 0;
-  ip_csum_t sum;
-  snat_user_key_t u_key;
-  snat_user_t *u;
-  dlist_elt_t *head, *elt, *oldest;
-  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
-  u32 elt_index, head_index, ses_index, oldest_index;
-  snat_session_t * s;
-  snat_unk_proto_ses_key_t key;
-  u32 address_index = ~0;
-  int i;
-  u8 is_sm = 0;
-
-  old_addr = ip->src_address.as_u32;
-
-  key.l_addr = ip->src_address;
-  key.r_addr = ip->dst_address;
-  key.fib_index = rx_fib_index;
-  key.proto = ip->protocol;
-  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
-  s_kv.key[0] = key.as_u64[0];
-  s_kv.key[1] = key.as_u64[1];
-
-  if (!clib_bihash_search_16_8 (&sm->in2out_unk_proto, &s_kv, &s_value))
-    {
-      s = pool_elt_at_index (tsm->sessions, s_value.value);
-      new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
-    }
-  else
-    {
-      u_key.addr = ip->src_address;
-      u_key.fib_index = rx_fib_index;
-      kv.key = u_key.as_u64;
-
-      /* Ever heard of the "user" = src ip4 address before? */
-      if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
-        {
-          /* no, make a new one */
-          pool_get (tsm->users, u);
-          memset (u, 0, sizeof (*u));
-          u->addr = ip->src_address;
-          u->fib_index = rx_fib_index;
-
-          pool_get (tsm->list_pool, head);
-          u->sessions_per_user_list_head_index = head - tsm->list_pool;
-
-          clib_dlist_init (tsm->list_pool,
-                           u->sessions_per_user_list_head_index);
-
-          kv.value = u - tsm->users;
-
-          /* add user */
-          clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
-        }
-      else
-        {
-          u = pool_elt_at_index (tsm->users, value.value);
-        }
-
-      m_key.addr = ip->src_address;
-      m_key.port = 0;
-      m_key.protocol = 0;
-      m_key.fib_index = rx_fib_index;
-      kv.key = m_key.as_u64;
-
-      /* Try to find static mapping first */
-      if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
-        {
-          m = pool_elt_at_index (sm->static_mappings, value.value);
-          new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
-          is_sm = 1;
-          goto create_ses;
-        }
-      /* Fallback to 3-tuple key */
-      else
-        {
-          /* Choose same out address as for TCP/UDP session to same destination */
-          if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
-            {
-              head_index = u->sessions_per_user_list_head_index;
-              head = pool_elt_at_index (tsm->list_pool, head_index);
-              elt_index = head->next;
-              elt = pool_elt_at_index (tsm->list_pool, elt_index);
-              ses_index = elt->value;
-              while (ses_index != ~0)
-                {
-                  s =  pool_elt_at_index (tsm->sessions, ses_index);
-                  elt_index = elt->next;
-                  elt = pool_elt_at_index (tsm->list_pool, elt_index);
-                  ses_index = elt->value;
-
-                  if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
-                    {
-                      new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
-                      address_index = s->outside_address_index;
-
-                      key.fib_index = sm->outside_fib_index;
-                      key.l_addr.as_u32 = new_addr;
-                      s_kv.key[0] = key.as_u64[0];
-                      s_kv.key[1] = key.as_u64[1];
-                      if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
-                        break;
-
-                      goto create_ses;
-                    }
-                }
-            }
-          key.fib_index = sm->outside_fib_index;
-          for (i = 0; i < vec_len (sm->addresses); i++)
-            {
-              key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
-              s_kv.key[0] = key.as_u64[0];
-              s_kv.key[1] = key.as_u64[1];
-              if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
-                {
-                  new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
-                  address_index = i;
-                  goto create_ses;
-                }
-            }
-          return;
-        }
-
-create_ses:
-      /* Over quota? Recycle the least recently used dynamic translation */
-      if (u->nsessions >= sm->max_translations_per_user && !is_sm)
-        {
-          /* Remove the oldest dynamic translation */
-          do {
-              oldest_index = clib_dlist_remove_head (
-                tsm->list_pool, u->sessions_per_user_list_head_index);
-
-              ASSERT (oldest_index != ~0);
-
-              /* add it back to the end of the LRU list */
-              clib_dlist_addtail (tsm->list_pool,
-                                  u->sessions_per_user_list_head_index,
-                                  oldest_index);
-              /* Get the list element */
-              oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
-
-              /* Get the session index from the list element */
-              ses_index = oldest->value;
-
-              /* Get the session */
-              s = pool_elt_at_index (tsm->sessions, ses_index);
-          } while (snat_is_session_static (s));
-
-          if (snat_is_unk_proto_session (s))
-            {
-              /* Remove from lookup tables */
-              key.l_addr = s->in2out.addr;
-              key.r_addr = s->ext_host_addr;
-              key.fib_index = s->in2out.fib_index;
-              key.proto = s->in2out.port;
-              s_kv.key[0] = key.as_u64[0];
-              s_kv.key[1] = key.as_u64[1];
-              if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 0))
-                clib_warning ("in2out key del failed");
-
-              key.l_addr = s->out2in.addr;
-              key.fib_index = s->out2in.fib_index;
-              s_kv.key[0] = key.as_u64[0];
-              s_kv.key[1] = key.as_u64[1];
-              if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 0))
-                clib_warning ("out2in key del failed");
-            }
-          else
-            {
-              /* log NAT event */
-              snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
-                                                  s->out2in.addr.as_u32,
-                                                  s->in2out.protocol,
-                                                  s->in2out.port,
-                                                  s->out2in.port,
-                                                  s->in2out.fib_index);
-
-              snat_free_outside_address_and_port (sm, &s->out2in,
-                                                  s->outside_address_index);
-
-              /* Remove in2out, out2in keys */
-              kv.key = s->in2out.as_u64;
-              if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
-                clib_warning ("in2out key del failed");
-              kv.key = s->out2in.as_u64;
-              if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
-                clib_warning ("out2in key del failed");
-            }
-        }
-      else
-        {
-          /* Create a new session */
-          pool_get (tsm->sessions, s);
-          memset (s, 0, sizeof (*s));
-
-          /* Create list elts */
-          pool_get (tsm->list_pool, elt);
-          clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
-          elt->value = s - tsm->sessions;
-          s->per_user_index = elt - tsm->list_pool;
-          s->per_user_list_head_index = u->sessions_per_user_list_head_index;
-          clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
-                              s->per_user_index);
-        }
-
-      s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
-      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
-      s->outside_address_index = address_index;
-      s->out2in.addr.as_u32 = new_addr;
-      s->out2in.fib_index = sm->outside_fib_index;
-      s->in2out.addr.as_u32 = old_addr;
-      s->in2out.fib_index = rx_fib_index;
-      s->in2out.port = s->out2in.port = ip->protocol;
-      if (is_sm)
-        {
-          u->nstaticsessions++;
-          s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
-        }
-      else
-        {
-          u->nsessions++;
-        }
-
-      /* Add to lookup tables */
-      key.l_addr.as_u32 = old_addr;
-      key.r_addr = ip->dst_address;
-      key.proto = ip->protocol;
-      key.fib_index = rx_fib_index;
-      s_kv.key[0] = key.as_u64[0];
-      s_kv.key[1] = key.as_u64[1];
-      s_kv.value = s - tsm->sessions;
-      if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
-        clib_warning ("in2out key add failed");
-
-      key.l_addr.as_u32 = new_addr;
-      key.fib_index = sm->outside_fib_index;
-      s_kv.key[0] = key.as_u64[0];
-      s_kv.key[1] = key.as_u64[1];
-      if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
-        clib_warning ("out2in key add failed");
-  }
-
-  /* Update IP checksum */
-  sum = ip->checksum;
-  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
-  ip->checksum = ip_csum_fold (sum);
-
-  /* Accounting */
-  s->last_heard = now;
-  s->total_pkts++;
-  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
-  /* Per-user LRU list maintenance */
-  clib_dlist_remove (tsm->list_pool, s->per_user_index);
-  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
-                      s->per_user_index);
-
-  /* Hairpinning */
-  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
-    snat_hairpinning_unknown_proto(sm, b, ip);
-
-  if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
-    vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-}
-
-static inline uword
-snat_in2out_node_fn_inline (vlib_main_t * vm,
-                            vlib_node_runtime_t * node,
-                            vlib_frame_t * frame, int is_slow_path,
-                            int is_output_feature)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_in2out_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-  f64 now = vlib_time_now (vm);
-  u32 stats_node_index;
-  u32 thread_index = vlib_get_thread_index ();
-
-  stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
-    snat_in2out_node.index;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0, next1;
-          u32 sw_if_index0, sw_if_index1;
-          ip4_header_t * ip0, * ip1;
-          ip_csum_t sum0, sum1;
-          u32 new_addr0, old_addr0, new_addr1, old_addr1;
-          u16 old_port0, new_port0, old_port1, new_port1;
-          udp_header_t * udp0, * udp1;
-          tcp_header_t * tcp0, * tcp1;
-          icmp46_header_t * icmp0, * icmp1;
-          snat_session_key_t key0, key1;
-          u32 rx_fib_index0, rx_fib_index1;
-          u32 proto0, proto1;
-          snat_session_t * s0 = 0, * s1 = 0;
-          clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
-          u32 iph_offset0 = 0, iph_offset1 = 0;
-          
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-            
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-            
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-          
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          if (is_output_feature)
-            iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
-
-          ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
-                 iph_offset0);
-
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-          icmp0 = (icmp46_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, 
-                                   sw_if_index0);
-
-          next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace00;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          /* Next configured feature, probably ip4-lookup */
-          if (is_slow_path)
-            {
-              if (PREDICT_FALSE (proto0 == ~0))
-                {
-                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
-                                             thread_index, now, vm);
-                  goto trace00;
-                }
-
-              if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
-                {
-                  next0 = icmp_in2out_slow_path 
-                    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, 
-                     node, next0, now, thread_index, &s0);
-                  goto trace00;
-                }
-            }
-          else
-            {
-              if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
-                {
-                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
-                  goto trace00;
-                }
-            }
-
-          key0.addr = ip0->src_address;
-          key0.port = udp0->src_port;
-          key0.protocol = proto0;
-          key0.fib_index = rx_fib_index0;
-          
-          kv0.key = key0.as_u64;
-
-          if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
-            {
-              if (is_slow_path)
-                {
-                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
-                      ip0, proto0, rx_fib_index0)) && !is_output_feature)
-                    goto trace00;
-
-                  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
-                                     &s0, node, next0, thread_index);
-                  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
-                    goto trace00;
-                }
-              else
-                {
-                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
-                  goto trace00;
-                }
-            }
-          else
-            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                    value0.value);
-
-          old_addr0 = ip0->src_address.as_u32;
-          ip0->src_address = s0->out2in.addr;
-          new_addr0 = ip0->src_address.as_u32;
-          if (!is_output_feature)
-            vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              old_port0 = tcp0->src_port;
-              tcp0->src_port = s0->out2in.port;
-              new_port0 = tcp0->src_port;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              old_port0 = udp0->src_port;
-              udp0->src_port = s0->out2in.port;
-              udp0->checksum = 0;
-            }
-
-          /* Hairpinning */
-          if (!is_output_feature)
-            snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
-
-          /* Accounting */
-          s0->last_heard = now;
-          s0->total_pkts++;
-          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
-          /* Per-user LRU list maintenance for dynamic translation */
-          if (!snat_is_session_static (s0))
-            {
-              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                                 s0->per_user_index);
-              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                                  s0->per_user_list_head_index,
-                                  s0->per_user_index);
-            }
-        trace00:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              snat_in2out_trace_t *t = 
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->is_slow_path = is_slow_path;
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-                  t->session_index = ~0;
-              if (s0)
-                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          if (is_output_feature)
-            iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
-
-          ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
-                 iph_offset1);
-
-          udp1 = ip4_next_header (ip1);
-          tcp1 = (tcp_header_t *) udp1;
-          icmp1 = (icmp46_header_t *) udp1;
-
-          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-	  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, 
-                                   sw_if_index1);
-
-          if (PREDICT_FALSE(ip1->ttl == 1))
-            {
-              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace01;
-            }
-
-          proto1 = ip_proto_to_snat_proto (ip1->protocol);
-
-          /* Next configured feature, probably ip4-lookup */
-          if (is_slow_path)
-            {
-              if (PREDICT_FALSE (proto1 == ~0))
-                {
-                  snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
-                                             thread_index, now, vm);
-                  goto trace01;
-                }
-
-              if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
-                {
-                  next1 = icmp_in2out_slow_path 
-                    (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
-                     next1, now, thread_index, &s1);
-                  goto trace01;
-                }
-            }
-          else
-            {
-              if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
-                {
-                  next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
-                  goto trace01;
-                }
-            }
-
-          key1.addr = ip1->src_address;
-          key1.port = udp1->src_port;
-          key1.protocol = proto1;
-          key1.fib_index = rx_fib_index1;
-          
-          kv1.key = key1.as_u64;
-
-            if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
-            {
-              if (is_slow_path)
-                {
-                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
-                      ip1, proto1, rx_fib_index1)) && !is_output_feature)
-                    goto trace01;
-
-                  next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
-                                     &s1, node, next1, thread_index);
-                  if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
-                    goto trace01;
-                }
-              else
-                {
-                  next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
-                  goto trace01;
-                }
-            }
-          else
-            s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                    value1.value);
-
-          old_addr1 = ip1->src_address.as_u32;
-          ip1->src_address = s1->out2in.addr;
-          new_addr1 = ip1->src_address.as_u32;
-          if (!is_output_feature)
-            vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
-
-          sum1 = ip1->checksum;
-          sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip1->checksum = ip_csum_fold (sum1);
-
-          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
-            {
-              old_port1 = tcp1->src_port;
-              tcp1->src_port = s1->out2in.port;
-              new_port1 = tcp1->src_port;
-
-              sum1 = tcp1->checksum;
-              sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-              sum1 = ip_csum_update (sum1, old_port1, new_port1,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp1->checksum = ip_csum_fold(sum1);
-            }
-          else
-            {
-              old_port1 = udp1->src_port;
-              udp1->src_port = s1->out2in.port;
-              udp1->checksum = 0;
-            }
-
-          /* Hairpinning */
-          if (!is_output_feature)
-            snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
-
-          /* Accounting */
-          s1->last_heard = now;
-          s1->total_pkts++;
-          s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
-          /* Per-user LRU list maintenance for dynamic translation */
-          if (!snat_is_session_static (s1))
-            {
-              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                                 s1->per_user_index);
-              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                                  s1->per_user_list_head_index,
-                                  s1->per_user_index);
-            }
-        trace01:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b1->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              snat_in2out_trace_t *t = 
-                 vlib_add_trace (vm, node, b1, sizeof (*t));
-              t->sw_if_index = sw_if_index1;
-              t->next_index = next1;
-              t->session_index = ~0;
-              if (s1)
-                t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
-            }
-
-          pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueues, maybe switch current next frame */
-          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                           to_next, n_left_to_next,
-                                           bi0, bi1, next0, next1);
-        }
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          u32 sw_if_index0;
-          ip4_header_t * ip0;
-          ip_csum_t sum0;
-          u32 new_addr0, old_addr0;
-          u16 old_port0, new_port0;
-          udp_header_t * udp0;
-          tcp_header_t * tcp0;
-          icmp46_header_t * icmp0;
-          snat_session_key_t key0;
-          u32 rx_fib_index0;
-          u32 proto0;
-          snat_session_t * s0 = 0;
-          clib_bihash_kv_8_8_t kv0, value0;
-          u32 iph_offset0 = 0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
-
-          if (is_output_feature)
-            iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
-
-          ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
-                 iph_offset0);
-
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-          icmp0 = (icmp46_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, 
-                                   sw_if_index0);
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace0;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          /* Next configured feature, probably ip4-lookup */
-          if (is_slow_path)
-            {
-              if (PREDICT_FALSE (proto0 == ~0))
-                {
-                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
-                                             thread_index, now, vm);
-                  goto trace0;
-                }
-
-              if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
-                {
-                  next0 = icmp_in2out_slow_path 
-                    (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
-                     next0, now, thread_index, &s0);
-                  goto trace0;
-                }
-            }
-          else
-            {
-              if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
-                {
-                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
-                  goto trace0;
-                }
-            }
-
-          key0.addr = ip0->src_address;
-          key0.port = udp0->src_port;
-          key0.protocol = proto0;
-          key0.fib_index = rx_fib_index0;
-          
-          kv0.key = key0.as_u64;
-
-          if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
-            {
-              if (is_slow_path)
-                {
-                  if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
-                      ip0, proto0, rx_fib_index0)) && !is_output_feature)
-                    goto trace0;
-
-                  next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
-                                     &s0, node, next0, thread_index);
-
-                  if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
-                    goto trace0;
-                }
-              else
-                {
-                  next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
-                  goto trace0;
-                }
-            }
-          else
-            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                    value0.value);
-
-          old_addr0 = ip0->src_address.as_u32;
-          ip0->src_address = s0->out2in.addr;
-          new_addr0 = ip0->src_address.as_u32;
-          if (!is_output_feature)
-            vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              old_port0 = tcp0->src_port;
-              tcp0->src_port = s0->out2in.port;
-              new_port0 = tcp0->src_port;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              old_port0 = udp0->src_port;
-              udp0->src_port = s0->out2in.port;
-              udp0->checksum = 0;
-            }
-
-          /* Hairpinning */
-          if (!is_output_feature)
-            snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
-
-          /* Accounting */
-          s0->last_heard = now;
-          s0->total_pkts++;
-          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
-          /* Per-user LRU list maintenance for dynamic translation */
-          if (!snat_is_session_static (s0))
-            {
-              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                                 s0->per_user_index);
-              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                                  s0->per_user_list_head_index,
-                                  s0->per_user_index);
-            }
-
-        trace0:
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              snat_in2out_trace_t *t = 
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->is_slow_path = is_slow_path;
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-                  t->session_index = ~0;
-              if (s0)
-                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, stats_node_index, 
-                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS, 
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-static uword
-snat_in2out_fast_path_fn (vlib_main_t * vm,
-                          vlib_node_runtime_t * node,
-                          vlib_frame_t * frame)
-{
-  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
-}
-
-VLIB_REGISTER_NODE (snat_in2out_node) = {
-  .function = snat_in2out_fast_path_fn,
-  .name = "snat-in2out",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-
-  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
-    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
-
-static uword
-snat_in2out_output_fast_path_fn (vlib_main_t * vm,
-                                 vlib_node_runtime_t * node,
-                                 vlib_frame_t * frame)
-{
-  return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
-}
-
-VLIB_REGISTER_NODE (snat_in2out_output_node) = {
-  .function = snat_in2out_output_fast_path_fn,
-  .name = "snat-in2out-output",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-
-  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
-    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-output-slowpath",
-    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
-                              snat_in2out_output_fast_path_fn);
-
-static uword
-snat_in2out_slow_path_fn (vlib_main_t * vm,
-                          vlib_node_runtime_t * node,
-                          vlib_frame_t * frame)
-{
-  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
-}
-
-VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
-  .function = snat_in2out_slow_path_fn,
-  .name = "snat-in2out-slowpath",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-
-  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
-    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
-                              snat_in2out_slow_path_fn);
-
-static uword
-snat_in2out_output_slow_path_fn (vlib_main_t * vm,
-                                 vlib_node_runtime_t * node,
-                                 vlib_frame_t * frame)
-{
-  return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
-}
-
-VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
-  .function = snat_in2out_output_slow_path_fn,
-  .name = "snat-in2out-output-slowpath",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-
-  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
-    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-output-slowpath",
-    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
-                              snat_in2out_output_slow_path_fn);
-
-/**************************/
-/*** deterministic mode ***/
-/**************************/
-static uword
-snat_det_in2out_node_fn (vlib_main_t * vm,
-                         vlib_node_runtime_t * node,
-                         vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_in2out_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-  u32 now = (u32) vlib_time_now (vm);
-  u32 thread_index = vlib_get_thread_index ();
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-        {
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0, next1;
-          u32 sw_if_index0, sw_if_index1;
-          ip4_header_t * ip0, * ip1;
-          ip_csum_t sum0, sum1;
-          ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
-          u16 old_port0, new_port0, lo_port0, i0;
-          u16 old_port1, new_port1, lo_port1, i1;
-          udp_header_t * udp0, * udp1;
-          tcp_header_t * tcp0, * tcp1;
-          u32 proto0, proto1;
-          snat_det_out_key_t key0, key1;
-          snat_det_map_t * dm0, * dm1;
-          snat_det_session_t * ses0 = 0, * ses1 = 0;
-          u32 rx_fib_index0, rx_fib_index1;
-          icmp46_header_t * icmp0, * icmp1;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
-          next1 = SNAT_IN2OUT_NEXT_LOOKUP;
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace0;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-              icmp0 = (icmp46_header_t *) udp0;
-
-              next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
-                                  rx_fib_index0, node, next0, thread_index,
-                                  &ses0, &dm0);
-              goto trace0;
-            }
-
-          dm0 = snat_det_map_by_user(sm, &ip0->src_address);
-          if (PREDICT_FALSE(!dm0))
-            {
-              clib_warning("no match for internal host %U",
-                           format_ip4_address, &ip0->src_address);
-              next0 = SNAT_IN2OUT_NEXT_DROP;
-              b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
-              goto trace0;
-            }
-
-          snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
-
-          key0.ext_host_addr = ip0->dst_address;
-          key0.ext_host_port = tcp0->dst;
-
-          ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
-          if (PREDICT_FALSE(!ses0))
-            {
-              for (i0 = 0; i0 < dm0->ports_per_host; i0++)
-                {
-                  key0.out_port = clib_host_to_net_u16 (lo_port0 +
-                    ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
-
-                  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
-                    continue;
-
-                  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
-                  break;
-                }
-              if (PREDICT_FALSE(!ses0))
-                {
-                  /* too many sessions for user, send ICMP error packet */
-
-                  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-                  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
-                                               ICMP4_destination_unreachable_destination_unreachable_host,
-                                               0);
-                  next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-                  goto trace0;
-                }
-            }
-
-          new_port0 = ses0->out.out_port;
-
-          old_addr0.as_u32 = ip0->src_address.as_u32;
-          ip0->src_address.as_u32 = new_addr0.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              if (tcp0->flags & TCP_FLAG_SYN)
-                ses0->state = SNAT_SESSION_TCP_SYN_SENT;
-              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
-                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
-              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
-                ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
-              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
-                snat_det_ses_close(dm0, ses0);
-              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
-                ses0->state = SNAT_SESSION_TCP_LAST_ACK;
-              else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
-                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
-
-              old_port0 = tcp0->src;
-              tcp0->src = new_port0;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              ses0->state = SNAT_SESSION_UDP_ACTIVE;
-              old_port0 = udp0->src_port;
-              udp0->src_port = new_port0;
-              udp0->checksum = 0;
-            }
-
-          switch(ses0->state)
-            {
-            case SNAT_SESSION_UDP_ACTIVE:
-                ses0->expire = now + sm->udp_timeout;
-                break;
-            case SNAT_SESSION_TCP_SYN_SENT:
-            case SNAT_SESSION_TCP_FIN_WAIT:
-            case SNAT_SESSION_TCP_CLOSE_WAIT:
-            case SNAT_SESSION_TCP_LAST_ACK:
-                ses0->expire = now + sm->tcp_transitory_timeout;
-                break;
-            case SNAT_SESSION_TCP_ESTABLISHED:
-                ses0->expire = now + sm->tcp_established_timeout;
-                break;
-            }
-
-        trace0:
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_in2out_trace_t *t =
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->is_slow_path = 0;
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-              t->session_index = ~0;
-              if (ses0)
-                t->session_index = ses0 - dm0->sessions;
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          ip1 = vlib_buffer_get_current (b1);
-          udp1 = ip4_next_header (ip1);
-          tcp1 = (tcp_header_t *) udp1;
-
-          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-
-          if (PREDICT_FALSE(ip1->ttl == 1))
-            {
-              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace1;
-            }
-
-          proto1 = ip_proto_to_snat_proto (ip1->protocol);
-
-          if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
-            {
-              rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
-              icmp1 = (icmp46_header_t *) udp1;
-
-              next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
-                                  rx_fib_index1, node, next1, thread_index,
-                                  &ses1, &dm1);
-              goto trace1;
-            }
-
-          dm1 = snat_det_map_by_user(sm, &ip1->src_address);
-          if (PREDICT_FALSE(!dm1))
-            {
-              clib_warning("no match for internal host %U",
-                           format_ip4_address, &ip0->src_address);
-              next1 = SNAT_IN2OUT_NEXT_DROP;
-              b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
-              goto trace1;
-            }
-
-          snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
-
-          key1.ext_host_addr = ip1->dst_address;
-          key1.ext_host_port = tcp1->dst;
-
-          ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
-          if (PREDICT_FALSE(!ses1))
-            {
-              for (i1 = 0; i1 < dm1->ports_per_host; i1++)
-                {
-                  key1.out_port = clib_host_to_net_u16 (lo_port1 +
-                    ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
-
-                  if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
-                    continue;
-
-                  ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
-                  break;
-                }
-              if (PREDICT_FALSE(!ses1))
-                {
-                  /* too many sessions for user, send ICMP error packet */
-
-                  vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-                  icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
-                                               ICMP4_destination_unreachable_destination_unreachable_host,
-                                               0);
-                  next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-                  goto trace1;
-                }
-            }
-
-          new_port1 = ses1->out.out_port;
-
-          old_addr1.as_u32 = ip1->src_address.as_u32;
-          ip1->src_address.as_u32 = new_addr1.as_u32;
-          vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-
-          sum1 = ip1->checksum;
-          sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip1->checksum = ip_csum_fold (sum1);
-
-          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
-            {
-              if (tcp1->flags & TCP_FLAG_SYN)
-                ses1->state = SNAT_SESSION_TCP_SYN_SENT;
-              else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
-                ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
-              else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
-                ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
-              else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
-                snat_det_ses_close(dm1, ses1);
-              else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
-                ses1->state = SNAT_SESSION_TCP_LAST_ACK;
-              else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
-                ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
-
-              old_port1 = tcp1->src;
-              tcp1->src = new_port1;
-
-              sum1 = tcp1->checksum;
-              sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-              sum1 = ip_csum_update (sum1, old_port1, new_port1,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp1->checksum = ip_csum_fold(sum1);
-            }
-          else
-            {
-              ses1->state = SNAT_SESSION_UDP_ACTIVE;
-              old_port1 = udp1->src_port;
-              udp1->src_port = new_port1;
-              udp1->checksum = 0;
-            }
-
-          switch(ses1->state)
-            {
-            case SNAT_SESSION_UDP_ACTIVE:
-                ses1->expire = now + sm->udp_timeout;
-                break;
-            case SNAT_SESSION_TCP_SYN_SENT:
-            case SNAT_SESSION_TCP_FIN_WAIT:
-            case SNAT_SESSION_TCP_CLOSE_WAIT:
-            case SNAT_SESSION_TCP_LAST_ACK:
-                ses1->expire = now + sm->tcp_transitory_timeout;
-                break;
-            case SNAT_SESSION_TCP_ESTABLISHED:
-                ses1->expire = now + sm->tcp_established_timeout;
-                break;
-            }
-
-        trace1:
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b1->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_in2out_trace_t *t =
-                 vlib_add_trace (vm, node, b1, sizeof (*t));
-              t->is_slow_path = 0;
-              t->sw_if_index = sw_if_index1;
-              t->next_index = next1;
-              t->session_index = ~0;
-              if (ses1)
-                t->session_index = ses1 - dm1->sessions;
-            }
-
-          pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueues, maybe switch current next frame */
-          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                           to_next, n_left_to_next,
-                                           bi0, bi1, next0, next1);
-         }
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          u32 sw_if_index0;
-          ip4_header_t * ip0;
-          ip_csum_t sum0;
-          ip4_address_t new_addr0, old_addr0;
-          u16 old_port0, new_port0, lo_port0, i0;
-          udp_header_t * udp0;
-          tcp_header_t * tcp0;
-          u32 proto0;
-          snat_det_out_key_t key0;
-          snat_det_map_t * dm0;
-          snat_det_session_t * ses0 = 0;
-          u32 rx_fib_index0;
-          icmp46_header_t * icmp0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace00;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-              icmp0 = (icmp46_header_t *) udp0;
-
-              next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
-                                  rx_fib_index0, node, next0, thread_index,
-                                  &ses0, &dm0);
-              goto trace00;
-            }
-
-          dm0 = snat_det_map_by_user(sm, &ip0->src_address);
-          if (PREDICT_FALSE(!dm0))
-            {
-              clib_warning("no match for internal host %U",
-                           format_ip4_address, &ip0->src_address);
-              next0 = SNAT_IN2OUT_NEXT_DROP;
-              b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
-              goto trace00;
-            }
-
-          snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
-
-          key0.ext_host_addr = ip0->dst_address;
-          key0.ext_host_port = tcp0->dst;
-
-          ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
-          if (PREDICT_FALSE(!ses0))
-            {
-              for (i0 = 0; i0 < dm0->ports_per_host; i0++)
-                {
-                  key0.out_port = clib_host_to_net_u16 (lo_port0 +
-                    ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
-
-                  if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
-                    continue;
-
-                  ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
-                  break;
-                }
-              if (PREDICT_FALSE(!ses0))
-                {
-                  /* too many sessions for user, send ICMP error packet */
-
-                  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-                  icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
-                                               ICMP4_destination_unreachable_destination_unreachable_host,
-                                               0);
-                  next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-                  goto trace00;
-                }
-            }
-
-          new_port0 = ses0->out.out_port;
-
-          old_addr0.as_u32 = ip0->src_address.as_u32;
-          ip0->src_address.as_u32 = new_addr0.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              if (tcp0->flags & TCP_FLAG_SYN)
-                ses0->state = SNAT_SESSION_TCP_SYN_SENT;
-              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
-                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
-              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
-                ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
-              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
-                snat_det_ses_close(dm0, ses0);
-              else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
-                ses0->state = SNAT_SESSION_TCP_LAST_ACK;
-              else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
-                ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
-
-              old_port0 = tcp0->src;
-              tcp0->src = new_port0;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              ses0->state = SNAT_SESSION_UDP_ACTIVE;
-              old_port0 = udp0->src_port;
-              udp0->src_port = new_port0;
-              udp0->checksum = 0;
-            }
-
-          switch(ses0->state)
-            {
-            case SNAT_SESSION_UDP_ACTIVE:
-                ses0->expire = now + sm->udp_timeout;
-                break;
-            case SNAT_SESSION_TCP_SYN_SENT:
-            case SNAT_SESSION_TCP_FIN_WAIT:
-            case SNAT_SESSION_TCP_CLOSE_WAIT:
-            case SNAT_SESSION_TCP_LAST_ACK:
-                ses0->expire = now + sm->tcp_transitory_timeout;
-                break;
-            case SNAT_SESSION_TCP_ESTABLISHED:
-                ses0->expire = now + sm->tcp_established_timeout;
-                break;
-            }
-
-        trace00:
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_in2out_trace_t *t =
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->is_slow_path = 0;
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-              t->session_index = ~0;
-              if (ses0)
-                t->session_index = ses0 - dm0->sessions;
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, snat_det_in2out_node.index,
-                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_det_in2out_node) = {
-  .function = snat_det_in2out_node_fn,
-  .name = "snat-det-in2out",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-
-  .n_next_nodes = 3,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
-
-/**
- * Get address and port values to be used for packet SNAT translation
- * and create session if needed
- *
- * @param[in,out] sm             SNAT main
- * @param[in,out] node           SNAT node runtime
- * @param[in] thread_index       thread index
- * @param[in,out] b0             buffer containing packet to be translated
- * @param[out] p_proto           protocol used for matching
- * @param[out] p_value           address and port after NAT translation
- * @param[out] p_dont_translate  if packet should not be translated
- * @param d                      optional parameter
- * @param e                      optional parameter
- */
-u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
-                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                          snat_session_key_t *p_value,
-                          u8 *p_dont_translate, void *d, void *e)
-{
-  ip4_header_t *ip0;
-  icmp46_header_t *icmp0;
-  u32 sw_if_index0;
-  u32 rx_fib_index0;
-  u8 protocol;
-  snat_det_out_key_t key0;
-  u8 dont_translate = 0;
-  u32 next0 = ~0;
-  icmp_echo_header_t *echo0, *inner_echo0 = 0;
-  ip4_header_t *inner_ip0;
-  void *l4_header = 0;
-  icmp46_header_t *inner_icmp0;
-  snat_det_map_t * dm0 = 0;
-  ip4_address_t new_addr0;
-  u16 lo_port0, i0;
-  snat_det_session_t * ses0 = 0;
-  ip4_address_t in_addr;
-  u16 in_port;
-
-  ip0 = vlib_buffer_get_current (b0);
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  echo0 = (icmp_echo_header_t *)(icmp0+1);
-  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
-
-  if (!icmp_is_error_message (icmp0))
-    {
-      protocol = SNAT_PROTOCOL_ICMP;
-      in_addr = ip0->src_address;
-      in_port = echo0->identifier;
-    }
-  else
-    {
-      inner_ip0 = (ip4_header_t *)(echo0+1);
-      l4_header = ip4_next_header (inner_ip0);
-      protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
-      in_addr = inner_ip0->dst_address;
-      switch (protocol)
-        {
-        case SNAT_PROTOCOL_ICMP:
-          inner_icmp0 = (icmp46_header_t*)l4_header;
-          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
-          in_port = inner_echo0->identifier;
-          break;
-        case SNAT_PROTOCOL_UDP:
-        case SNAT_PROTOCOL_TCP:
-          in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
-          break;
-        default:
-          b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          goto out;
-        }
-    }
-
-  dm0 = snat_det_map_by_user(sm, &in_addr);
-  if (PREDICT_FALSE(!dm0))
-    {
-      clib_warning("no match for internal host %U",
-                   format_ip4_address, &in_addr);
-      if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
-          IP_PROTOCOL_ICMP, rx_fib_index0)))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
-      goto out;
-    }
-
-  snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
-
-  key0.ext_host_addr = ip0->dst_address;
-  key0.ext_host_port = 0;
-
-  ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
-  if (PREDICT_FALSE(!ses0))
-    {
-      if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
-          IP_PROTOCOL_ICMP, rx_fib_index0)))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-      if (icmp0->type != ICMP4_echo_request)
-        {
-          b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          goto out;
-        }
-      for (i0 = 0; i0 < dm0->ports_per_host; i0++)
-        {
-          key0.out_port = clib_host_to_net_u16 (lo_port0 +
-            ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
-
-          if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
-            continue;
-
-          ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
-          break;
-        }
-      if (PREDICT_FALSE(!ses0))
-        {
-          next0 = SNAT_IN2OUT_NEXT_DROP;
-          b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
-          goto out;
-        }
-    }
-
-  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
-                    !icmp_is_error_message (icmp0)))
-    {
-      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
-      next0 = SNAT_IN2OUT_NEXT_DROP;
-      goto out;
-    }
-
-  u32 now = (u32) vlib_time_now (sm->vlib_main);
-
-  ses0->state = SNAT_SESSION_ICMP_ACTIVE;
-  ses0->expire = now + sm->icmp_timeout;
-
-out:
-  *p_proto = protocol;
-  if (ses0)
-    {
-      p_value->addr = new_addr0;
-      p_value->fib_index = sm->outside_fib_index;
-      p_value->port = ses0->out.out_port;
-    }
-  *p_dont_translate = dont_translate;
-  if (d)
-    *(snat_det_session_t**)d = ses0;
-  if (e)
-    *(snat_det_map_t**)e = dm0;
-  return next0;
-}
-
-/**********************/
-/*** worker handoff ***/
-/**********************/
-static inline uword
-snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
-                                      vlib_node_runtime_t * node,
-                                      vlib_frame_t * frame,
-                                      u8 is_output)
-{
-  snat_main_t *sm = &snat_main;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  u32 n_left_from, *from, *to_next = 0;
-  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
-  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
-    = 0;
-  vlib_frame_queue_elt_t *hf = 0;
-  vlib_frame_t *f = 0;
-  int i;
-  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
-  u32 next_worker_index = 0;
-  u32 current_worker_index = ~0;
-  u32 thread_index = vlib_get_thread_index ();
-  u32 fq_index;
-  u32 to_node_index;
-
-  ASSERT (vec_len (sm->workers));
-
-  if (is_output)
-    {
-      fq_index = sm->fq_in2out_output_index;
-      to_node_index = sm->in2out_output_node_index;
-    }
-  else
-    {
-      fq_index = sm->fq_in2out_index;
-      to_node_index = sm->in2out_node_index;
-    }
-
-  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
-    {
-      vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
-
-      vec_validate_init_empty (congested_handoff_queue_by_worker_index,
-			       sm->first_worker_index + sm->num_workers - 1,
-			       (vlib_frame_queue_t *) (~0));
-    }
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-
-  while (n_left_from > 0)
-    {
-      u32 bi0;
-      vlib_buffer_t *b0;
-      u32 sw_if_index0;
-      u32 rx_fib_index0;
-      ip4_header_t * ip0;
-      u8 do_handoff;
-
-      bi0 = from[0];
-      from += 1;
-      n_left_from -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-
-      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-      rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-
-      ip0 = vlib_buffer_get_current (b0);
-
-      next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
-
-      if (PREDICT_FALSE (next_worker_index != thread_index))
-        {
-          do_handoff = 1;
-
-          if (next_worker_index != current_worker_index)
-            {
-              if (hf)
-                hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
-
-              hf = vlib_get_worker_handoff_queue_elt (fq_index,
-                                                      next_worker_index,
-                                                      handoff_queue_elt_by_worker_index);
-
-              n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
-              to_next_worker = &hf->buffer_index[hf->n_vectors];
-              current_worker_index = next_worker_index;
-            }
-
-          /* enqueue to correct worker thread */
-          to_next_worker[0] = bi0;
-          to_next_worker++;
-          n_left_to_next_worker--;
-
-          if (n_left_to_next_worker == 0)
-            {
-              hf->n_vectors = VLIB_FRAME_SIZE;
-              vlib_put_frame_queue_elt (hf);
-              current_worker_index = ~0;
-              handoff_queue_elt_by_worker_index[next_worker_index] = 0;
-              hf = 0;
-            }
-        }
-      else
-        {
-          do_handoff = 0;
-          /* if this is 1st frame */
-          if (!f)
-            {
-              f = vlib_get_frame_to_node (vm, to_node_index);
-              to_next = vlib_frame_vector_args (f);
-            }
-
-          to_next[0] = bi0;
-          to_next += 1;
-          f->n_vectors++;
-        }
-
-      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	{
-          snat_in2out_worker_handoff_trace_t *t =
-            vlib_add_trace (vm, node, b0, sizeof (*t));
-          t->next_worker_index = next_worker_index;
-          t->do_handoff = do_handoff;
-        }
-    }
-
-  if (f)
-    vlib_put_frame_to_node (vm, to_node_index, f);
-
-  if (hf)
-    hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
-
-  /* Ship frames to the worker nodes */
-  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
-    {
-      if (handoff_queue_elt_by_worker_index[i])
-	{
-	  hf = handoff_queue_elt_by_worker_index[i];
-	  /*
-	   * It works better to let the handoff node
-	   * rate-adapt, always ship the handoff queue element.
-	   */
-	  if (1 || hf->n_vectors == hf->last_n_vectors)
-	    {
-	      vlib_put_frame_queue_elt (hf);
-	      handoff_queue_elt_by_worker_index[i] = 0;
-	    }
-	  else
-	    hf->last_n_vectors = hf->n_vectors;
-	}
-      congested_handoff_queue_by_worker_index[i] =
-	(vlib_frame_queue_t *) (~0);
-    }
-  hf = 0;
-  current_worker_index = ~0;
-  return frame->n_vectors;
-}
-
-static uword
-snat_in2out_worker_handoff_fn (vlib_main_t * vm,
-                               vlib_node_runtime_t * node,
-                               vlib_frame_t * frame)
-{
-  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
-}
-
-VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
-  .function = snat_in2out_worker_handoff_fn,
-  .name = "snat-in2out-worker-handoff",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_worker_handoff_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_next_nodes = 1,
-
-  .next_nodes = {
-    [0] = "error-drop",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
-                              snat_in2out_worker_handoff_fn);
-
-static uword
-snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
-                                      vlib_node_runtime_t * node,
-                                      vlib_frame_t * frame)
-{
-  return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
-}
-
-VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
-  .function = snat_in2out_output_worker_handoff_fn,
-  .name = "snat-in2out-output-worker-handoff",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_worker_handoff_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_next_nodes = 1,
-
-  .next_nodes = {
-    [0] = "error-drop",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
-                              snat_in2out_output_worker_handoff_fn);
-
-static_always_inline int
-is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
-{
-  snat_address_t * ap;
-  clib_bihash_kv_8_8_t kv, value;
-  snat_session_key_t m_key;
-
-  vec_foreach (ap, sm->addresses)
-    {
-      if (ap->addr.as_u32 == dst_addr->as_u32)
-        return 1;
-    }
-
-  m_key.addr.as_u32 = dst_addr->as_u32;
-  m_key.fib_index = sm->outside_fib_index;
-  m_key.port = 0;
-  m_key.protocol = 0;
-  kv.key = m_key.as_u64;
-  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
-    return 1;
-
-  return 0;
-}
-
-static uword
-snat_hairpin_dst_fn (vlib_main_t * vm,
-                     vlib_node_runtime_t * node,
-                     vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_in2out_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          ip4_header_t * ip0;
-          u32 proto0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
-          ip0 = vlib_buffer_get_current (b0);
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          vnet_buffer (b0)->snat.flags = 0;
-          if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
-            {
-              if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
-                {
-                  udp_header_t * udp0 = ip4_next_header (ip0);
-                  tcp_header_t * tcp0 = (tcp_header_t *) udp0;
-
-                  snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
-                }
-              else if (proto0 == SNAT_PROTOCOL_ICMP)
-                {
-                  icmp46_header_t * icmp0 = ip4_next_header (ip0);
-
-                  snat_icmp_hairpinning (sm, b0, ip0, icmp0);
-                }
-              else
-                {
-                  snat_hairpinning_unknown_proto (sm, b0, ip0);
-                }
-
-              vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
-              clib_warning("is hairpinning");
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-         }
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
-                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
-  .function = snat_hairpin_dst_fn,
-  .name = "snat-hairpin-dst",
-  .vector_size = sizeof (u32),
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-  .n_next_nodes = 2,
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
-                              snat_hairpin_dst_fn);
-
-static uword
-snat_hairpin_src_fn (vlib_main_t * vm,
-                     vlib_node_runtime_t * node,
-                     vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_in2out_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t *sm = &snat_main;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-          next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
-
-          if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) & SNAT_FLAG_HAIRPINNING))
-            {
-              if (PREDICT_TRUE (sm->num_workers > 1))
-                next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
-              else
-                next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-         }
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
-                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
-  .function = snat_hairpin_src_fn,
-  .name = "snat-hairpin-src",
-  .vector_size = sizeof (u32),
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-  .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
-  .next_nodes = {
-     [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
-     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "snat-in2out-output",
-     [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
-     [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "snat-in2out-output-worker-handoff",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
-                              snat_hairpin_src_fn);
-
-static uword
-snat_in2out_fast_static_map_fn (vlib_main_t * vm,
-                                vlib_node_runtime_t * node,
-                                vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_in2out_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-  u32 stats_node_index;
-
-  stats_node_index = snat_in2out_fast_node.index;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          u32 sw_if_index0;
-          ip4_header_t * ip0;
-          ip_csum_t sum0;
-          u32 new_addr0, old_addr0;
-          u16 old_port0, new_port0;
-          udp_header_t * udp0;
-          tcp_header_t * tcp0;
-          icmp46_header_t * icmp0;
-          snat_session_key_t key0, sm0;
-          u32 proto0;
-          u32 rx_fib_index0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-          next0 = SNAT_IN2OUT_NEXT_LOOKUP;
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-          icmp0 = (icmp46_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-	  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
-              goto trace0;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE (proto0 == ~0))
-              goto trace0;
-
-          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
-                                  rx_fib_index0, node, next0, ~0, 0, 0);
-              goto trace0;
-            }
-
-          key0.addr = ip0->src_address;
-          key0.port = udp0->src_port;
-          key0.fib_index = rx_fib_index0;
-
-          if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
-            {
-              b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
-              next0= SNAT_IN2OUT_NEXT_DROP;
-              goto trace0;
-            }
-
-          new_addr0 = sm0.addr.as_u32;
-          new_port0 = sm0.port;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
-          old_addr0 = ip0->src_address.as_u32;
-          ip0->src_address.as_u32 = new_addr0;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                 ip4_header_t,
-                                 src_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_FALSE(new_port0 != udp0->dst_port))
-            {
-              if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-                {
-                  old_port0 = tcp0->src_port;
-                  tcp0->src_port = new_port0;
-
-                  sum0 = tcp0->checksum;
-                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                         ip4_header_t,
-                                         dst_address /* changed member */);
-                  sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                         ip4_header_t /* cheat */,
-                                         length /* changed member */);
-                  tcp0->checksum = ip_csum_fold(sum0);
-                }
-              else
-                {
-                  old_port0 = udp0->src_port;
-                  udp0->src_port = new_port0;
-                  udp0->checksum = 0;
-                }
-            }
-          else
-            {
-              if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-                {
-                  sum0 = tcp0->checksum;
-                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                         ip4_header_t,
-                                         dst_address /* changed member */);
-                  tcp0->checksum = ip_csum_fold(sum0);
-                }
-            }
-
-          /* Hairpinning */
-          snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
-
-        trace0:
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_in2out_trace_t *t =
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-            }
-
-          pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, stats_node_index,
-                               SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-
-VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
-  .function = snat_in2out_fast_static_map_fn,
-  .name = "snat-in2out-fast",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_in2out_fast_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_errors = ARRAY_LEN(snat_in2out_error_strings),
-  .error_strings = snat_in2out_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-  
-  .n_next_nodes = SNAT_IN2OUT_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
-    [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
-    [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);
diff --git a/src/plugins/snat/nat64.c b/src/plugins/snat/nat64.c
deleted file mode 100644
index bd915b59..00000000
--- a/src/plugins/snat/nat64.c
+++ /dev/null
@@ -1,858 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 implementation
- */
-
-#include 
-#include 
-#include 
-
-
-nat64_main_t nat64_main;
-
-/* *INDENT-OFF* */
-
-/* Hook up input features */
-VNET_FEATURE_INIT (nat64_in2out, static) = {
-  .arc_name = "ip6-unicast",
-  .node_name = "nat64-in2out",
-  .runs_before = VNET_FEATURES ("ip6-lookup"),
-};
-VNET_FEATURE_INIT (nat64_out2in, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "nat64-out2in",
-  .runs_before = VNET_FEATURES ("ip4-lookup"),
-};
-
-static u8 well_known_prefix[] = {
-  0x00, 0x64, 0xff, 0x9b,
-  0x00, 0x00, 0x00, 0x00,
-  0x00, 0x00, 0x00, 0x00,
-  0x00, 0x00, 0x00, 0x00
-};
-
-/* *INDENT-ON* */
-
-clib_error_t *
-nat64_init (vlib_main_t * vm)
-{
-  nat64_main_t *nm = &nat64_main;
-  clib_error_t *error = 0;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-
-  nm->is_disabled = 0;
-
-  if (tm->n_vlib_mains > 1)
-    {
-      nm->is_disabled = 1;
-      goto error;
-    }
-
-  if (nat64_db_init (&nm->db))
-    {
-      error = clib_error_return (0, "NAT64 DB init failed");
-      goto error;
-    }
-
-  /* set session timeouts to default values */
-  nm->udp_timeout = SNAT_UDP_TIMEOUT;
-  nm->icmp_timeout = SNAT_ICMP_TIMEOUT;
-  nm->tcp_trans_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
-  nm->tcp_est_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
-  nm->tcp_incoming_syn_timeout = SNAT_TCP_INCOMING_SYN;
-
-error:
-  return error;
-}
-
-int
-nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add)
-{
-  nat64_main_t *nm = &nat64_main;
-  snat_address_t *a = 0;
-  snat_interface_t *interface;
-  int i;
-
-  /* Check if address already exists */
-  for (i = 0; i < vec_len (nm->addr_pool); i++)
-    {
-      if (nm->addr_pool[i].addr.as_u32 == addr->as_u32)
-	{
-	  a = nm->addr_pool + i;
-	  break;
-	}
-    }
-
-  if (is_add)
-    {
-      if (a)
-	return VNET_API_ERROR_VALUE_EXIST;
-
-      vec_add2 (nm->addr_pool, a, 1);
-      a->addr = *addr;
-      a->fib_index = 0;
-      if (vrf_id != ~0)
-	a->fib_index =
-	  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
-#define _(N, i, n, s) \
-      clib_bitmap_alloc (a->busy_##n##_port_bitmap, 65535);
-      foreach_snat_protocol
-#undef _
-    }
-  else
-    {
-      if (!a)
-	return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-      if (a->fib_index)
-	fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP6);
-
-#define _(N, id, n, s) \
-      clib_bitmap_free (a->busy_##n##_port_bitmap);
-      foreach_snat_protocol
-#undef _
-	/* Delete sessions using address */
-	nat64_db_free_out_addr (&nm->db, &a->addr);
-      vec_del1 (nm->addr_pool, i);
-    }
-
-  /* Add/del external address to FIB */
-  /* *INDENT-OFF* */
-  pool_foreach (interface, nm->interfaces,
-  ({
-    if (interface->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib (addr, 32, interface->sw_if_index, is_add);
-    break;
-  }));
-  /* *INDENT-ON* */
-
-  return 0;
-}
-
-void
-nat64_pool_addr_walk (nat64_pool_addr_walk_fn_t fn, void *ctx)
-{
-  nat64_main_t *nm = &nat64_main;
-  snat_address_t *a = 0;
-
-  /* *INDENT-OFF* */
-  vec_foreach (a, nm->addr_pool)
-    {
-      if (fn (a, ctx))
-        break;
-    };
-  /* *INDENT-ON* */
-}
-
-int
-nat64_add_del_interface (u32 sw_if_index, u8 is_inside, u8 is_add)
-{
-  nat64_main_t *nm = &nat64_main;
-  snat_interface_t *interface = 0, *i;
-  snat_address_t *ap;
-  const char *feature_name, *arc_name;
-
-  /* Check if address already exists */
-  /* *INDENT-OFF* */
-  pool_foreach (i, nm->interfaces,
-  ({
-    if (i->sw_if_index == sw_if_index)
-      {
-        interface = i;
-        break;
-      }
-  }));
-  /* *INDENT-ON* */
-
-  if (is_add)
-    {
-      if (interface)
-	return VNET_API_ERROR_VALUE_EXIST;
-
-      pool_get (nm->interfaces, interface);
-      interface->sw_if_index = sw_if_index;
-      interface->is_inside = is_inside;
-
-    }
-  else
-    {
-      if (!interface)
-	return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-      pool_put (nm->interfaces, interface);
-    }
-
-  if (!is_inside)
-    {
-      /* *INDENT-OFF* */
-      vec_foreach (ap, nm->addr_pool)
-        snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, is_add);
-      /* *INDENT-ON* */
-    }
-
-  arc_name = is_inside ? "ip6-unicast" : "ip4-unicast";
-  feature_name = is_inside ? "nat64-in2out" : "nat64-out2in";
-
-  return vnet_feature_enable_disable (arc_name, feature_name, sw_if_index,
-				      is_add, 0, 0);
-}
-
-void
-nat64_interfaces_walk (nat64_interface_walk_fn_t fn, void *ctx)
-{
-  nat64_main_t *nm = &nat64_main;
-  snat_interface_t *i = 0;
-
-  /* *INDENT-OFF* */
-  pool_foreach (i, nm->interfaces,
-  ({
-    if (fn (i, ctx))
-      break;
-  }));
-  /* *INDENT-ON* */
-}
-
-int
-nat64_alloc_out_addr_and_port (u32 fib_index, snat_protocol_t proto,
-			       ip4_address_t * addr, u16 * port)
-{
-  nat64_main_t *nm = &nat64_main;
-  snat_main_t *sm = &snat_main;
-  int i;
-  snat_address_t *a, *ga = 0;
-  u32 portnum;
-
-  for (i = 0; i < vec_len (nm->addr_pool); i++)
-    {
-      a = nm->addr_pool + i;
-      switch (proto)
-	{
-#define _(N, j, n, s) \
-        case SNAT_PROTOCOL_##N: \
-          if (a->busy_##n##_ports < (65535-1024)) \
-            { \
-              if (a->fib_index == fib_index) \
-                { \
-                  while (1) \
-                    { \
-                      portnum = random_u32 (&sm->random_seed); \
-                      portnum &= 0xFFFF; \
-                      if (portnum < 1024) \
-                        continue; \
-                      if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
-                                                    portnum)) \
-                        continue; \
-                      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
-                                                portnum, 1); \
-                      a->busy_##n##_ports++; \
-                      *port = portnum; \
-                      addr->as_u32 = a->addr.as_u32; \
-                      return 0; \
-                    } \
-                 } \
-               else if (a->fib_index == 0) \
-                 ga = a; \
-            } \
-          break;
-	  foreach_snat_protocol
-#undef _
-	default:
-	  clib_warning ("unknown protocol");
-	  return 1;
-	}
-    }
-
-  if (ga)
-    {
-      switch (proto)
-	{
-#define _(N, j, n, s) \
-        case SNAT_PROTOCOL_##N: \
-          while (1) \
-            { \
-              portnum = random_u32 (&sm->random_seed); \
-              portnum &= 0xFFFF; \
-              if (portnum < 1024) \
-                continue; \
-              if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
-                                            portnum)) \
-                continue; \
-              clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
-                                        portnum, 1); \
-              a->busy_##n##_ports++; \
-              *port = portnum; \
-              addr->as_u32 = a->addr.as_u32; \
-              return 0; \
-            }
-	  break;
-	  foreach_snat_protocol
-#undef _
-	default:
-	  clib_warning ("unknown protocol");
-	  return 1;
-	}
-    }
-
-  /* Totally out of translations to use... */
-  //TODO: IPFix
-  return 1;
-}
-
-void
-nat64_free_out_addr_and_port (ip4_address_t * addr, u16 port,
-			      snat_protocol_t proto)
-{
-  nat64_main_t *nm = &nat64_main;
-  int i;
-  snat_address_t *a;
-
-  for (i = 0; i < vec_len (nm->addr_pool); i++)
-    {
-      a = nm->addr_pool + i;
-      if (addr->as_u32 != a->addr.as_u32)
-	continue;
-      switch (proto)
-	{
-#define _(N, j, n, s) \
-        case SNAT_PROTOCOL_##N: \
-          ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
-                  port) == 1); \
-          clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, port, 0); \
-          a->busy_##n##_ports--; \
-          break;
-	  foreach_snat_protocol
-#undef _
-	default:
-	  clib_warning ("unknown protocol");
-	  return;
-	}
-      break;
-    }
-}
-
-int
-nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
-				ip4_address_t * out_addr, u16 in_port,
-				u16 out_port, u8 proto, u32 vrf_id, u8 is_add)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_db_bib_entry_t *bibe;
-  u32 fib_index =
-    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
-  snat_protocol_t p = ip_proto_to_snat_proto (proto);
-  ip46_address_t addr;
-  int i;
-  snat_address_t *a;
-
-  addr.as_u64[0] = in_addr->as_u64[0];
-  addr.as_u64[1] = in_addr->as_u64[1];
-  bibe =
-    nat64_db_bib_entry_find (&nm->db, &addr, clib_host_to_net_u16 (in_port),
-			     proto, fib_index, 1);
-
-  if (is_add)
-    {
-      if (bibe)
-	return VNET_API_ERROR_VALUE_EXIST;
-
-      for (i = 0; i < vec_len (nm->addr_pool); i++)
-	{
-	  a = nm->addr_pool + i;
-	  if (out_addr->as_u32 != a->addr.as_u32)
-	    continue;
-	  switch (p)
-	    {
-#define _(N, j, n, s) \
-            case SNAT_PROTOCOL_##N: \
-              if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
-                                            out_port)) \
-                return VNET_API_ERROR_INVALID_VALUE; \
-              clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
-                                        out_port, 1); \
-              if (out_port > 1024) \
-                a->busy_##n##_ports++; \
-              break;
-	      foreach_snat_protocol
-#undef _
-	    default:
-	      memset (&addr, 0, sizeof (addr));
-	      addr.ip4.as_u32 = out_addr->as_u32;
-	      if (nat64_db_bib_entry_find
-		  (&nm->db, &addr, 0, proto, fib_index, 0))
-		return VNET_API_ERROR_INVALID_VALUE;
-	    }
-	  break;
-	}
-      bibe =
-	nat64_db_bib_entry_create (&nm->db, in_addr, out_addr,
-				   clib_host_to_net_u16 (in_port),
-				   clib_host_to_net_u16 (out_port), fib_index,
-				   proto, 1);
-      if (!bibe)
-	return VNET_API_ERROR_UNSPECIFIED;
-    }
-  else
-    {
-      if (!bibe)
-	return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-      nat64_free_out_addr_and_port (out_addr, out_port, p);
-      nat64_db_bib_entry_free (&nm->db, bibe);
-    }
-
-  return 0;
-}
-
-int
-nat64_set_udp_timeout (u32 timeout)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (timeout == 0)
-    nm->udp_timeout = SNAT_UDP_TIMEOUT;
-  else if (timeout < SNAT_UDP_TIMEOUT_MIN)
-    return VNET_API_ERROR_INVALID_VALUE;
-  else
-    nm->udp_timeout = timeout;
-
-  return 0;
-}
-
-u32
-nat64_get_udp_timeout (void)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  return nm->udp_timeout;
-}
-
-int
-nat64_set_icmp_timeout (u32 timeout)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (timeout == 0)
-    nm->icmp_timeout = SNAT_ICMP_TIMEOUT;
-  else
-    nm->icmp_timeout = timeout;
-
-  return 0;
-}
-
-u32
-nat64_get_icmp_timeout (void)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  return nm->icmp_timeout;
-}
-
-int
-nat64_set_tcp_timeouts (u32 trans, u32 est, u32 incoming_syn)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (trans == 0)
-    nm->tcp_trans_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
-  else
-    nm->tcp_trans_timeout = trans;
-
-  if (est == 0)
-    nm->tcp_est_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
-  else
-    nm->tcp_est_timeout = est;
-
-  if (incoming_syn == 0)
-    nm->tcp_incoming_syn_timeout = SNAT_TCP_INCOMING_SYN;
-  else
-    nm->tcp_incoming_syn_timeout = incoming_syn;
-
-  return 0;
-}
-
-u32
-nat64_get_tcp_trans_timeout (void)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  return nm->tcp_trans_timeout;
-}
-
-u32
-nat64_get_tcp_est_timeout (void)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  return nm->tcp_est_timeout;
-}
-
-u32
-nat64_get_tcp_incoming_syn_timeout (void)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  return nm->tcp_incoming_syn_timeout;
-}
-
-void
-nat64_session_reset_timeout (nat64_db_st_entry_t * ste, vlib_main_t * vm)
-{
-  nat64_main_t *nm = &nat64_main;
-  u32 now = (u32) vlib_time_now (vm);
-
-  switch (ip_proto_to_snat_proto (ste->proto))
-    {
-    case SNAT_PROTOCOL_ICMP:
-      ste->expire = now + nm->icmp_timeout;
-      return;
-    case SNAT_PROTOCOL_TCP:
-      {
-	switch (ste->tcp_state)
-	  {
-	  case NAT64_TCP_STATE_V4_INIT:
-	  case NAT64_TCP_STATE_V6_INIT:
-	  case NAT64_TCP_STATE_V4_FIN_RCV:
-	  case NAT64_TCP_STATE_V6_FIN_RCV:
-	  case NAT64_TCP_STATE_V6_FIN_V4_FIN_RCV:
-	  case NAT64_TCP_STATE_TRANS:
-	    ste->expire = now + nm->tcp_trans_timeout;
-	    return;
-	  case NAT64_TCP_STATE_ESTABLISHED:
-	    ste->expire = now + nm->tcp_est_timeout;
-	    return;
-	  default:
-	    return;
-	  }
-      }
-    case SNAT_PROTOCOL_UDP:
-      ste->expire = now + nm->udp_timeout;
-      return;
-    default:
-      ste->expire = now + nm->udp_timeout;
-      return;
-    }
-}
-
-void
-nat64_tcp_session_set_state (nat64_db_st_entry_t * ste, tcp_header_t * tcp,
-			     u8 is_ip6)
-{
-  switch (ste->tcp_state)
-    {
-    case NAT64_TCP_STATE_CLOSED:
-      {
-	if (tcp->flags & TCP_FLAG_SYN)
-	  {
-	    if (is_ip6)
-	      ste->tcp_state = NAT64_TCP_STATE_V6_INIT;
-	    else
-	      ste->tcp_state = NAT64_TCP_STATE_V4_INIT;
-	  }
-	return;
-      }
-    case NAT64_TCP_STATE_V4_INIT:
-      {
-	if (is_ip6 && (tcp->flags & TCP_FLAG_SYN))
-	  ste->tcp_state = NAT64_TCP_STATE_ESTABLISHED;
-	return;
-      }
-    case NAT64_TCP_STATE_V6_INIT:
-      {
-	if (!is_ip6 && (tcp->flags & TCP_FLAG_SYN))
-	  ste->tcp_state = NAT64_TCP_STATE_ESTABLISHED;
-	return;
-      }
-    case NAT64_TCP_STATE_ESTABLISHED:
-      {
-	if (tcp->flags & TCP_FLAG_FIN)
-	  {
-	    if (is_ip6)
-	      ste->tcp_state = NAT64_TCP_STATE_V6_FIN_RCV;
-	    else
-	      ste->tcp_state = NAT64_TCP_STATE_V4_FIN_RCV;
-	  }
-	else if (tcp->flags & TCP_FLAG_RST)
-	  {
-	    ste->tcp_state = NAT64_TCP_STATE_TRANS;
-	  }
-	return;
-      }
-    case NAT64_TCP_STATE_V4_FIN_RCV:
-      {
-	if (is_ip6 && (tcp->flags & TCP_FLAG_FIN))
-	  ste->tcp_state = NAT64_TCP_STATE_V6_FIN_V4_FIN_RCV;
-	return;
-      }
-    case NAT64_TCP_STATE_V6_FIN_RCV:
-      {
-	if (!is_ip6 && (tcp->flags & TCP_FLAG_FIN))
-	  ste->tcp_state = NAT64_TCP_STATE_V6_FIN_V4_FIN_RCV;
-	return;
-      }
-    case NAT64_TCP_STATE_TRANS:
-      {
-	if (!(tcp->flags & TCP_FLAG_RST))
-	  ste->tcp_state = NAT64_TCP_STATE_ESTABLISHED;
-	return;
-      }
-    default:
-      return;
-    }
-}
-
-int
-nat64_add_del_prefix (ip6_address_t * prefix, u8 plen, u32 vrf_id, u8 is_add)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_prefix_t *p = 0;
-  int i;
-
-  /* Verify prefix length */
-  if (plen != 32 && plen != 40 && plen != 48 && plen != 56 && plen != 64
-      && plen != 96)
-    return VNET_API_ERROR_INVALID_VALUE;
-
-  /* Check if tenant already have prefix */
-  for (i = 0; i < vec_len (nm->pref64); i++)
-    {
-      if (nm->pref64[i].vrf_id == vrf_id)
-	{
-	  p = nm->pref64 + i;
-	  break;
-	}
-    }
-
-  if (is_add)
-    {
-      if (!p)
-	{
-	  vec_add2 (nm->pref64, p, 1);
-	  p->fib_index =
-	    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
-	  p->vrf_id = vrf_id;
-	}
-
-      p->prefix.as_u64[0] = prefix->as_u64[0];
-      p->prefix.as_u64[1] = prefix->as_u64[1];
-      p->plen = plen;
-    }
-  else
-    {
-      if (!p)
-	return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-      vec_del1 (nm->pref64, i);
-    }
-
-  return 0;
-}
-
-void
-nat64_prefix_walk (nat64_prefix_walk_fn_t fn, void *ctx)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_prefix_t *p = 0;
-
-  /* *INDENT-OFF* */
-  vec_foreach (p, nm->pref64)
-    {
-      if (fn (p, ctx))
-        break;
-    };
-  /* *INDENT-ON* */
-}
-
-void
-nat64_compose_ip6 (ip6_address_t * ip6, ip4_address_t * ip4, u32 fib_index)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_prefix_t *p, *gp = 0, *prefix = 0;
-
-  /* *INDENT-OFF* */
-  vec_foreach (p, nm->pref64)
-    {
-      if (p->fib_index == fib_index)
-        {
-          prefix = p;
-          break;
-        }
-
-      if (p->fib_index == 0)
-        gp = p;
-    };
-  /* *INDENT-ON* */
-
-  if (!prefix)
-    prefix = gp;
-
-  if (prefix)
-    {
-      memset (ip6, 0, 16);
-      memcpy (ip6, &p->prefix, p->plen);
-      switch (p->plen)
-	{
-	case 32:
-	  ip6->as_u32[1] = ip4->as_u32;
-	  break;
-	case 40:
-	  ip6->as_u8[5] = ip4->as_u8[0];
-	  ip6->as_u8[6] = ip4->as_u8[1];
-	  ip6->as_u8[7] = ip4->as_u8[2];
-	  ip6->as_u8[9] = ip4->as_u8[3];
-	  break;
-	case 48:
-	  ip6->as_u8[6] = ip4->as_u8[0];
-	  ip6->as_u8[7] = ip4->as_u8[1];
-	  ip6->as_u8[9] = ip4->as_u8[2];
-	  ip6->as_u8[10] = ip4->as_u8[3];
-	  break;
-	case 56:
-	  ip6->as_u8[7] = ip4->as_u8[0];
-	  ip6->as_u8[9] = ip4->as_u8[1];
-	  ip6->as_u8[10] = ip4->as_u8[2];
-	  ip6->as_u8[11] = ip4->as_u8[3];
-	  break;
-	case 64:
-	  ip6->as_u8[9] = ip4->as_u8[0];
-	  ip6->as_u8[10] = ip4->as_u8[1];
-	  ip6->as_u8[11] = ip4->as_u8[2];
-	  ip6->as_u8[12] = ip4->as_u8[3];
-	  break;
-	case 96:
-	  ip6->as_u32[3] = ip4->as_u32;
-	  break;
-	default:
-	  clib_warning ("invalid prefix length");
-	  break;
-	}
-    }
-  else
-    {
-      memcpy (ip6, well_known_prefix, 16);
-      ip6->as_u32[3] = ip4->as_u32;
-    }
-}
-
-void
-nat64_extract_ip4 (ip6_address_t * ip6, ip4_address_t * ip4, u32 fib_index)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_prefix_t *p, *gp = 0;
-  u8 plen = 0;
-
-  /* *INDENT-OFF* */
-  vec_foreach (p, nm->pref64)
-    {
-      if (p->fib_index == fib_index)
-        {
-          plen = p->plen;
-          break;
-        }
-
-      if (p->vrf_id == 0)
-        gp = p;
-    };
-  /* *INDENT-ON* */
-
-  if (!plen)
-    {
-      if (gp)
-	plen = gp->plen;
-      else
-	plen = 96;
-    }
-
-  switch (plen)
-    {
-    case 32:
-      ip4->as_u32 = ip6->as_u32[1];
-      break;
-    case 40:
-      ip4->as_u8[0] = ip6->as_u8[5];
-      ip4->as_u8[1] = ip6->as_u8[6];
-      ip4->as_u8[2] = ip6->as_u8[7];
-      ip4->as_u8[3] = ip6->as_u8[9];
-      break;
-    case 48:
-      ip4->as_u8[0] = ip6->as_u8[6];
-      ip4->as_u8[1] = ip6->as_u8[7];
-      ip4->as_u8[2] = ip6->as_u8[9];
-      ip4->as_u8[3] = ip6->as_u8[10];
-      break;
-    case 56:
-      ip4->as_u8[0] = ip6->as_u8[7];
-      ip4->as_u8[1] = ip6->as_u8[9];
-      ip4->as_u8[2] = ip6->as_u8[10];
-      ip4->as_u8[3] = ip6->as_u8[11];
-      break;
-    case 64:
-      ip4->as_u8[0] = ip6->as_u8[9];
-      ip4->as_u8[1] = ip6->as_u8[10];
-      ip4->as_u8[2] = ip6->as_u8[11];
-      ip4->as_u8[3] = ip6->as_u8[12];
-      break;
-    case 96:
-      ip4->as_u32 = ip6->as_u32[3];
-      break;
-    default:
-      clib_warning ("invalid prefix length");
-      break;
-    }
-}
-
-/**
- * @brief The 'nat64-expire-walk' process's main loop.
- *
- * Check expire time for NAT64 sessions.
- */
-static uword
-nat64_expire_walk_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
-		      vlib_frame_t * f)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  while (!nm->is_disabled)
-    {
-      vlib_process_wait_for_event_or_clock (vm, 10.0);
-      vlib_process_get_events (vm, NULL);
-      u32 now = (u32) vlib_time_now (vm);
-
-      nad64_db_st_free_expired (&nm->db, now);
-    }
-
-  return 0;
-}
-
-static vlib_node_registration_t nat64_expire_walk_node;
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (nat64_expire_walk_node, static) = {
-    .function = nat64_expire_walk_fn,
-    .type = VLIB_NODE_TYPE_PROCESS,
-    .name = "nat64-expire-walk",
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/nat64.h b/src/plugins/snat/nat64.h
deleted file mode 100644
index 771b9075..00000000
--- a/src/plugins/snat/nat64.h
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 global declarations
- */
-#ifndef __included_nat64_h__
-#define __included_nat64_h__
-
-#include 
-#include 
-
-#define foreach_nat64_tcp_ses_state            \
-  _(0, CLOSED, "closed")                       \
-  _(1, V4_INIT, "v4-init")                     \
-  _(2, V6_INIT, "v6-init")                     \
-  _(3, ESTABLISHED, "established")             \
-  _(4, V4_FIN_RCV, "v4-fin-rcv")               \
-  _(5, V6_FIN_RCV, "v6-fin-rcv")               \
-  _(6, V6_FIN_V4_FIN_RCV, "v6-fin-v4-fin-rcv") \
-  _(7, TRANS, "trans")
-
-typedef enum
-{
-#define _(v, N, s) NAT64_TCP_STATE_##N = v,
-  foreach_nat64_tcp_ses_state
-#undef _
-} nat64_tcp_ses_state_t;
-
-typedef struct
-{
-  ip6_address_t prefix;
-  u8 plen;
-  u32 vrf_id;
-  u32 fib_index;
-} nat64_prefix_t;
-
-typedef struct
-{
-  /** Interface pool */
-  snat_interface_t *interfaces;
-
-  /** Address pool vector */
-  snat_address_t *addr_pool;
-
-  /** Pref64 vector */
-  nat64_prefix_t *pref64;
-
-  /** BIB and session DB */
-  nat64_db_t db;
-
-  /* values of various timeouts */
-  u32 udp_timeout;
-  u32 icmp_timeout;
-  u32 tcp_trans_timeout;
-  u32 tcp_est_timeout;
-  u32 tcp_incoming_syn_timeout;
-
-  u8 is_disabled;
-
-  snat_main_t *sm;
-} nat64_main_t;
-
-extern nat64_main_t nat64_main;
-extern vlib_node_registration_t nat64_in2out_node;
-extern vlib_node_registration_t nat64_out2in_node;
-
-/**
- * @brief Add/delete address to NAT64 pool.
- *
- * @param addr   IPv4 address.
- * @param vrf_id VRF id of tenant, ~0 means independent of VRF.
- * @param is_add 1 if add, 0 if delete.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add);
-
-/**
- * @brief Call back function when walking addresses in NAT64 pool, non-zero
- * return value stop walk.
- */
-typedef int (*nat64_pool_addr_walk_fn_t) (snat_address_t * addr, void *ctx);
-
-/**
- * @brief Walk NAT64 pool.
- *
- * @param fn The function to invoke on each entry visited.
- * @param ctx A context passed in the visit function.
- */
-void nat64_pool_addr_walk (nat64_pool_addr_walk_fn_t fn, void *ctx);
-
-/**
- * @brief Enable/disable NAT64 feature on the interface.
- *
- * @param sw_if_index Index of the interface.
- * @param is_inside   1 if inside, 0 if outside.
- * @param is_add      1 if add, 0 if delete.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_add_del_interface (u32 sw_if_index, u8 is_inside, u8 is_add);
-
-/**
- * @brief Call back function when walking interfaces with NAT64 feature,
- * non-zero return value stop walk.
- */
-typedef int (*nat64_interface_walk_fn_t) (snat_interface_t * i, void *ctx);
-
-/**
- * @brief Walk NAT64 interfaces.
- *
- * @param fn The function to invoke on each entry visited.
- * @param ctx A context passed in the visit function.
- */
-void nat64_interfaces_walk (nat64_interface_walk_fn_t fn, void *ctx);
-
-/**
- * @brief Initialize NAT64.
- *
- * @param vm vlib main.
- *
- * @return error code.
- */
-clib_error_t *nat64_init (vlib_main_t * vm);
-
-/**
- * @brief Add/delete static NAT64 BIB entry.
- *
- * @param in_addr  Inside IPv6 address.
- * @param out_addr Outside IPv4 address.
- * @param in_port  Inside port number.
- * @param out_port Outside port number.
- * @param proto    L4 protocol.
- * @param vrf_id   VRF id of tenant.
- * @param is_add   1 if add, 0 if delete.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
-				    ip4_address_t * out_addr, u16 in_port,
-				    u16 out_port, u8 proto, u32 vrf_id,
-				    u8 is_add);
-
-/**
- * @brief Alloce IPv4 address and port pair from NAT64 pool.
- *
- * @param fib_index FIB index of tenant.
- * @param proto     L4 protocol.
- * @param addr      Allocated IPv4 address.
- * @param port      Allocated port number.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_alloc_out_addr_and_port (u32 fib_index, snat_protocol_t proto,
-				   ip4_address_t * addr, u16 * port);
-
-/**
- * @brief Free IPv4 address and port pair from NAT64 pool.
- *
- * @param addr  IPv4 address to free.
- * @param port  Port number to free.
- * @param proto L4 protocol.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-void nat64_free_out_addr_and_port (ip4_address_t * addr, u16 port,
-				   snat_protocol_t proto);
-
-/**
- * @brief Set UDP session timeout.
- *
- * @param timeout Timeout value in seconds (if 0 reset to default value 300sec).
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_set_udp_timeout (u32 timeout);
-
-/**
- * @brief Get UDP session timeout.
- *
- * @returns UDP session timeout in seconds.
- */
-u32 nat64_get_udp_timeout (void);
-
-/**
- * @brief Set ICMP session timeout.
- *
- * @param timeout Timeout value in seconds (if 0 reset to default value 60sec).
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_set_icmp_timeout (u32 timeout);
-
-/**
- * @brief Get ICMP session timeout.
- *
- * @returns ICMP session timeout in seconds.
- */
-u32 nat64_get_icmp_timeout (void);
-
-/**
- * @brief Set TCP session timeouts.
- *
- * @param trans Transitory timeout in seconds (if 0 reset to default value 240sec).
- * @param est Established timeout in seconds (if 0 reset to default value 7440sec).
- * @param incoming_syn Incoming SYN timeout in seconds (if 0 reset to default value 6sec).
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_set_tcp_timeouts (u32 trans, u32 est, u32 incoming_syn);
-
-/**
- * @brief Get TCP transitory timeout.
- *
- * @returns TCP transitory timeout in seconds.
- */
-u32 nat64_get_tcp_trans_timeout (void);
-
-/**
- * @brief Get TCP established timeout.
- *
- * @returns TCP established timeout in seconds.
- */
-u32 nat64_get_tcp_est_timeout (void);
-
-/**
- * @brief Get TCP incoming SYN timeout.
- *
- * @returns TCP incoming SYN timeout in seconds.
- */
-u32 nat64_get_tcp_incoming_syn_timeout (void);
-
-/**
- * @brief Reset NAT64 session timeout.
- *
- * @param ste Session table entry.
- * @param vm VLIB main.
- **/
-void nat64_session_reset_timeout (nat64_db_st_entry_t * ste,
-				  vlib_main_t * vm);
-
-/**
- * @brief Set NAT64 TCP session state.
- *
- * @param ste Session table entry.
- * @param tcp TCP header.
- * @param is_ip6 1 if IPv6 packet, 0 if IPv4.
- */
-void nat64_tcp_session_set_state (nat64_db_st_entry_t * ste,
-				  tcp_header_t * tcp, u8 is_ip6);
-
-/**
- * @brief Add/delete NAT64 prefix.
- *
- * @param prefix NAT64 prefix.
- * @param plen Prefix length.
- * @param vrf_id VRF id of tenant.
- * @param is_add 1 if add, 0 if delete.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_add_del_prefix (ip6_address_t * prefix, u8 plen, u32 vrf_id,
-			  u8 is_add);
-
-/**
- * @brief Call back function when walking addresses in NAT64 prefixes, non-zero
- * return value stop walk.
- */
-typedef int (*nat64_prefix_walk_fn_t) (nat64_prefix_t * pref64, void *ctx);
-
-/**
- * @brief Walk NAT64 prefixes.
- *
- * @param fn The function to invoke on each entry visited.
- * @param ctx A context passed in the visit function.
- */
-void nat64_prefix_walk (nat64_prefix_walk_fn_t fn, void *ctx);
-
-/**
- * Compose IPv4-embedded IPv6 addresses.
- * @param ip6 IPv4-embedded IPv6 addresses.
- * @param ip4 IPv4 address.
- * @param fib_index Tenant FIB index.
- */
-void nat64_compose_ip6 (ip6_address_t * ip6, ip4_address_t * ip4,
-			u32 fib_index);
-
-/**
- * Extract IPv4 address from the IPv4-embedded IPv6 addresses.
- *
- * @param ip6 IPv4-embedded IPv6 addresses.
- * @param ip4 IPv4 address.
- * @param fib_index Tenant FIB index.
- */
-void nat64_extract_ip4 (ip6_address_t * ip6, ip4_address_t * ip4,
-			u32 fib_index);
-
-#define u8_ptr_add(ptr, index) (((u8 *)ptr) + index)
-#define u16_net_add(u, val) clib_host_to_net_u16(clib_net_to_host_u16(u) + (val))
-
-#endif /* __included_nat64_h__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/nat64_cli.c b/src/plugins/snat/nat64_cli.c
deleted file mode 100644
index ca60b12c..00000000
--- a/src/plugins/snat/nat64_cli.c
+++ /dev/null
@@ -1,983 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 CLI
- */
-
-#include 
-#include 
-#include 
-
-static clib_error_t *
-nat64_add_del_pool_addr_command_fn (vlib_main_t * vm,
-				    unformat_input_t * input,
-				    vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  ip4_address_t start_addr, end_addr, this_addr;
-  u32 start_host_order, end_host_order;
-  int i, count, rv;
-  u32 vrf_id = ~0;
-  u8 is_add = 1;
-  clib_error_t *error = 0;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U - %U",
-		    unformat_ip4_address, &start_addr,
-		    unformat_ip4_address, &end_addr))
-	;
-      else if (unformat (line_input, "tenant-vrf %u", &vrf_id))
-	;
-      else if (unformat (line_input, "%U", unformat_ip4_address, &start_addr))
-	end_addr = start_addr;
-      else if (unformat (line_input, "del"))
-	is_add = 0;
-      else
-	{
-	  error = clib_error_return (0, "unknown input '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
-  end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
-
-  if (end_host_order < start_host_order)
-    {
-      error = clib_error_return (0, "end address less than start address");
-      goto done;
-    }
-
-  count = (end_host_order - start_host_order) + 1;
-  this_addr = start_addr;
-
-  for (i = 0; i < count; i++)
-    {
-      rv = nat64_add_del_pool_addr (&this_addr, vrf_id, is_add);
-
-      switch (rv)
-	{
-	case VNET_API_ERROR_NO_SUCH_ENTRY:
-	  error =
-	    clib_error_return (0, "NAT64 pool address %U not exist.",
-			       format_ip4_address, &this_addr);
-	  goto done;
-	case VNET_API_ERROR_VALUE_EXIST:
-	  error =
-	    clib_error_return (0, "NAT64 pool address %U exist.",
-			       format_ip4_address, &this_addr);
-	  goto done;
-	default:
-	  break;
-
-	}
-      increment_v4_address (&this_addr);
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-static int
-nat64_cli_pool_walk (snat_address_t * ap, void *ctx)
-{
-  vlib_main_t *vm = ctx;
-
-  if (ap->fib_index != ~0)
-    {
-      fib_table_t *fib;
-      fib = fib_table_get (ap->fib_index, FIB_PROTOCOL_IP6);
-      if (!fib)
-	return -1;
-      vlib_cli_output (vm, " %U tenant VRF: %u", format_ip4_address,
-		       &ap->addr, fib->ft_table_id);
-    }
-  else
-    vlib_cli_output (vm, " %U", format_ip4_address, &ap->addr);
-
-  return 0;
-}
-
-static clib_error_t *
-nat64_show_pool_command_fn (vlib_main_t * vm,
-			    unformat_input_t * input,
-			    vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  vlib_cli_output (vm, "NAT64 pool:");
-  nat64_pool_addr_walk (nat64_cli_pool_walk, vm);
-
-  return 0;
-}
-
-static clib_error_t *
-nat64_interface_feature_command_fn (vlib_main_t * vm,
-				    unformat_input_t *
-				    input, vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  vnet_main_t *vnm = vnet_get_main ();
-  clib_error_t *error = 0;
-  u32 sw_if_index;
-  u32 *inside_sw_if_indices = 0;
-  u32 *outside_sw_if_indices = 0;
-  u8 is_add = 1;
-  int i, rv;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "in %U", unformat_vnet_sw_interface,
-		    vnm, &sw_if_index))
-	vec_add1 (inside_sw_if_indices, sw_if_index);
-      else if (unformat (line_input, "out %U", unformat_vnet_sw_interface,
-			 vnm, &sw_if_index))
-	vec_add1 (outside_sw_if_indices, sw_if_index);
-      else if (unformat (line_input, "del"))
-	is_add = 0;
-      else
-	{
-	  error = clib_error_return (0, "unknown input '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (vec_len (inside_sw_if_indices))
-    {
-      for (i = 0; i < vec_len (inside_sw_if_indices); i++)
-	{
-	  sw_if_index = inside_sw_if_indices[i];
-	  rv = nat64_add_del_interface (sw_if_index, 1, is_add);
-	  switch (rv)
-	    {
-	    case VNET_API_ERROR_NO_SUCH_ENTRY:
-	      error =
-		clib_error_return (0, "%U NAT64 feature not enabled.",
-				   format_vnet_sw_interface_name, vnm,
-				   vnet_get_sw_interface (vnm, sw_if_index));
-	      goto done;
-	    case VNET_API_ERROR_VALUE_EXIST:
-	      error =
-		clib_error_return (0, "%U NAT64 feature already enabled.",
-				   format_vnet_sw_interface_name, vnm,
-				   vnet_get_sw_interface (vnm, sw_if_index));
-	      goto done;
-	    case VNET_API_ERROR_INVALID_VALUE:
-	    case VNET_API_ERROR_INVALID_VALUE_2:
-	      error =
-		clib_error_return (0,
-				   "%U NAT64 feature enable/disable failed.",
-				   format_vnet_sw_interface_name, vnm,
-				   vnet_get_sw_interface (vnm, sw_if_index));
-	      goto done;
-	    default:
-	      break;
-
-	    }
-	}
-    }
-
-  if (vec_len (outside_sw_if_indices))
-    {
-      for (i = 0; i < vec_len (outside_sw_if_indices); i++)
-	{
-	  sw_if_index = outside_sw_if_indices[i];
-	  rv = nat64_add_del_interface (sw_if_index, 0, is_add);
-	  switch (rv)
-	    {
-	    case VNET_API_ERROR_NO_SUCH_ENTRY:
-	      error =
-		clib_error_return (0, "%U NAT64 feature not enabled.",
-				   format_vnet_sw_interface_name, vnm,
-				   vnet_get_sw_interface (vnm, sw_if_index));
-	      goto done;
-	    case VNET_API_ERROR_VALUE_EXIST:
-	      error =
-		clib_error_return (0, "%U NAT64 feature already enabled.",
-				   format_vnet_sw_interface_name, vnm,
-				   vnet_get_sw_interface (vnm, sw_if_index));
-	      goto done;
-	    case VNET_API_ERROR_INVALID_VALUE:
-	    case VNET_API_ERROR_INVALID_VALUE_2:
-	      error =
-		clib_error_return (0,
-				   "%U NAT64 feature enable/disable failed.",
-				   format_vnet_sw_interface_name, vnm,
-				   vnet_get_sw_interface (vnm, sw_if_index));
-	      goto done;
-	    default:
-	      break;
-
-	    }
-	}
-    }
-
-done:
-  unformat_free (line_input);
-  vec_free (inside_sw_if_indices);
-  vec_free (outside_sw_if_indices);
-
-  return error;
-}
-
-static int
-nat64_cli_interface_walk (snat_interface_t * i, void *ctx)
-{
-  vlib_main_t *vm = ctx;
-  vnet_main_t *vnm = vnet_get_main ();
-
-  vlib_cli_output (vm, " %U %s", format_vnet_sw_interface_name, vnm,
-		   vnet_get_sw_interface (vnm, i->sw_if_index),
-		   i->is_inside ? "in" : "out");
-  return 0;
-}
-
-static clib_error_t *
-nat64_show_interfaces_command_fn (vlib_main_t * vm,
-				  unformat_input_t *
-				  input, vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  vlib_cli_output (vm, "NAT64 interfaces:");
-  nat64_interfaces_walk (nat64_cli_interface_walk, vm);
-
-  return 0;
-}
-
-static clib_error_t *
-nat64_add_del_static_bib_command_fn (vlib_main_t *
-				     vm,
-				     unformat_input_t
-				     * input, vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t *error = 0;
-  u8 is_add = 1;
-  ip6_address_t in_addr;
-  ip4_address_t out_addr;
-  u16 in_port = 0;
-  u16 out_port = 0;
-  u32 vrf_id = 0, protocol;
-  snat_protocol_t proto = 0;
-  u8 p = 0;
-  int rv;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U %u", unformat_ip6_address,
-		    &in_addr, &in_port))
-	;
-      else if (unformat (line_input, "%U %u", unformat_ip4_address,
-			 &out_addr, &out_port))
-	;
-      else if (unformat (line_input, "vrf %u", &vrf_id))
-	;
-      else if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
-	;
-      else
-	if (unformat
-	    (line_input, "%U %U %u", unformat_ip6_address, &in_addr,
-	     unformat_ip4_address, &out_addr, &protocol))
-	p = (u8) protocol;
-      else if (unformat (line_input, "del"))
-	is_add = 0;
-      else
-	{
-	  error = clib_error_return (0, "unknown input: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (!p)
-    {
-      if (!in_port)
-	{
-	  error =
-	    clib_error_return (0, "inside port and address  must be set");
-	  goto done;
-	}
-
-      if (!out_port)
-	{
-	  error =
-	    clib_error_return (0, "outside port and address  must be set");
-	  goto done;
-	}
-
-      p = snat_proto_to_ip_proto (proto);
-    }
-
-  rv =
-    nat64_add_del_static_bib_entry (&in_addr, &out_addr, in_port, out_port, p,
-				    vrf_id, is_add);
-
-  switch (rv)
-    {
-    case VNET_API_ERROR_NO_SUCH_ENTRY:
-      error = clib_error_return (0, "NAT64 BIB entry not exist.");
-      goto done;
-    case VNET_API_ERROR_VALUE_EXIST:
-      error = clib_error_return (0, "NAT64 BIB entry exist.");
-      goto done;
-    case VNET_API_ERROR_UNSPECIFIED:
-      error = clib_error_return (0, "Crerate NAT64 BIB entry failed.");
-      goto done;
-    case VNET_API_ERROR_INVALID_VALUE:
-      error =
-	clib_error_return (0, "Outside addres %U and port %u already in use.",
-			   format_ip4_address, &out_addr, out_port);
-      goto done;
-    default:
-      break;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-static int
-nat64_cli_bib_walk (nat64_db_bib_entry_t * bibe, void *ctx)
-{
-  vlib_main_t *vm = ctx;
-  fib_table_t *fib;
-
-  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  if (!fib)
-    return -1;
-
-  switch (bibe->proto)
-    {
-    case IP_PROTOCOL_ICMP:
-    case IP_PROTOCOL_TCP:
-    case IP_PROTOCOL_UDP:
-      vlib_cli_output (vm, " %U %u %U %u protocol %U vrf %u %s %u sessions",
-		       format_ip6_address, &bibe->in_addr,
-		       clib_net_to_host_u16 (bibe->in_port),
-		       format_ip4_address, &bibe->out_addr,
-		       clib_net_to_host_u16 (bibe->out_port),
-		       format_snat_protocol,
-		       ip_proto_to_snat_proto (bibe->proto), fib->ft_table_id,
-		       bibe->is_static ? "static" : "dynamic", bibe->ses_num);
-      break;
-    default:
-      vlib_cli_output (vm, " %U %U protocol %u vrf %u %s %u sessions",
-		       format_ip6_address, &bibe->in_addr,
-		       format_ip4_address, &bibe->out_addr,
-		       bibe->proto, fib->ft_table_id,
-		       bibe->is_static ? "static" : "dynamic", bibe->ses_num);
-    }
-  return 0;
-}
-
-static clib_error_t *
-nat64_show_bib_command_fn (vlib_main_t * vm,
-			   unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t *error = 0;
-  u32 proto = ~0;
-  u8 p = 255;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
-    p = snat_proto_to_ip_proto (proto);
-  else if (unformat (line_input, "unknown"))
-    p = 0;
-  else if (unformat (line_input, "all"))
-    ;
-  else
-    {
-      error = clib_error_return (0, "unknown input: '%U'",
-				 format_unformat_error, line_input);
-      goto done;
-    }
-
-  if (p == 255)
-    vlib_cli_output (vm, "NAT64 BIB entries:");
-  else
-    vlib_cli_output (vm, "NAT64 %U BIB entries:", format_snat_protocol,
-		     proto);
-  nat64_db_bib_walk (&nm->db, p, nat64_cli_bib_walk, vm);
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-static clib_error_t *
-nat64_set_timeouts_command_fn (vlib_main_t * vm, unformat_input_t * input,
-			       vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t *error = 0;
-  u32 timeout, tcp_trans, tcp_est, tcp_incoming_syn;
-
-  tcp_trans = nat64_get_tcp_trans_timeout ();
-  tcp_est = nat64_get_tcp_est_timeout ();
-  tcp_incoming_syn = nat64_get_tcp_incoming_syn_timeout ();
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "udp %u", &timeout))
-	{
-	  if (nat64_set_udp_timeout (timeout))
-	    {
-	      error = clib_error_return (0, "Invalid UDP timeout value");
-	      goto done;
-	    }
-	}
-      else if (unformat (line_input, "icmp %u", &timeout))
-	{
-	  if (nat64_set_icmp_timeout (timeout))
-	    {
-	      error = clib_error_return (0, "Invalid ICMP timeout value");
-	      goto done;
-	    }
-	}
-      else if (unformat (line_input, "tcp-trans %u", &tcp_trans))
-	{
-	  if (nat64_set_tcp_timeouts (tcp_trans, tcp_est, tcp_incoming_syn))
-	    {
-	      error =
-		clib_error_return (0, "Invalid TCP transitory tiemout value");
-	      goto done;
-	    }
-	}
-      else if (unformat (line_input, "tcp-est %u", &tcp_est))
-	{
-	  if (nat64_set_tcp_timeouts (tcp_trans, tcp_est, tcp_incoming_syn))
-	    {
-	      error =
-		clib_error_return (0,
-				   "Invalid TCP established tiemout value");
-	      goto done;
-	    }
-	}
-      else
-	if (unformat (line_input, "tcp-incoming-syn %u", &tcp_incoming_syn))
-	{
-	  if (nat64_set_tcp_timeouts (tcp_trans, tcp_est, tcp_incoming_syn))
-	    {
-	      error =
-		clib_error_return (0,
-				   "Invalid TCP incoming SYN tiemout value");
-	      goto done;
-	    }
-	}
-      else if (unformat (line_input, "reset"))
-	{
-	  nat64_set_udp_timeout (0);
-	  nat64_set_icmp_timeout (0);
-	  nat64_set_tcp_timeouts (0, 0, 0);
-	}
-      else
-	{
-	  error = clib_error_return (0, "unknown input '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-static clib_error_t *
-nat64_show_timeouts_command_fn (vlib_main_t * vm, unformat_input_t * input,
-				vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  vlib_cli_output (vm, "NAT64 session timeouts:");
-  vlib_cli_output (vm, " UDP %usec", nat64_get_udp_timeout ());
-  vlib_cli_output (vm, " ICMP %usec", nat64_get_icmp_timeout ());
-  vlib_cli_output (vm, " TCP transitory %usec",
-		   nat64_get_tcp_trans_timeout ());
-  vlib_cli_output (vm, " TCP established %usec",
-		   nat64_get_tcp_est_timeout ());
-  vlib_cli_output (vm, " TCP incoming SYN %usec",
-		   nat64_get_tcp_incoming_syn_timeout ());
-
-  return 0;
-}
-
-static int
-nat64_cli_st_walk (nat64_db_st_entry_t * ste, void *ctx)
-{
-  vlib_main_t *vm = ctx;
-  nat64_main_t *nm = &nat64_main;
-  nat64_db_bib_entry_t *bibe;
-  fib_table_t *fib;
-
-  bibe = nat64_db_bib_entry_by_index (&nm->db, ste->proto, ste->bibe_index);
-  if (!bibe)
-    return -1;
-
-  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  if (!fib)
-    return -1;
-
-  u32 vrf_id = fib->ft_table_id;
-
-  if (ste->proto == IP_PROTOCOL_ICMP)
-    vlib_cli_output (vm, " %U %U %u %U %U %u protocol %U vrf %u",
-		     format_ip6_address, &bibe->in_addr,
-		     format_ip6_address, &ste->in_r_addr,
-		     clib_net_to_host_u16 (bibe->in_port),
-		     format_ip4_address, &bibe->out_addr,
-		     format_ip4_address, &ste->out_r_addr,
-		     clib_net_to_host_u16 (bibe->out_port),
-		     format_snat_protocol,
-		     ip_proto_to_snat_proto (bibe->proto), vrf_id);
-  else if (ste->proto == IP_PROTOCOL_TCP || ste->proto == IP_PROTOCOL_UDP)
-    vlib_cli_output (vm, " %U %u %U %u %U %u %U %u protcol %U vrf %u",
-		     format_ip6_address, &bibe->in_addr,
-		     clib_net_to_host_u16 (bibe->in_port),
-		     format_ip6_address, &ste->in_r_addr,
-		     clib_net_to_host_u16 (ste->r_port),
-		     format_ip4_address, &bibe->out_addr,
-		     clib_net_to_host_u16 (bibe->out_port),
-		     format_ip4_address, &ste->out_r_addr,
-		     clib_net_to_host_u16 (ste->r_port),
-		     format_snat_protocol,
-		     ip_proto_to_snat_proto (bibe->proto), vrf_id);
-  else
-    vlib_cli_output (vm, " %U %U %U %U protocol %u vrf %u",
-		     format_ip6_address, &bibe->in_addr,
-		     format_ip6_address, &ste->in_r_addr,
-		     format_ip4_address, &bibe->out_addr,
-		     format_ip4_address, &ste->out_r_addr,
-		     bibe->proto, vrf_id);
-
-  return 0;
-}
-
-static clib_error_t *
-nat64_show_st_command_fn (vlib_main_t * vm,
-			  unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t *error = 0;
-  u32 proto = ~0;
-  u8 p = 255;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
-    p = snat_proto_to_ip_proto (proto);
-  else if (unformat (line_input, "unknown"))
-    p = 0;
-  else if (unformat (line_input, "all"))
-    ;
-  else
-    {
-      error = clib_error_return (0, "unknown input: '%U'",
-				 format_unformat_error, line_input);
-      goto done;
-    }
-
-  if (p == 255)
-    vlib_cli_output (vm, "NAT64 sessions:");
-  else
-    vlib_cli_output (vm, "NAT64 %U sessions:", format_snat_protocol, proto);
-  nat64_db_st_walk (&nm->db, p, nat64_cli_st_walk, vm);
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-static clib_error_t *
-nat64_add_del_prefix_command_fn (vlib_main_t * vm, unformat_input_t * input,
-				 vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-  clib_error_t *error = 0;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  u8 is_add = 1;
-  u32 vrf_id = 0;
-  ip6_address_t prefix;
-  u32 plen = 0;
-  int rv;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat
-	  (line_input, "%U/%u", unformat_ip6_address, &prefix, &plen))
-	;
-      else if (unformat (line_input, "tenant-vrf %u", &vrf_id))
-	;
-      else if (unformat (line_input, "del"))
-	is_add = 0;
-      else
-	{
-	  error = clib_error_return (0, "unknown input: '%U'",
-				     format_unformat_error, line_input);
-	  goto done;
-	}
-    }
-
-  if (!plen)
-    {
-      error = clib_error_return (0, "NAT64 prefix must be set.");
-      goto done;
-    }
-
-  rv = nat64_add_del_prefix (&prefix, (u8) plen, vrf_id, is_add);
-
-  switch (rv)
-    {
-    case VNET_API_ERROR_NO_SUCH_ENTRY:
-      error = clib_error_return (0, "NAT64 prefix not exist.");
-      goto done;
-    case VNET_API_ERROR_INVALID_VALUE:
-      error = clib_error_return (0, "Invalid prefix length.");
-      goto done;
-    default:
-      break;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-static int
-nat64_cli_prefix_walk (nat64_prefix_t * p, void *ctx)
-{
-  vlib_main_t *vm = ctx;
-
-  vlib_cli_output (vm, " %U/%u tenant-vrf %u",
-		   format_ip6_address, &p->prefix, p->plen, p->vrf_id);
-
-  return 0;
-}
-
-static clib_error_t *
-nat64_show_prefix_command_fn (vlib_main_t * vm, unformat_input_t * input,
-			      vlib_cli_command_t * cmd)
-{
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return clib_error_return (0,
-			      "NAT64 disabled, multi thread not supported");
-
-  vlib_cli_output (vm, "NAT64 prefix:");
-  nat64_prefix_walk (nat64_cli_prefix_walk, vm);
-
-  return 0;
-}
-
-/* *INDENT-OFF* */
-
-/*?
- * @cliexpar
- * @cliexstart{nat64 add pool address}
- * Add/delete NAT64 pool address.
- * To add single NAT64 pool address use:
- *  vpp# nat64 add pool address 10.1.1.10
- * To add NAT64 pool address range use:
- *  vpp# nat64 add pool address 10.1.1.2 - 10.1.1.5
- * To add NAT64 pool address for specific tenant use:
- *  vpp# nat64 add pool address 10.1.1.100 tenant-vrf 100
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (nat64_add_pool_address_command, static) = {
-  .path = "nat64 add pool address",
-  .short_help = "nat64 add pool address  [- ] "
-                "[tenant-vrf ] [del]",
-  .function = nat64_add_del_pool_addr_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{show nat64 pool}
- * Show NAT64 pool.
- *  vpp# show nat64 pool
- *  NAT64 pool:
- *   10.1.1.3 tenant VRF: 0
- *   10.1.1.10 tenant VRF: 10
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (show_nat64_pool_command, static) = {
-  .path = "show nat64 pool",
-  .short_help = "show nat64 pool",
-  .function = nat64_show_pool_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{set interface nat64}
- * Enable/disable NAT64 feature on the interface.
- * To enable NAT64 feature with local (IPv6) network interface
- * GigabitEthernet0/8/0 and external (IPv4) network interface
- * GigabitEthernet0/a/0 use:
- *  vpp# set interface nat64 in GigabitEthernet0/8/0 out GigabitEthernet0/a/0
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (set_interface_nat64_command, static) = {
-  .path = "set interface nat64",
-  .short_help = "set interface nat64 in|out  [del]",
-  .function = nat64_interface_feature_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{show nat64 interfaces}
- * Show interfaces with NAT64 feature.
- * To show interfaces with NAT64 feature use:
- *  vpp# show nat64 interfaces
- *  NAT64 interfaces:
- *   GigabitEthernet0/8/0 in
- *   GigabitEthernet0/a/0 out
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (show_nat64_interfaces_command, static) = {
-  .path = "show nat64 interfaces",
-  .short_help = "show nat64 interfaces",
-  .function = nat64_show_interfaces_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{nat64 add static bib}
- * Add/delete NAT64 static BIB entry.
- * To create NAT64 satatic BIB entry use:
- *  vpp# nat64 add static bib 2001:db8:c000:221:: 1234 10.1.1.3 5678 tcp
- *  vpp# nat64 add static bib 2001:db8:c000:221:: 1234 10.1.1.3 5678 udp vrf 10
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (nat64_add_del_static_bib_command, static) = {
-  .path = "nat64 add static bib",
-  .short_help = "nat64 add static bib     "
-                "tcp|udp|icmp [vfr ] [del]",
-  .function = nat64_add_del_static_bib_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{show nat64 bib}
- * Show NAT64 BIB entries.
- * To show NAT64 TCP BIB entries use:
- *  vpp# show nat64 bib tcp
- *  NAT64 tcp BIB:
- *   fd01:1::2 6303 10.0.0.3 62303 tcp vrf 0 dynamic 1 sessions
- *   2001:db8:c000:221:: 1234 10.1.1.3 5678 tcp vrf 0 static 2 sessions
- * To show NAT64 UDP BIB entries use:
- *  vpp# show nat64 bib udp
- *  NAT64 udp BIB:
- *   fd01:1::2 6304 10.0.0.3 10546 udp vrf 0 dynamic 10 sessions
- *   2001:db8:c000:221:: 1234 10.1.1.3 5678 udp vrf 10 static 0 sessions
- * To show NAT64 ICMP BIB entries use:
- *  vpp# show nat64 bib icmp
- *  NAT64 icmp BIB:
- *   fd01:1::2 6305 10.0.0.3 63209 icmp vrf 10 dynamic 1 sessions
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (show_nat64_bib_command, static) = {
-  .path = "show nat64 bib",
-  .short_help = "show nat64 bib all|tcp|udp|icmp|unknown",
-  .function = nat64_show_bib_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{set nat64 timeouts}
- * Set NAT64 session timeouts (in seconds).
- * To set NAT64 session timeoutes use use:
- *  vpp# set nat64 timeouts udp 200 icmp 30 tcp-trans 250 tcp-est 7450
- * To reset NAT64 session timeoutes to default values use:
- *  vpp# set nat64 timeouts reset
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (set_nat64_timeouts_command, static) = {
-  .path = "set nat64 timeouts",
-  .short_help = "set nat64 timeouts udp  icmp  tcp-trans  "
-                "tcp-est  tcp-incoming-syn  | reset",
-  .function = nat64_set_timeouts_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{show nat64 tiemouts}
- * Show NAT64 session timeouts:
- *  vpp# show nat64 tiemouts
- *  NAT64 session timeouts:
- *   UDP 300sec
- *   ICMP 60sec
- *   TCP transitory 240sec
- *   TCP established 7440sec
- *   TCP incoming SYN 6sec
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (show_nat64_timeouts_command, static) = {
-  .path = "show nat64 tiemouts",
-  .short_help = "show nat64 tiemouts",
-  .function = nat64_show_timeouts_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{show nat64 session table}
- * Show NAT64 session table.
- * To show NAT64 TCP session table use:
- *  vpp# show nat64 session table tcp
- *  NAT64 tcp session table:
- *   fd01:1::2 6303 64:ff9b::ac10:202 20 10.0.0.3 62303 172.16.2.2 20 tcp vrf 0
- *   fd01:3::2 6303 64:ff9b::ac10:202 20 10.0.10.3 21300 172.16.2.2 20 tcp vrf 10
- * To show NAT64 UDP session table use:
- * #vpp show nat64 session table udp
- * NAT64 udp session table:
- *  fd01:1::2 6304 64:ff9b::ac10:202 20 10.0.0.3 10546 172.16.2.2 20 udp vrf 0
- *  fd01:3::2 6304 64:ff9b::ac10:202 20 10.0.10.3 58627 172.16.2.2 20 udp vrf 10
- *  fd01:1::2 1235 64:ff9b::a00:3 4023 10.0.0.3 24488 10.0.0.3 4023 udp vrf 0
- *  fd01:1::3 23 64:ff9b::a00:3 24488 10.0.0.3 4023 10.0.0.3 24488 udp vrf 0
- * To show NAT64 ICMP session table use:
- * #vpp show nat64 session table icmp
- * NAT64 icmp session table:
- *  fd01:1::2 64:ff9b::ac10:202 6305 10.0.0.3 172.16.2.2 63209 icmp vrf 0
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (show_nat64_st_command, static) = {
-  .path = "show nat64 session table",
-  .short_help = "show nat64 session table all|tcp|udp|icmp|unknown",
-  .function = nat64_show_st_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{nat64 add prefix}
- * Set NAT64 prefix for generating IPv6 representations of IPv4 addresses.
- * To set NAT64 global prefix use:
- *  vpp# nat64 add prefix 2001:db8::/32
- * To set NAT64 prefix for specific tenant use:
- *  vpp# nat64 add prefix 2001:db8:122:300::/56 tenant-vrf 10
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (nat64_add_del_prefix_command, static) = {
-  .path = "nat64 add prefix",
-  .short_help = "nat64 add prefix / [tenant-vrf ] "
-                "[del]",
-  .function = nat64_add_del_prefix_command_fn,
-};
-
-/*?
- * @cliexpar
- * @cliexstart{show nat64 prefix}
- * Show NAT64 prefix.
- * To show NAT64 prefix use:
- *  vpp# show nat64 prefix
- *  NAT64 prefix:
- *   2001:db8::/32 tenant-vrf 0
- *   2001:db8:122:300::/56 tenant-vrf 10
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (show_nat64_prefix_command, static) = {
-  .path = "show nat64 prefix",
-  .short_help = "show nat64 prefix",
-  .function = nat64_show_prefix_command_fn,
-};
-
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/nat64_db.c b/src/plugins/snat/nat64_db.c
deleted file mode 100644
index 9584827e..00000000
--- a/src/plugins/snat/nat64_db.c
+++ /dev/null
@@ -1,603 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 DB
- */
-#include 
-
-int
-nat64_db_init (nat64_db_t * db)
-{
-  u32 bib_buckets = 1024;
-  u32 bib_memory_size = 128 << 20;
-  u32 st_buckets = 2048;
-  u32 st_memory_size = 256 << 20;
-
-  clib_bihash_init_24_8 (&db->bib.in2out, "bib-in2out", bib_buckets,
-			 bib_memory_size);
-
-  clib_bihash_init_24_8 (&db->bib.out2in, "bib-out2in", bib_buckets,
-			 bib_memory_size);
-
-  clib_bihash_init_48_8 (&db->st.in2out, "st-in2out", st_buckets,
-			 st_memory_size);
-
-  clib_bihash_init_48_8 (&db->st.out2in, "st-out2in", st_buckets,
-			 st_memory_size);
-
-  return 0;
-}
-
-nat64_db_bib_entry_t *
-nat64_db_bib_entry_create (nat64_db_t * db, ip6_address_t * in_addr,
-			   ip4_address_t * out_addr, u16 in_port,
-			   u16 out_port, u32 fib_index, u8 proto,
-			   u8 is_static)
-{
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_bib_entry_key_t bibe_key;
-  clib_bihash_kv_24_8_t kv;
-
-  /* create pool entry */
-  switch (ip_proto_to_snat_proto (proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      pool_get (db->bib._##n##_bib, bibe); \
-      kv.value = bibe - db->bib._##n##_bib; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      pool_get (db->bib._unk_proto_bib, bibe);
-      kv.value = bibe - db->bib._unk_proto_bib;
-      break;
-    }
-  memset (bibe, 0, sizeof (*bibe));
-  bibe->in_addr.as_u64[0] = in_addr->as_u64[0];
-  bibe->in_addr.as_u64[1] = in_addr->as_u64[1];
-  bibe->in_port = in_port;
-  bibe->out_addr.as_u32 = out_addr->as_u32;
-  bibe->out_port = out_port;
-  bibe->fib_index = fib_index;
-  bibe->proto = proto;
-  bibe->is_static = is_static;
-
-  /* create hash lookup */
-  bibe_key.addr.as_u64[0] = bibe->in_addr.as_u64[0];
-  bibe_key.addr.as_u64[1] = bibe->in_addr.as_u64[1];
-  bibe_key.fib_index = bibe->fib_index;
-  bibe_key.port = bibe->in_port;
-  bibe_key.proto = bibe->proto;
-  bibe_key.rsvd = 0;
-  kv.key[0] = bibe_key.as_u64[0];
-  kv.key[1] = bibe_key.as_u64[1];
-  kv.key[2] = bibe_key.as_u64[2];
-  clib_bihash_add_del_24_8 (&db->bib.in2out, &kv, 1);
-
-  memset (&bibe_key.addr, 0, sizeof (bibe_key.addr));
-  bibe_key.addr.ip4.as_u32 = bibe->out_addr.as_u32;
-  bibe_key.fib_index = 0;
-  bibe_key.port = bibe->out_port;
-  kv.key[0] = bibe_key.as_u64[0];
-  kv.key[1] = bibe_key.as_u64[1];
-  kv.key[2] = bibe_key.as_u64[2];
-  clib_bihash_add_del_24_8 (&db->bib.out2in, &kv, 1);
-
-  return bibe;
-}
-
-void
-nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe)
-{
-  nat64_db_bib_entry_key_t bibe_key;
-  clib_bihash_kv_24_8_t kv;
-  nat64_db_bib_entry_t *bib;
-  u32 *ste_to_be_free = 0, *ste_index, bibe_index;
-  nat64_db_st_entry_t *st, *ste;
-
-  switch (ip_proto_to_snat_proto (bibe->proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      bib = db->bib._##n##_bib; \
-      st = db->st._##n##_st; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      bib = db->bib._unk_proto_bib;
-      st = db->st._unk_proto_st;
-      break;
-    }
-
-  bibe_index = bibe - bib;
-
-  /* delete ST entries for static BIB entry */
-  if (bibe->is_static)
-    {
-      pool_foreach (ste, st, (
-			       {
-			       if (ste->bibe_index == bibe_index)
-			       vec_add1 (ste_to_be_free, ste - st);}
-		    ));
-      vec_foreach (ste_index, ste_to_be_free)
-	nat64_db_st_entry_free (db, pool_elt_at_index (st, ste_index[0]));
-      vec_free (ste_to_be_free);
-    }
-
-  /* delete hash lookup */
-  bibe_key.addr.as_u64[0] = bibe->in_addr.as_u64[0];
-  bibe_key.addr.as_u64[1] = bibe->in_addr.as_u64[1];
-  bibe_key.fib_index = bibe->fib_index;
-  bibe_key.port = bibe->in_port;
-  bibe_key.proto = bibe->proto;
-  bibe_key.rsvd = 0;
-  kv.key[0] = bibe_key.as_u64[0];
-  kv.key[1] = bibe_key.as_u64[1];
-  kv.key[2] = bibe_key.as_u64[2];
-  clib_bihash_add_del_24_8 (&db->bib.in2out, &kv, 0);
-
-  memset (&bibe_key.addr, 0, sizeof (bibe_key.addr));
-  bibe_key.addr.ip4.as_u32 = bibe->out_addr.as_u32;
-  bibe_key.fib_index = 0;
-  bibe_key.port = bibe->out_port;
-  kv.key[0] = bibe_key.as_u64[0];
-  kv.key[1] = bibe_key.as_u64[1];
-  kv.key[2] = bibe_key.as_u64[2];
-  clib_bihash_add_del_24_8 (&db->bib.out2in, &kv, 0);
-
-  /* delete from pool */
-  pool_put (bib, bibe);
-
-}
-
-nat64_db_bib_entry_t *
-nat64_db_bib_entry_find (nat64_db_t * db, ip46_address_t * addr, u16 port,
-			 u8 proto, u32 fib_index, u8 is_ip6)
-{
-  nat64_db_bib_entry_t *bibe = 0;
-  nat64_db_bib_entry_key_t bibe_key;
-  clib_bihash_kv_24_8_t kv, value;
-  nat64_db_bib_entry_t *bib;
-
-  switch (ip_proto_to_snat_proto (proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      bib = db->bib._##n##_bib; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      bib = db->bib._unk_proto_bib;
-      break;
-    }
-
-  bibe_key.addr.as_u64[0] = addr->as_u64[0];
-  bibe_key.addr.as_u64[1] = addr->as_u64[1];
-  bibe_key.fib_index = fib_index;
-  bibe_key.port = port;
-  bibe_key.proto = proto;
-  bibe_key.rsvd = 0;
-
-  kv.key[0] = bibe_key.as_u64[0];
-  kv.key[1] = bibe_key.as_u64[1];
-  kv.key[2] = bibe_key.as_u64[2];
-
-  if (!clib_bihash_search_24_8
-      (is_ip6 ? &db->bib.in2out : &db->bib.out2in, &kv, &value))
-    bibe = pool_elt_at_index (bib, value.value);
-
-  return bibe;
-}
-
-void
-nat64_db_bib_walk (nat64_db_t * db, u8 proto,
-		   nat64_db_bib_walk_fn_t fn, void *ctx)
-{
-  nat64_db_bib_entry_t *bib, *bibe;
-
-  if (proto == 255)
-    {
-    /* *INDENT-OFF* */
-    #define _(N, i, n, s) \
-      bib = db->bib._##n##_bib; \
-      pool_foreach (bibe, bib, ({ \
-        if (fn (bibe, ctx)) \
-          return; \
-      }));
-      foreach_snat_protocol
-    #undef _
-      bib = db->bib._unk_proto_bib;
-      pool_foreach (bibe, bib, ({
-        if (fn (bibe, ctx))
-          return;
-      }));
-    /* *INDENT-ON* */
-    }
-  else
-    {
-      switch (ip_proto_to_snat_proto (proto))
-	{
-    /* *INDENT-OFF* */
-    #define _(N, i, n, s) \
-        case SNAT_PROTOCOL_##N: \
-          bib = db->bib._##n##_bib; \
-          break;
-          foreach_snat_protocol
-    #undef _
-    /* *INDENT-ON* */
-	default:
-	  bib = db->bib._unk_proto_bib;
-	  break;
-	}
-
-      /* *INDENT-OFF* */
-      pool_foreach (bibe, bib,
-      ({
-        if (fn (bibe, ctx))
-          return;
-      }));
-      /* *INDENT-ON* */
-    }
-}
-
-nat64_db_bib_entry_t *
-nat64_db_bib_entry_by_index (nat64_db_t * db, u8 proto, u32 bibe_index)
-{
-  nat64_db_bib_entry_t *bib;
-
-  switch (ip_proto_to_snat_proto (proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      bib = db->bib._##n##_bib; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      bib = db->bib._unk_proto_bib;
-      break;
-    }
-
-  return pool_elt_at_index (bib, bibe_index);
-}
-
-void
-nat64_db_st_walk (nat64_db_t * db, u8 proto,
-		  nat64_db_st_walk_fn_t fn, void *ctx)
-{
-  nat64_db_st_entry_t *st, *ste;
-
-  if (proto == 255)
-    {
-    /* *INDENT-OFF* */
-    #define _(N, i, n, s) \
-      st = db->st._##n##_st; \
-      pool_foreach (ste, st, ({ \
-        if (fn (ste, ctx)) \
-          return; \
-      }));
-      foreach_snat_protocol
-    #undef _
-      st = db->st._unk_proto_st;
-      pool_foreach (ste, st, ({
-        if (fn (ste, ctx))
-          return;
-      }));
-    /* *INDENT-ON* */
-    }
-  else
-    {
-      switch (ip_proto_to_snat_proto (proto))
-	{
-    /* *INDENT-OFF* */
-    #define _(N, i, n, s) \
-        case SNAT_PROTOCOL_##N: \
-          st = db->st._##n##_st; \
-          break;
-          foreach_snat_protocol
-    #undef _
-    /* *INDENT-ON* */
-	default:
-	  st = db->st._unk_proto_st;
-	  break;
-	}
-
-      /* *INDENT-OFF* */
-      pool_foreach (ste, st,
-      ({
-        if (fn (ste, ctx))
-          return;
-      }));
-      /* *INDENT-ON* */
-    }
-}
-
-nat64_db_st_entry_t *
-nat64_db_st_entry_create (nat64_db_t * db, nat64_db_bib_entry_t * bibe,
-			  ip6_address_t * in_r_addr,
-			  ip4_address_t * out_r_addr, u16 r_port)
-{
-  nat64_db_st_entry_t *ste;
-  nat64_db_bib_entry_t *bib;
-  nat64_db_st_entry_key_t ste_key;
-  clib_bihash_kv_48_8_t kv;
-
-  /* create pool entry */
-  switch (ip_proto_to_snat_proto (bibe->proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      pool_get (db->st._##n##_st, ste); \
-      kv.value = ste - db->st._##n##_st; \
-      bib = db->bib._##n##_bib; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      pool_get (db->st._unk_proto_st, ste);
-      kv.value = ste - db->st._unk_proto_st;
-      bib = db->bib._unk_proto_bib;
-      break;
-    }
-  memset (ste, 0, sizeof (*ste));
-  ste->in_r_addr.as_u64[0] = in_r_addr->as_u64[0];
-  ste->in_r_addr.as_u64[1] = in_r_addr->as_u64[1];
-  ste->out_r_addr.as_u32 = out_r_addr->as_u32;
-  ste->r_port = r_port;
-  ste->bibe_index = bibe - bib;
-  ste->proto = bibe->proto;
-
-  /* increment session number for BIB entry */
-  bibe->ses_num++;
-
-  /* create hash lookup */
-  memset (&ste_key, 0, sizeof (ste_key));
-  ste_key.l_addr.as_u64[0] = bibe->in_addr.as_u64[0];
-  ste_key.l_addr.as_u64[1] = bibe->in_addr.as_u64[1];
-  ste_key.r_addr.as_u64[0] = ste->in_r_addr.as_u64[0];
-  ste_key.r_addr.as_u64[1] = ste->in_r_addr.as_u64[1];
-  ste_key.fib_index = bibe->fib_index;
-  ste_key.l_port = bibe->in_port;
-  ste_key.r_port = ste->r_port;
-  ste_key.proto = ste->proto;
-  kv.key[0] = ste_key.as_u64[0];
-  kv.key[1] = ste_key.as_u64[1];
-  kv.key[2] = ste_key.as_u64[2];
-  kv.key[3] = ste_key.as_u64[3];
-  kv.key[4] = ste_key.as_u64[4];
-  kv.key[5] = ste_key.as_u64[5];
-  clib_bihash_add_del_48_8 (&db->st.in2out, &kv, 1);
-
-  memset (&ste_key, 0, sizeof (ste_key));
-  ste_key.l_addr.ip4.as_u32 = bibe->out_addr.as_u32;
-  ste_key.r_addr.ip4.as_u32 = ste->out_r_addr.as_u32;
-  ste_key.l_port = bibe->out_port;
-  ste_key.r_port = ste->r_port;
-  ste_key.proto = ste->proto;
-  kv.key[0] = ste_key.as_u64[0];
-  kv.key[1] = ste_key.as_u64[1];
-  kv.key[2] = ste_key.as_u64[2];
-  kv.key[3] = ste_key.as_u64[3];
-  kv.key[4] = ste_key.as_u64[4];
-  kv.key[5] = ste_key.as_u64[5];
-  clib_bihash_add_del_48_8 (&db->st.out2in, &kv, 1);
-
-  return ste;
-}
-
-void
-nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste)
-{
-  nat64_db_st_entry_t *st;
-  nat64_db_bib_entry_t *bib, *bibe;
-  nat64_db_st_entry_key_t ste_key;
-  clib_bihash_kv_48_8_t kv;
-
-  switch (ip_proto_to_snat_proto (ste->proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      st = db->st._##n##_st; \
-      bib = db->bib._##n##_bib; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      st = db->st._unk_proto_st;
-      bib = db->bib._unk_proto_bib;
-      break;
-    }
-
-  bibe = pool_elt_at_index (bib, ste->bibe_index);
-
-  /* delete hash lookup */
-  memset (&ste_key, 0, sizeof (ste_key));
-  ste_key.l_addr.as_u64[0] = bibe->in_addr.as_u64[0];
-  ste_key.l_addr.as_u64[1] = bibe->in_addr.as_u64[1];
-  ste_key.r_addr.as_u64[0] = ste->in_r_addr.as_u64[0];
-  ste_key.r_addr.as_u64[1] = ste->in_r_addr.as_u64[1];
-  ste_key.fib_index = bibe->fib_index;
-  ste_key.l_port = bibe->in_port;
-  ste_key.r_port = ste->r_port;
-  ste_key.proto = ste->proto;
-  kv.key[0] = ste_key.as_u64[0];
-  kv.key[1] = ste_key.as_u64[1];
-  kv.key[2] = ste_key.as_u64[2];
-  kv.key[3] = ste_key.as_u64[3];
-  kv.key[4] = ste_key.as_u64[4];
-  kv.key[5] = ste_key.as_u64[5];
-  clib_bihash_add_del_48_8 (&db->st.in2out, &kv, 0);
-
-  memset (&ste_key, 0, sizeof (ste_key));
-  ste_key.l_addr.ip4.as_u32 = bibe->out_addr.as_u32;
-  ste_key.r_addr.ip4.as_u32 = ste->out_r_addr.as_u32;
-  ste_key.l_port = bibe->out_port;
-  ste_key.r_port = ste->r_port;
-  ste_key.proto = ste->proto;
-  kv.key[0] = ste_key.as_u64[0];
-  kv.key[1] = ste_key.as_u64[1];
-  kv.key[2] = ste_key.as_u64[2];
-  kv.key[3] = ste_key.as_u64[3];
-  kv.key[4] = ste_key.as_u64[4];
-  kv.key[5] = ste_key.as_u64[5];
-  clib_bihash_add_del_48_8 (&db->st.out2in, &kv, 0);
-
-  /* delete from pool */
-  pool_put (st, ste);
-
-  /* decrement session number for BIB entry */
-  bibe->ses_num--;
-
-  /* delete BIB entry if last session and dynamic */
-  if (!bibe->is_static && !bibe->ses_num)
-    nat64_db_bib_entry_free (db, bibe);
-}
-
-nat64_db_st_entry_t *
-nat64_db_st_entry_find (nat64_db_t * db, ip46_address_t * l_addr,
-			ip46_address_t * r_addr, u16 l_port, u16 r_port,
-			u8 proto, u32 fib_index, u8 is_ip6)
-{
-  nat64_db_st_entry_t *ste = 0;
-  nat64_db_st_entry_t *st;
-  nat64_db_st_entry_key_t ste_key;
-  clib_bihash_kv_48_8_t kv, value;
-
-  switch (ip_proto_to_snat_proto (proto))
-    {
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      st = db->st._##n##_st; \
-      break;
-      foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-    default:
-      st = db->st._unk_proto_st;
-      break;
-    }
-
-  memset (&ste_key, 0, sizeof (ste_key));
-  ste_key.l_addr.as_u64[0] = l_addr->as_u64[0];
-  ste_key.l_addr.as_u64[1] = l_addr->as_u64[1];
-  ste_key.r_addr.as_u64[0] = r_addr->as_u64[0];
-  ste_key.r_addr.as_u64[1] = r_addr->as_u64[1];
-  ste_key.fib_index = fib_index;
-  ste_key.l_port = l_port;
-  ste_key.r_port = r_port;
-  ste_key.proto = proto;
-  kv.key[0] = ste_key.as_u64[0];
-  kv.key[1] = ste_key.as_u64[1];
-  kv.key[2] = ste_key.as_u64[2];
-  kv.key[3] = ste_key.as_u64[3];
-  kv.key[4] = ste_key.as_u64[4];
-  kv.key[5] = ste_key.as_u64[5];
-
-  if (!clib_bihash_search_48_8
-      (is_ip6 ? &db->st.in2out : &db->st.out2in, &kv, &value))
-    ste = pool_elt_at_index (st, value.value);
-
-  return ste;
-}
-
-void
-nad64_db_st_free_expired (nat64_db_t * db, u32 now)
-{
-  u32 *ste_to_be_free = 0, *ste_index;
-  nat64_db_st_entry_t *st, *ste;
-
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-  st = db->st._##n##_st; \
-  pool_foreach (ste, st, ({\
-    if (i == SNAT_PROTOCOL_TCP && !ste->tcp_state) \
-      continue; \
-    if (ste->expire < now) \
-      vec_add1 (ste_to_be_free, ste - st); \
-  })); \
-  vec_foreach (ste_index, ste_to_be_free) \
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0])); \
-  vec_free (ste_to_be_free); \
-  ste_to_be_free = 0;
-  foreach_snat_protocol
-#undef _
-  st = db->st._unk_proto_st;
-  pool_foreach (ste, st, ({
-    if (ste->expire < now)
-      vec_add1 (ste_to_be_free, ste - st);
-  }));
-  vec_foreach (ste_index, ste_to_be_free)
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0]));
-  vec_free (ste_to_be_free);
-/* *INDENT-ON* */
-}
-
-void
-nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr)
-{
-  u32 *ste_to_be_free = 0, *ste_index;
-  nat64_db_st_entry_t *st, *ste;
-  nat64_db_bib_entry_t *bibe;
-
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-  st = db->st._##n##_st; \
-  pool_foreach (ste, st, ({ \
-    bibe = pool_elt_at_index (db->bib._##n##_bib, ste->bibe_index); \
-    if (bibe->out_addr.as_u32 == out_addr->as_u32) \
-      vec_add1 (ste_to_be_free, ste - st); \
-  })); \
-  vec_foreach (ste_index, ste_to_be_free) \
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0])); \
-  vec_free (ste_to_be_free); \
-  ste_to_be_free = 0;
-  foreach_snat_protocol
-#undef _
-  st = db->st._unk_proto_st;
-  pool_foreach (ste, st, ({
-    bibe = pool_elt_at_index (db->bib._unk_proto_bib, ste->bibe_index);
-    if (bibe->out_addr.as_u32 == out_addr->as_u32)
-      vec_add1 (ste_to_be_free, ste - st);
-  }));
-  vec_foreach (ste_index, ste_to_be_free)
-    nat64_db_st_entry_free (db, pool_elt_at_index(st, ste_index[0]));
-  vec_free (ste_to_be_free);
-/* *INDENT-ON* */
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/nat64_db.h b/src/plugins/snat/nat64_db.h
deleted file mode 100644
index 1e2dcc93..00000000
--- a/src/plugins/snat/nat64_db.h
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 DB
- */
-#ifndef __included_nat64_db_h__
-#define __included_nat64_db_h__
-
-#include 
-#include 
-#include 
-
-
-typedef struct
-{
-  union
-  {
-    struct
-    {
-      ip46_address_t addr;
-      u32 fib_index;
-      u16 port;
-      u8 proto;
-      u8 rsvd;
-    };
-    u64 as_u64[3];
-  };
-} nat64_db_bib_entry_key_t;
-
-/* *INDENT-OFF* */
-typedef CLIB_PACKED(struct
-{
-  ip6_address_t in_addr;
-  u16 in_port;
-  ip4_address_t out_addr;
-  u16 out_port;
-  u32 fib_index;
-  u32 ses_num;
-  u8 proto;
-  u8 is_static;
-}) nat64_db_bib_entry_t;
-/* *INDENT-ON* */
-
-typedef struct
-{
-  /* BIBs */
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-  nat64_db_bib_entry_t *_##n##_bib;
-  foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-  nat64_db_bib_entry_t *_unk_proto_bib;
-
-  /* BIB lookup */
-  clib_bihash_24_8_t in2out;
-  clib_bihash_24_8_t out2in;
-} nat64_db_bib_t;
-
-typedef struct
-{
-  union
-  {
-    struct
-    {
-      ip46_address_t l_addr;
-      ip46_address_t r_addr;
-      u32 fib_index;
-      u16 l_port;
-      u16 r_port;
-      u8 proto;
-      u8 rsvd[7];
-    };
-    u64 as_u64[6];
-  };
-} nat64_db_st_entry_key_t;
-
-/* *INDENT-OFF* */
-typedef CLIB_PACKED(struct
-{
-  ip6_address_t in_r_addr;
-  ip4_address_t out_r_addr;
-  u16 r_port;
-  u32 bibe_index;
-  u32 expire;
-  u8 proto;
-  u8 tcp_state;
-}) nat64_db_st_entry_t;
-/* *INDENT-ON* */
-
-typedef struct
-{
-  /* session tables */
-/* *INDENT-OFF* */
-#define _(N, i, n, s) \
-  nat64_db_st_entry_t *_##n##_st;
-  foreach_snat_protocol
-#undef _
-/* *INDENT-ON* */
-  nat64_db_st_entry_t *_unk_proto_st;
-
-  /* session lookup */
-  clib_bihash_48_8_t in2out;
-  clib_bihash_48_8_t out2in;
-} nat64_db_st_t;
-
-typedef struct
-{
-  nat64_db_bib_t bib;
-  nat64_db_st_t st;
-} nat64_db_t;
-
-/**
- * @brief Initialize NAT64 DB.
- *
- * @param db NAT64 DB.
- *
- * @returns 0 on success, non-zero value otherwise.
- */
-int nat64_db_init (nat64_db_t * db);
-
-/**
- * @brief Create new NAT64 BIB entry.
- *
- * @param db NAT64 DB.
- * @param in_addr Inside IPv6 address.
- * @param out_addr Outside IPv4 address.
- * @param in_port Inside port number.
- * @param out_port Outside port number.
- * @param fib_index FIB index.
- * @param proto L4 protocol.
- * @param is_static 1 if static, 0 if dynamic.
- *
- * @returns BIB entry on success, 0 otherwise.
- */
-nat64_db_bib_entry_t *nat64_db_bib_entry_create (nat64_db_t * db,
-						 ip6_address_t * in_addr,
-						 ip4_address_t * out_addr,
-						 u16 in_port, u16 out_port,
-						 u32 fib_index,
-						 u8 proto, u8 is_static);
-
-/**
- * @brief Free NAT64 BIB entry.
- *
- * @param db NAT64 DB.
- * @param bibe BIB entry.
- */
-void nat64_db_bib_entry_free (nat64_db_t * db, nat64_db_bib_entry_t * bibe);
-
-/**
- * @brief Call back function when walking NAT64 BIB, non-zero
- * return value stop walk.
- */
-typedef int (*nat64_db_bib_walk_fn_t) (nat64_db_bib_entry_t * bibe,
-				       void *ctx);
-/**
- * @brief Walk NAT64 BIB.
- *
- * @param db NAT64 DB.
- * @param proto BIB L4 protocol:
- *  - 255 all BIBs
- *  - 6 TCP BIB
- *  - 17 UDP BIB
- *  - 1/58 ICMP BIB
- *  - otherwise "unknown" protocol BIB
- * @param fn The function to invoke on each entry visited.
- * @param ctx A context passed in the visit function.
- */
-void nat64_db_bib_walk (nat64_db_t * db, u8 proto,
-			nat64_db_bib_walk_fn_t fn, void *ctx);
-
-/**
- * @brief Find NAT64 BIB entry.
- *
- * @param db NAT64 DB.
- * @param addr IP address.
- * @param port Port number.
- * @param proto L4 protocol.
- * @param fib_index FIB index.
- * @param is_ip6 1 if find by IPv6 (inside) address, 0 by IPv4 (outside).
- *
- * @return BIB entry if found.
- */
-nat64_db_bib_entry_t *nat64_db_bib_entry_find (nat64_db_t * db,
-					       ip46_address_t * addr,
-					       u16 port,
-					       u8 proto,
-					       u32 fib_index, u8 is_ip6);
-
-/**
- * @brief Get BIB entry by index and protocol.
- *
- * @param db NAT64 DB.
- * @param proto L4 protocol.
- * @param bibe_index BIB entry index.
- *
- * @return BIB entry if found.
- */
-nat64_db_bib_entry_t *nat64_db_bib_entry_by_index (nat64_db_t * db,
-						   u8 proto, u32 bibe_index);
-/**
- * @brief Create new NAT64 session table entry.
- *
- * @param db NAT64 DB.
- * @param bibe Corresponding BIB entry.
- * @param in_r_addr Inside IPv6 address of the remote host.
- * @param out_r_addr Outside IPv4 address of the remote host.
- * @param r_port Remote host port number.
- *
- * @returns BIB entry on success, 0 otherwise.
- */
-nat64_db_st_entry_t *nat64_db_st_entry_create (nat64_db_t * db,
-					       nat64_db_bib_entry_t * bibe,
-					       ip6_address_t * in_r_addr,
-					       ip4_address_t * out_r_addr,
-					       u16 r_port);
-
-/**
- * @brief Free NAT64 session table entry.
- *
- * @param db NAT64 DB.
- * @param ste Session table entry.
- */
-void nat64_db_st_entry_free (nat64_db_t * db, nat64_db_st_entry_t * ste);
-
-/**
- * @brief Find NAT64 session table entry.
- *
- * @param db NAT64 DB.
- * @param l_addr Local host address.
- * @param r_addr Remote host address.
- * @param l_port Local host port number.
- * @param r_port Remote host port number.
- * @param proto L4 protocol.
- * @param fib_index FIB index.
- * @param is_ip6 1 if find by IPv6 (inside) address, 0 by IPv4 (outside).
- *
- * @return BIB entry if found.
- */
-nat64_db_st_entry_t *nat64_db_st_entry_find (nat64_db_t * db,
-					     ip46_address_t * l_addr,
-					     ip46_address_t * r_addr,
-					     u16 l_port, u16 r_port,
-					     u8 proto,
-					     u32 fib_index, u8 is_ip6);
-
-/**
- * @brief Call back function when walking NAT64 session table, non-zero
- * return value stop walk.
- */
-typedef int (*nat64_db_st_walk_fn_t) (nat64_db_st_entry_t * ste, void *ctx);
-
-/**
- * @brief Walk NAT64 session table.
- *
- * @param db NAT64 DB.
- * @param proto L4 protocol:
- *  - 255 all session tables
- *  - 6 TCP session table
- *  - 17 UDP session table
- *  - 1/58 ICMP session table
- *  - otherwise "unknown" protocol session table
- * @param fn The function to invoke on each entry visited.
- * @param ctx A context passed in the visit function.
- */
-void nat64_db_st_walk (nat64_db_t * db, u8 proto,
-		       nat64_db_st_walk_fn_t fn, void *ctx);
-
-/**
- * @brief Free expired session entries in session tables.
- *
- * @param db NAT64 DB.
- * @param now Current time.
- */
-void nad64_db_st_free_expired (nat64_db_t * db, u32 now);
-
-/**
- * @brief Free sessions using specific outside address.
- *
- * @param db NAT64 DB.
- * @param out_addr Outside address to match.
- */
-void nat64_db_free_out_addr (nat64_db_t * db, ip4_address_t * out_addr);
-
-#endif /* __included_nat64_db_h__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/nat64_doc.md b/src/plugins/snat/nat64_doc.md
deleted file mode 100644
index f94467da..00000000
--- a/src/plugins/snat/nat64_doc.md
+++ /dev/null
@@ -1,73 +0,0 @@
-# Stateful NAT64: Network Address and Protocol Translation from IPv6 Clients to IPv4 Servers {#nat64_doc}
-
-## Introduction
-
-Stateful NAT64 in VPP allows IPv6-only clients to contact IPv4 servers using unicast UDP, TCP, or ICMP based on RFC 6146.
-
-## Configuration
-
-### Enable/disable NAT64 feature on the interface
-
-> set interface nat64 in|out  [del]
-
-in: inside/local/IPv6 network
-out: outside/external/IPv4 network
-intfc: interface name
-
-### Add/delete NAT64 pool address
-
-One or more public IPv4 addresses assigned to a NAT64 are shared among several IPv6-only clients.
-
-> nat64 add pool address  [- ] [tenant-vrf ] [del]
-
-ip4-range-start: First IPv4 address of the range 
-ip4-range-end: Last IPv4 address of the range (optional, not used for single address)
-tenant-vrf-id: VRF id of the tenant associated with the pool address (optional, if not set pool address is global)
-
-### Add/delete static BIB entry
-
-Stateful NAT64 also supports IPv4-initiated communications to a subset of the IPv6 hosts through staticaly configured bindings.
-
-> nat64 add static bib     tcp|udp|icmp [vfr ] [del]
-
-ip6-addr: inside IPv6 address of the host
-in-port: inside port or ICMPv6 identifier
-ip4-addr: outside IPv4 address of the host
-out-port: outside port or ICMPv4 identifier
-table-id: VRF id of the tenant associated with the BIB entry (optional, default use global VRF)
-
-### Set NAT64 session timeouts
-
-Session is deleted when timer expires. If all sessions corresponding to a dynamically create BIB entry are deleted, then the BIB entry is also deleted. When packets are flowing sessiom timer is refreshed to keep the session alive.
-
-> set nat64 timeouts udp  icmp  tcp-trans  tcp-est  tcp-incoming-syn  | reset
-
-udp: UDP session timeout value (default 300sec)
-icmp: ICMP session timeout value (default 60sec)
-tcp-trans: transitory TCP session timeout value (default 240sec)
-tcp-est: established TCP session timeout value (default 7440sec)
-tcp-incoming-syn: incoming SYN TCP session timeout value (default 6sec)
-reset: reset timers to default values
-
-### Set NAT64 prefix 
-
-Stateful NAT64 support the algorithm for generating IPv6 representations of IPv4 addresses defined in RFC 6052. If no prefix is configured, Well-Known Prefix (64:ff9b::/96) is used. 
-
-> nat64 add prefix / [tenant-vrf ] [del]
-
-ip6-prefix: IPv6 prefix
-plen: prefix length (valid values: 32, 40, 48, 56, 64, or 96)
-tenant-vrf: VRF id of the tenant associated with the prefix
-
-### Show commands
-
-> show nat64 pool
-> show nat64 interfaces
-> show nat64 bib tcp|udp|icmp
-> show nat64 session table tcp|udp|icmp
-> show nat64 tiemouts
-> show nat64 prefix
-
-## Notes
-
-Multi thread is not supported yet (CLI/API commands are disabled when VPP runs with multiple threads).
diff --git a/src/plugins/snat/nat64_in2out.c b/src/plugins/snat/nat64_in2out.c
deleted file mode 100644
index 8c67fec2..00000000
--- a/src/plugins/snat/nat64_in2out.c
+++ /dev/null
@@ -1,1118 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 IPv6 to IPv4 translation (inside to outside network)
- */
-
-#include 
-#include 
-#include 
-
-typedef struct
-{
-  u32 sw_if_index;
-  u32 next_index;
-  u8 is_slow_path;
-} nat64_in2out_trace_t;
-
-static u8 *
-format_nat64_in2out_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  nat64_in2out_trace_t *t = va_arg (*args, nat64_in2out_trace_t *);
-  char *tag;
-
-  tag = t->is_slow_path ? "NAT64-in2out-slowpath" : "NAT64-in2out";
-
-  s =
-    format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
-	    t->next_index);
-
-  return s;
-}
-
-vlib_node_registration_t nat64_in2out_node;
-vlib_node_registration_t nat64_in2out_slowpath_node;
-
-#define foreach_nat64_in2out_error                 \
-_(UNSUPPORTED_PROTOCOL, "unsupported protocol")    \
-_(IN2OUT_PACKETS, "good in2out packets processed") \
-_(NO_TRANSLATION, "no translation")                \
-_(UNKNOWN, "unknown")
-
-typedef enum
-{
-#define _(sym,str) NAT64_IN2OUT_ERROR_##sym,
-  foreach_nat64_in2out_error
-#undef _
-    NAT64_IN2OUT_N_ERROR,
-} nat64_in2out_error_t;
-
-static char *nat64_in2out_error_strings[] = {
-#define _(sym,string) string,
-  foreach_nat64_in2out_error
-#undef _
-};
-
-typedef enum
-{
-  NAT64_IN2OUT_NEXT_IP4_LOOKUP,
-  NAT64_IN2OUT_NEXT_IP6_LOOKUP,
-  NAT64_IN2OUT_NEXT_DROP,
-  NAT64_IN2OUT_NEXT_SLOWPATH,
-  NAT64_IN2OUT_N_NEXT,
-} nat64_in2out_next_t;
-
-typedef struct nat64_in2out_set_ctx_t_
-{
-  vlib_buffer_t *b;
-  vlib_main_t *vm;
-} nat64_in2out_set_ctx_t;
-
-/**
- * @brief Check whether is a hairpinning.
- *
- * If the destination IP address of the packet is an IPv4 address assigned to
- * the NAT64 itself, then the packet is a hairpin packet.
- *
- * param dst_addr Destination address of the packet.
- *
- * @returns 1 if hairpinning, otherwise 0.
- */
-static_always_inline int
-is_hairpinning (ip6_address_t * dst_addr)
-{
-  nat64_main_t *nm = &nat64_main;
-  int i;
-
-  for (i = 0; i < vec_len (nm->addr_pool); i++)
-    {
-      if (nm->addr_pool[i].addr.as_u32 == dst_addr->as_u32[3])
-	return 1;
-    }
-
-  return 0;
-}
-
-static int
-nat64_in2out_tcp_udp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
-			     void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_in2out_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  u32 sw_if_index, fib_index;
-  udp_header_t *udp = ip6_next_header (ip6);
-  u8 proto = ip6->protocol;
-  u16 sport = udp->src_port;
-  u16 dport = udp->dst_port;
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto,
-			    fib_index, 1);
-
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe =
-	nat64_db_bib_entry_find (&nm->db, &saddr, sport, proto, fib_index, 1);
-
-      if (!bibe)
-	{
-	  u16 out_port;
-	  ip4_address_t out_addr;
-	  if (nat64_alloc_out_addr_and_port
-	      (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
-	       &out_port))
-	    return -1;
-
-	  bibe =
-	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address, &out_addr,
-				       sport, clib_host_to_net_u16 (out_port),
-				       fib_index, proto, 0);
-	  if (!bibe)
-	    return -1;
-	}
-
-      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
-				  &daddr.ip4, dport);
-      if (!ste)
-	return -1;
-    }
-
-  nat64_session_reset_timeout (ste, ctx->vm);
-
-  ip4->src_address.as_u32 = bibe->out_addr.as_u32;
-  udp->src_port = bibe->out_port;
-
-  ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
-
-  if (proto == IP_PROTOCOL_TCP)
-    {
-      u16 *checksum;
-      ip_csum_t csum;
-      tcp_header_t *tcp = ip6_next_header (ip6);
-
-      checksum = &tcp->checksum;
-      csum = ip_csum_sub_even (*checksum, sport);
-      csum = ip_csum_add_even (csum, udp->src_port);
-      *checksum = ip_csum_fold (csum);
-    }
-
-  return 0;
-}
-
-static int
-nat64_in2out_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_in2out_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  u32 sw_if_index, fib_index;
-  icmp46_header_t *icmp = ip6_next_header (ip6);
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
-
-  if (icmp->type == ICMP4_echo_request || icmp->type == ICMP4_echo_reply)
-    {
-      u16 in_id = ((u16 *) (icmp))[2];
-      ste =
-	nat64_db_st_entry_find (&nm->db, &saddr, &daddr, in_id, 0,
-				IP_PROTOCOL_ICMP, fib_index, 1);
-
-      if (ste)
-	{
-	  bibe =
-	    nat64_db_bib_entry_by_index (&nm->db, IP_PROTOCOL_ICMP,
-					 ste->bibe_index);
-	  if (!bibe)
-	    return -1;
-	}
-      else
-	{
-	  bibe =
-	    nat64_db_bib_entry_find (&nm->db, &saddr, in_id,
-				     IP_PROTOCOL_ICMP, fib_index, 1);
-
-	  if (!bibe)
-	    {
-	      u16 out_id;
-	      ip4_address_t out_addr;
-	      if (nat64_alloc_out_addr_and_port
-		  (fib_index, SNAT_PROTOCOL_ICMP, &out_addr, &out_id))
-		return -1;
-
-	      bibe =
-		nat64_db_bib_entry_create (&nm->db, &ip6->src_address,
-					   &out_addr, in_id,
-					   clib_host_to_net_u16 (out_id),
-					   fib_index, IP_PROTOCOL_ICMP, 0);
-	      if (!bibe)
-		return -1;
-	    }
-
-	  nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
-	  ste =
-	    nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
-				      &daddr.ip4, 0);
-	  if (!ste)
-	    return -1;
-	}
-
-      nat64_session_reset_timeout (ste, ctx->vm);
-
-      ip4->src_address.as_u32 = bibe->out_addr.as_u32;
-      ((u16 *) (icmp))[2] = bibe->out_port;
-
-      ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
-    }
-  else
-    {
-      if (!vec_len (nm->addr_pool))
-	return -1;
-
-      ip4->src_address.as_u32 = nm->addr_pool[0].addr.as_u32;
-      nat64_extract_ip4 (&ip6->dst_address, &ip4->dst_address, fib_index);
-    }
-
-  return 0;
-}
-
-static int
-nat64_in2out_inner_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
-				void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_in2out_set_ctx_t *ctx = arg;
-  nat64_db_st_entry_t *ste;
-  nat64_db_bib_entry_t *bibe;
-  ip46_address_t saddr, daddr;
-  u32 sw_if_index, fib_index;
-  u8 proto = ip6->protocol;
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
-
-  if (proto == IP_PROTOCOL_ICMP6)
-    {
-      icmp46_header_t *icmp = ip6_next_header (ip6);
-      u16 in_id = ((u16 *) (icmp))[2];
-      proto = IP_PROTOCOL_ICMP;
-
-      if (!
-	  (icmp->type == ICMP4_echo_request
-	   || icmp->type == ICMP4_echo_reply))
-	return -1;
-
-      ste =
-	nat64_db_st_entry_find (&nm->db, &daddr, &saddr, in_id, 0, proto,
-				fib_index, 1);
-      if (!ste)
-	return -1;
-
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-
-      ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
-      ((u16 *) (icmp))[2] = bibe->out_port;
-      ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
-    }
-  else
-    {
-      udp_header_t *udp = ip6_next_header (ip6);
-      tcp_header_t *tcp = ip6_next_header (ip6);
-      u16 *checksum;
-      ip_csum_t csum;
-
-      u16 sport = udp->src_port;
-      u16 dport = udp->dst_port;
-
-      ste =
-	nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto,
-				fib_index, 1);
-      if (!ste)
-	return -1;
-
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-
-      ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
-      udp->dst_port = bibe->out_port;
-      ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
-
-      if (proto == IP_PROTOCOL_TCP)
-	checksum = &tcp->checksum;
-      else
-	checksum = &udp->checksum;
-      csum = ip_csum_sub_even (*checksum, dport);
-      csum = ip_csum_add_even (csum, udp->dst_port);
-      *checksum = ip_csum_fold (csum);
-    }
-
-  return 0;
-}
-
-typedef struct unk_proto_st_walk_ctx_t_
-{
-  ip6_address_t src_addr;
-  ip6_address_t dst_addr;
-  ip4_address_t out_addr;
-  u32 fib_index;
-  u8 proto;
-} unk_proto_st_walk_ctx_t;
-
-static int
-unk_proto_st_walk (nat64_db_st_entry_t * ste, void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  unk_proto_st_walk_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  ip46_address_t saddr, daddr;
-
-  if (ip46_address_is_equal (&ste->in_r_addr, &ctx->dst_addr))
-    {
-      bibe =
-	nat64_db_bib_entry_by_index (&nm->db, ste->proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-
-      if (ip46_address_is_equal (&bibe->in_addr, &ctx->src_addr)
-	  && bibe->fib_index == ctx->fib_index)
-	{
-	  memset (&saddr, 0, sizeof (saddr));
-	  saddr.ip4.as_u32 = bibe->out_addr.as_u32;
-	  memset (&daddr, 0, sizeof (daddr));
-	  nat64_extract_ip4 (&ctx->dst_addr, &daddr.ip4, ctx->fib_index);
-
-	  if (nat64_db_st_entry_find
-	      (&nm->db, &daddr, &saddr, 0, 0, ctx->proto, ctx->fib_index, 0))
-	    return -1;
-
-	  ctx->out_addr.as_u32 = bibe->out_addr.as_u32;
-	  return 1;
-	}
-    }
-
-  return 0;
-}
-
-static int
-nat64_in2out_unk_proto_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
-			       void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_in2out_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr, addr;
-  u32 sw_if_index, fib_index;
-  u8 proto = ip6->protocol;
-  int i;
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, 0, 0, proto, fib_index,
-			    1);
-
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe =
-	nat64_db_bib_entry_find (&nm->db, &saddr, 0, proto, fib_index, 1);
-
-      if (!bibe)
-	{
-	  /* Choose same out address as for TCP/UDP session to same dst */
-	  unk_proto_st_walk_ctx_t ctx = {
-	    .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
-	    .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
-	    .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
-	    .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
-	    .out_addr.as_u32 = 0,
-	    .fib_index = fib_index,
-	    .proto = proto,
-	  };
-
-	  nat64_db_st_walk (&nm->db, IP_PROTOCOL_TCP, unk_proto_st_walk,
-			    &ctx);
-
-	  if (!ctx.out_addr.as_u32)
-	    nat64_db_st_walk (&nm->db, IP_PROTOCOL_UDP, unk_proto_st_walk,
-			      &ctx);
-
-	  /* Verify if out address is not already in use for protocol */
-	  memset (&addr, 0, sizeof (addr));
-	  addr.ip4.as_u32 = ctx.out_addr.as_u32;
-	  if (nat64_db_bib_entry_find (&nm->db, &addr, 0, proto, 0, 0))
-	    ctx.out_addr.as_u32 = 0;
-
-	  if (!ctx.out_addr.as_u32)
-	    {
-	      for (i = 0; i < vec_len (nm->addr_pool); i++)
-		{
-		  addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
-		  if (!nat64_db_bib_entry_find
-		      (&nm->db, &addr, 0, proto, 0, 0))
-		    break;
-		}
-	    }
-
-	  if (!ctx.out_addr.as_u32)
-	    return -1;
-
-	  bibe =
-	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address,
-				       &ctx.out_addr, 0, 0, fib_index, proto,
-				       0);
-	  if (!bibe)
-	    return -1;
-	}
-
-      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
-				  &daddr.ip4, 0);
-      if (!ste)
-	return -1;
-    }
-
-  nat64_session_reset_timeout (ste, ctx->vm);
-
-  ip4->src_address.as_u32 = bibe->out_addr.as_u32;
-  ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
-
-  return 0;
-}
-
-
-
-static int
-nat64_in2out_tcp_udp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
-				  ip6_header_t * ip6)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  u32 sw_if_index, fib_index;
-  udp_header_t *udp = ip6_next_header (ip6);
-  tcp_header_t *tcp = ip6_next_header (ip6);
-  u8 proto = ip6->protocol;
-  u16 sport = udp->src_port;
-  u16 dport = udp->dst_port;
-  u16 *checksum;
-  ip_csum_t csum;
-
-  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
-
-  if (proto == IP_PROTOCOL_UDP)
-    checksum = &udp->checksum;
-  else
-    checksum = &tcp->checksum;
-
-  csum = ip_csum_sub_even (*checksum, ip6->src_address.as_u64[0]);
-  csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
-  csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
-  csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
-  csum = ip_csum_sub_even (csum, sport);
-  csum = ip_csum_sub_even (csum, dport);
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto,
-			    fib_index, 1);
-
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe =
-	nat64_db_bib_entry_find (&nm->db, &saddr, sport, proto, fib_index, 1);
-
-      if (!bibe)
-	{
-	  u16 out_port;
-	  ip4_address_t out_addr;
-	  if (nat64_alloc_out_addr_and_port
-	      (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
-	       &out_port))
-	    return -1;
-
-	  bibe =
-	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address, &out_addr,
-				       sport, clib_host_to_net_u16 (out_port),
-				       fib_index, proto, 0);
-	  if (!bibe)
-	    return -1;
-	}
-
-      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
-				  &daddr.ip4, dport);
-      if (!ste)
-	return -1;
-    }
-
-  nat64_session_reset_timeout (ste, vm);
-
-  sport = udp->src_port = bibe->out_port;
-  nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  memset (&daddr, 0, sizeof (daddr));
-  saddr.ip4.as_u32 = bibe->out_addr.as_u32;
-  daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto, 0,
-			    0);
-
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe = nat64_db_bib_entry_find (&nm->db, &daddr, dport, proto, 0, 0);
-
-      if (!bibe)
-	return -1;
-
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6->src_address,
-				  &saddr.ip4, sport);
-    }
-
-  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
-  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
-  udp->dst_port = bibe->in_port;
-
-  csum = ip_csum_add_even (csum, ip6->src_address.as_u64[0]);
-  csum = ip_csum_add_even (csum, ip6->src_address.as_u64[1]);
-  csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[0]);
-  csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[1]);
-  csum = ip_csum_add_even (csum, udp->src_port);
-  csum = ip_csum_add_even (csum, udp->dst_port);
-  *checksum = ip_csum_fold (csum);
-
-  return 0;
-}
-
-static int
-nat64_in2out_icmp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
-			       ip6_header_t * ip6)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  icmp46_header_t *icmp = ip6_next_header (ip6);
-  ip6_header_t *inner_ip6;
-  ip46_address_t saddr, daddr;
-  u32 sw_if_index, fib_index;
-  u8 proto;
-  udp_header_t *udp;
-  tcp_header_t *tcp;
-  u16 *checksum, sport, dport;
-  ip_csum_t csum;
-
-  if (icmp->type == ICMP6_echo_request || icmp->type == ICMP6_echo_reply)
-    return -1;
-
-  inner_ip6 = (ip6_header_t *) u8_ptr_add (icmp, 8);
-
-  proto = inner_ip6->protocol;
-
-  if (proto == IP_PROTOCOL_ICMP6)
-    return -1;
-
-  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = inner_ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = inner_ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = inner_ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = inner_ip6->dst_address.as_u64[1];
-
-  udp = ip6_next_header (inner_ip6);
-  tcp = ip6_next_header (inner_ip6);
-
-  sport = udp->src_port;
-  dport = udp->dst_port;
-
-  if (proto == IP_PROTOCOL_UDP)
-    checksum = &udp->checksum;
-  else
-    checksum = &tcp->checksum;
-
-  csum = ip_csum_sub_even (*checksum, inner_ip6->src_address.as_u64[0]);
-  csum = ip_csum_sub_even (csum, inner_ip6->src_address.as_u64[1]);
-  csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[0]);
-  csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[1]);
-  csum = ip_csum_sub_even (csum, sport);
-  csum = ip_csum_sub_even (csum, dport);
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto,
-			    fib_index, 1);
-  if (!ste)
-    return -1;
-
-  bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-  if (!bibe)
-    return -1;
-
-  dport = udp->dst_port = bibe->out_port;
-  nat64_compose_ip6 (&inner_ip6->dst_address, &bibe->out_addr, fib_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  memset (&daddr, 0, sizeof (daddr));
-  saddr.ip4.as_u32 = ste->out_r_addr.as_u32;
-  daddr.ip4.as_u32 = bibe->out_addr.as_u32;
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto, 0,
-			    0);
-  if (!ste)
-    return -1;
-
-  bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-  if (!bibe)
-    return -1;
-
-  inner_ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
-  inner_ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
-  udp->src_port = bibe->in_port;
-
-  csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[0]);
-  csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[1]);
-  csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[0]);
-  csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[1]);
-  csum = ip_csum_add_even (csum, udp->src_port);
-  csum = ip_csum_add_even (csum, udp->dst_port);
-  *checksum = ip_csum_fold (csum);
-
-  if (!vec_len (nm->addr_pool))
-    return -1;
-
-  nat64_compose_ip6 (&ip6->src_address, &nm->addr_pool[0].addr, fib_index);
-  ip6->dst_address.as_u64[0] = inner_ip6->src_address.as_u64[0];
-  ip6->dst_address.as_u64[1] = inner_ip6->src_address.as_u64[1];
-
-  icmp->checksum = 0;
-  csum = ip_csum_with_carry (0, ip6->payload_length);
-  csum = ip_csum_with_carry (csum, clib_host_to_net_u16 (ip6->protocol));
-  csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[0]);
-  csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[1]);
-  csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[0]);
-  csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[1]);
-  csum =
-    ip_incremental_checksum (csum, icmp,
-			     clib_net_to_host_u16 (ip6->payload_length));
-  icmp->checksum = ~ip_csum_fold (csum);
-
-  return 0;
-}
-
-static int
-nat64_in2out_unk_proto_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
-				    ip6_header_t * ip6)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr, addr;
-  u32 sw_if_index, fib_index;
-  u8 proto = ip6->protocol;
-  int i;
-
-  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  saddr.as_u64[0] = ip6->src_address.as_u64[0];
-  saddr.as_u64[1] = ip6->src_address.as_u64[1];
-  daddr.as_u64[0] = ip6->dst_address.as_u64[0];
-  daddr.as_u64[1] = ip6->dst_address.as_u64[1];
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &saddr, &daddr, 0, 0, proto, fib_index,
-			    1);
-
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe =
-	nat64_db_bib_entry_find (&nm->db, &saddr, 0, proto, fib_index, 1);
-
-      if (!bibe)
-	{
-	  /* Choose same out address as for TCP/UDP session to same dst */
-	  unk_proto_st_walk_ctx_t ctx = {
-	    .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
-	    .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
-	    .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
-	    .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
-	    .out_addr.as_u32 = 0,
-	    .fib_index = fib_index,
-	    .proto = proto,
-	  };
-
-	  nat64_db_st_walk (&nm->db, IP_PROTOCOL_TCP, unk_proto_st_walk,
-			    &ctx);
-
-	  if (!ctx.out_addr.as_u32)
-	    nat64_db_st_walk (&nm->db, IP_PROTOCOL_UDP, unk_proto_st_walk,
-			      &ctx);
-
-	  /* Verify if out address is not already in use for protocol */
-	  memset (&addr, 0, sizeof (addr));
-	  addr.ip4.as_u32 = ctx.out_addr.as_u32;
-	  if (nat64_db_bib_entry_find (&nm->db, &addr, 0, proto, 0, 0))
-	    ctx.out_addr.as_u32 = 0;
-
-	  if (!ctx.out_addr.as_u32)
-	    {
-	      for (i = 0; i < vec_len (nm->addr_pool); i++)
-		{
-		  addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
-		  if (!nat64_db_bib_entry_find
-		      (&nm->db, &addr, 0, proto, 0, 0))
-		    break;
-		}
-	    }
-
-	  if (!ctx.out_addr.as_u32)
-	    return -1;
-
-	  bibe =
-	    nat64_db_bib_entry_create (&nm->db, &ip6->src_address,
-				       &ctx.out_addr, 0, 0, fib_index, proto,
-				       0);
-	  if (!bibe)
-	    return -1;
-	}
-
-      nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6->dst_address,
-				  &daddr.ip4, 0);
-      if (!ste)
-	return -1;
-    }
-
-  nat64_session_reset_timeout (ste, vm);
-
-  nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  memset (&daddr, 0, sizeof (daddr));
-  saddr.ip4.as_u32 = bibe->out_addr.as_u32;
-  daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
-
-  ste = nat64_db_st_entry_find (&nm->db, &daddr, &saddr, 0, 0, proto, 0, 0);
-
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe = nat64_db_bib_entry_find (&nm->db, &daddr, 0, proto, 0, 0);
-
-      if (!bibe)
-	return -1;
-
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6->src_address,
-				  &saddr.ip4, 0);
-    }
-
-  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
-  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
-
-  return 0;
-}
-
-static inline uword
-nat64_in2out_node_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
-			     vlib_frame_t * frame, u8 is_slow_path)
-{
-  u32 n_left_from, *from, *to_next;
-  nat64_in2out_next_t next_index;
-  u32 pkts_processed = 0;
-  u32 stats_node_index;
-
-  stats_node_index =
-    is_slow_path ? nat64_in2out_slowpath_node.index : nat64_in2out_node.index;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  u32 next0;
-	  ip6_header_t *ip60;
-	  u16 l4_offset0, frag_offset0;
-	  u8 l4_protocol0;
-	  u32 proto0;
-	  nat64_in2out_set_ctx_t ctx0;
-
-	  /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  ip60 = vlib_buffer_get_current (b0);
-
-	  ctx0.b = b0;
-	  ctx0.vm = vm;
-
-	  next0 = NAT64_IN2OUT_NEXT_IP4_LOOKUP;
-
-	  if (PREDICT_FALSE
-	      (ip6_parse
-	       (ip60, b0->current_length, &l4_protocol0, &l4_offset0,
-		&frag_offset0)))
-	    {
-	      next0 = NAT64_IN2OUT_NEXT_DROP;
-	      b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
-	      goto trace0;
-	    }
-
-	  proto0 = ip_proto_to_snat_proto (l4_protocol0);
-	  if (frag_offset0 != 0)
-	    {
-	      next0 = NAT64_IN2OUT_NEXT_DROP;
-	      b0->error =
-		node->errors[NAT64_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
-	      goto trace0;
-	    }
-
-	  if (is_slow_path)
-	    {
-	      if (PREDICT_TRUE (proto0 == ~0))
-		{
-		  if (is_hairpinning (&ip60->dst_address))
-		    {
-		      next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
-		      if (nat64_in2out_unk_proto_hairpinning (vm, b0, ip60))
-			{
-			  next0 = NAT64_IN2OUT_NEXT_DROP;
-			  b0->error =
-			    node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
-			}
-		      goto trace0;
-		    }
-
-		  if (ip6_to_ip4 (b0, nat64_in2out_unk_proto_set_cb, &ctx0))
-		    {
-		      next0 = NAT64_IN2OUT_NEXT_DROP;
-		      b0->error =
-			node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
-		      goto trace0;
-		    }
-		}
-	      goto trace0;
-	    }
-	  else
-	    {
-	      if (PREDICT_FALSE (proto0 == ~0))
-		{
-		  next0 = NAT64_IN2OUT_NEXT_SLOWPATH;
-		  goto trace0;
-		}
-	    }
-
-	  if (proto0 == SNAT_PROTOCOL_ICMP)
-	    {
-	      if (is_hairpinning (&ip60->dst_address))
-		{
-		  next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
-		  if (nat64_in2out_icmp_hairpinning (vm, b0, ip60))
-		    {
-		      next0 = NAT64_IN2OUT_NEXT_DROP;
-		      b0->error =
-			node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
-		    }
-		  goto trace0;
-		}
-
-	      if (icmp6_to_icmp
-		  (b0, nat64_in2out_icmp_set_cb, &ctx0,
-		   nat64_in2out_inner_icmp_set_cb, &ctx0))
-		{
-		  next0 = NAT64_IN2OUT_NEXT_DROP;
-		  b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
-		  goto trace0;
-		}
-	    }
-	  else if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
-	    {
-	      if (is_hairpinning (&ip60->dst_address))
-		{
-		  next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
-		  if (nat64_in2out_tcp_udp_hairpinning (vm, b0, ip60))
-		    {
-		      next0 = NAT64_IN2OUT_NEXT_DROP;
-		      b0->error =
-			node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
-		    }
-		  goto trace0;
-		}
-
-	      if (ip6_to_ip4_tcp_udp
-		  (b0, nat64_in2out_tcp_udp_set_cb, &ctx0, 0))
-		{
-		  next0 = NAT64_IN2OUT_NEXT_DROP;
-		  b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
-		  goto trace0;
-		}
-	    }
-
-	trace0:
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	    {
-	      nat64_in2out_trace_t *t =
-		vlib_add_trace (vm, node, b0, sizeof (*t));
-	      t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	      t->next_index = next0;
-	      t->is_slow_path = is_slow_path;
-	    }
-
-	  pkts_processed += next0 != NAT64_IN2OUT_NEXT_DROP;
-
-	  /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-					   n_left_to_next, bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  vlib_node_increment_counter (vm, stats_node_index,
-			       NAT64_IN2OUT_ERROR_IN2OUT_PACKETS,
-			       pkts_processed);
-  return frame->n_vectors;
-}
-
-static uword
-nat64_in2out_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
-		      vlib_frame_t * frame)
-{
-  return nat64_in2out_node_fn_inline (vm, node, frame, 0);
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (nat64_in2out_node) = {
-  .function = nat64_in2out_node_fn,
-  .name = "nat64-in2out",
-  .vector_size = sizeof (u32),
-  .format_trace = format_nat64_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
-  .error_strings = nat64_in2out_error_strings,
-  .n_next_nodes = NAT64_IN2OUT_N_NEXT,
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
-    [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
-    [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
-    [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
-  },
-};
-/* *INDENT-ON* */
-
-VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_node, nat64_in2out_node_fn);
-
-static uword
-nat64_in2out_slowpath_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
-			       vlib_frame_t * frame)
-{
-  return nat64_in2out_node_fn_inline (vm, node, frame, 1);
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (nat64_in2out_slowpath_node) = {
-  .function = nat64_in2out_slowpath_node_fn,
-  .name = "nat64-in2out-slowpath",
-  .vector_size = sizeof (u32),
-  .format_trace = format_nat64_in2out_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
-  .error_strings = nat64_in2out_error_strings,
-  .n_next_nodes = NAT64_IN2OUT_N_NEXT,
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
-    [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
-    [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
-    [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
-  },
-};
-/* *INDENT-ON* */
-
-VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_slowpath_node,
-			      nat64_in2out_slowpath_node_fn);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/nat64_out2in.c b/src/plugins/snat/nat64_out2in.c
deleted file mode 100644
index cd5b253a..00000000
--- a/src/plugins/snat/nat64_out2in.c
+++ /dev/null
@@ -1,494 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief NAT64 IPv4 to IPv6 translation (otside to inside network)
- */
-
-#include 
-#include 
-#include 
-
-typedef struct
-{
-  u32 sw_if_index;
-  u32 next_index;
-} nat64_out2in_trace_t;
-
-static u8 *
-format_nat64_out2in_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  nat64_out2in_trace_t *t = va_arg (*args, nat64_out2in_trace_t *);
-
-  s =
-    format (s, "NAT64-out2in: sw_if_index %d, next index %d", t->sw_if_index,
-	    t->next_index);
-
-  return s;
-}
-
-vlib_node_registration_t nat64_out2in_node;
-
-#define foreach_nat64_out2in_error                 \
-_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")    \
-_(OUT2IN_PACKETS, "Good out2in packets processed") \
-_(NO_TRANSLATION, "No translation")                \
-_(UNKNOWN, "unknown")
-
-typedef enum
-{
-#define _(sym,str) NAT64_OUT2IN_ERROR_##sym,
-  foreach_nat64_out2in_error
-#undef _
-    NAT64_OUT2IN_N_ERROR,
-} nat64_out2in_error_t;
-
-static char *nat64_out2in_error_strings[] = {
-#define _(sym,string) string,
-  foreach_nat64_out2in_error
-#undef _
-};
-
-typedef enum
-{
-  NAT64_OUT2IN_NEXT_LOOKUP,
-  NAT64_OUT2IN_NEXT_DROP,
-  NAT64_OUT2IN_N_NEXT,
-} nat64_out2in_next_t;
-
-typedef struct nat64_out2in_set_ctx_t_
-{
-  vlib_buffer_t *b;
-  vlib_main_t *vm;
-} nat64_out2in_set_ctx_t;
-
-static int
-nat64_out2in_tcp_udp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
-			     void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_out2in_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  ip6_address_t ip6_saddr;
-  udp_header_t *udp = ip4_next_header (ip4);
-  tcp_header_t *tcp = ip4_next_header (ip4);
-  u8 proto = ip4->protocol;
-  u16 dport = udp->dst_port;
-  u16 sport = udp->src_port;
-  u32 sw_if_index, fib_index;
-  u16 *checksum;
-  ip_csum_t csum;
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  saddr.ip4.as_u32 = ip4->src_address.as_u32;
-  memset (&daddr, 0, sizeof (daddr));
-  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, dport, sport, proto,
-			    fib_index, 0);
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe =
-	nat64_db_bib_entry_find (&nm->db, &daddr, dport, proto, fib_index, 0);
-
-      if (!bibe)
-	return -1;
-
-      nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6_saddr, &saddr.ip4,
-				  sport);
-    }
-
-  nat64_session_reset_timeout (ste, ctx->vm);
-
-  ip6->src_address.as_u64[0] = ste->in_r_addr.as_u64[0];
-  ip6->src_address.as_u64[1] = ste->in_r_addr.as_u64[1];
-
-  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
-  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
-  udp->dst_port = bibe->in_port;
-
-  if (proto == IP_PROTOCOL_UDP)
-    checksum = &udp->checksum;
-  else
-    checksum = &tcp->checksum;
-  csum = ip_csum_sub_even (*checksum, dport);
-  csum = ip_csum_add_even (csum, udp->dst_port);
-  *checksum = ip_csum_fold (csum);
-
-  vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
-
-  return 0;
-}
-
-static int
-nat64_out2in_icmp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6, void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_out2in_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  ip6_address_t ip6_saddr;
-  u32 sw_if_index, fib_index;
-  icmp46_header_t *icmp = ip4_next_header (ip4);
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  saddr.ip4.as_u32 = ip4->src_address.as_u32;
-  memset (&daddr, 0, sizeof (daddr));
-  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
-
-  if (icmp->type == ICMP6_echo_request || icmp->type == ICMP6_echo_reply)
-    {
-      u16 out_id = ((u16 *) (icmp))[2];
-      ste =
-	nat64_db_st_entry_find (&nm->db, &daddr, &saddr, out_id, 0,
-				IP_PROTOCOL_ICMP, fib_index, 0);
-
-      if (ste)
-	{
-	  bibe =
-	    nat64_db_bib_entry_by_index (&nm->db, IP_PROTOCOL_ICMP,
-					 ste->bibe_index);
-	  if (!bibe)
-	    return -1;
-	}
-      else
-	{
-	  bibe =
-	    nat64_db_bib_entry_find (&nm->db, &daddr, out_id,
-				     IP_PROTOCOL_ICMP, fib_index, 0);
-	  if (!bibe)
-	    return -1;
-
-	  nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
-	  ste =
-	    nat64_db_st_entry_create (&nm->db, bibe, &ip6_saddr, &saddr.ip4,
-				      0);
-	}
-
-      nat64_session_reset_timeout (ste, ctx->vm);
-
-      ip6->src_address.as_u64[0] = ste->in_r_addr.as_u64[0];
-      ip6->src_address.as_u64[1] = ste->in_r_addr.as_u64[1];
-
-      ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
-      ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
-      ((u16 *) (icmp))[2] = bibe->in_port;
-
-      vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
-    }
-  else
-    {
-      ip6_header_t *inner_ip6 = (ip6_header_t *) u8_ptr_add (icmp, 8);
-
-      nat64_compose_ip6 (&ip6->src_address, &ip4->src_address,
-			 vnet_buffer (ctx->b)->sw_if_index[VLIB_TX]);
-      ip6->dst_address.as_u64[0] = inner_ip6->src_address.as_u64[0];
-      ip6->dst_address.as_u64[1] = inner_ip6->src_address.as_u64[1];
-    }
-
-  return 0;
-}
-
-static int
-nat64_out2in_inner_icmp_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
-				void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_out2in_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  u32 sw_if_index, fib_index;
-  u8 proto = ip4->protocol;
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index =
-    fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  saddr.ip4.as_u32 = ip4->src_address.as_u32;
-  memset (&daddr, 0, sizeof (daddr));
-  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
-
-  if (proto == IP_PROTOCOL_ICMP6)
-    {
-      icmp46_header_t *icmp = ip4_next_header (ip4);
-      u16 out_id = ((u16 *) (icmp))[2];
-      proto = IP_PROTOCOL_ICMP;
-
-      if (!
-	  (icmp->type == ICMP6_echo_request
-	   || icmp->type == ICMP6_echo_reply))
-	return -1;
-
-      ste =
-	nat64_db_st_entry_find (&nm->db, &saddr, &daddr, out_id, 0, proto,
-				fib_index, 0);
-      if (!ste)
-	return -1;
-
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-
-      ip6->dst_address.as_u64[0] = ste->in_r_addr.as_u64[0];
-      ip6->dst_address.as_u64[1] = ste->in_r_addr.as_u64[1];
-      ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
-      ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
-      ((u16 *) (icmp))[2] = bibe->in_port;
-
-      vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
-    }
-  else
-    {
-      udp_header_t *udp = ip4_next_header (ip4);
-      tcp_header_t *tcp = ip4_next_header (ip4);
-      u16 dport = udp->dst_port;
-      u16 sport = udp->src_port;
-      u16 *checksum;
-      ip_csum_t csum;
-
-      ste =
-	nat64_db_st_entry_find (&nm->db, &saddr, &daddr, sport, dport, proto,
-				fib_index, 0);
-      if (!ste)
-	return -1;
-
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-
-      nat64_compose_ip6 (&ip6->dst_address, &daddr.ip4, bibe->fib_index);
-      ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
-      ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
-      udp->src_port = bibe->in_port;
-
-      if (proto == IP_PROTOCOL_UDP)
-	checksum = &udp->checksum;
-      else
-	checksum = &tcp->checksum;
-      if (*checksum)
-	{
-	  csum = ip_csum_sub_even (*checksum, sport);
-	  csum = ip_csum_add_even (csum, udp->src_port);
-	  *checksum = ip_csum_fold (csum);
-	}
-
-      vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
-    }
-
-  return 0;
-}
-
-static int
-nat64_out2in_unk_proto_set_cb (ip4_header_t * ip4, ip6_header_t * ip6,
-			       void *arg)
-{
-  nat64_main_t *nm = &nat64_main;
-  nat64_out2in_set_ctx_t *ctx = arg;
-  nat64_db_bib_entry_t *bibe;
-  nat64_db_st_entry_t *ste;
-  ip46_address_t saddr, daddr;
-  ip6_address_t ip6_saddr;
-  u32 sw_if_index, fib_index;
-  u8 proto = ip4->protocol;
-
-  sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
-  fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
-
-  memset (&saddr, 0, sizeof (saddr));
-  saddr.ip4.as_u32 = ip4->src_address.as_u32;
-  memset (&daddr, 0, sizeof (daddr));
-  daddr.ip4.as_u32 = ip4->dst_address.as_u32;
-
-  ste =
-    nat64_db_st_entry_find (&nm->db, &daddr, &saddr, 0, 0, proto, fib_index,
-			    0);
-  if (ste)
-    {
-      bibe = nat64_db_bib_entry_by_index (&nm->db, proto, ste->bibe_index);
-      if (!bibe)
-	return -1;
-    }
-  else
-    {
-      bibe =
-	nat64_db_bib_entry_find (&nm->db, &daddr, 0, proto, fib_index, 0);
-
-      if (!bibe)
-	return -1;
-
-      nat64_compose_ip6 (&ip6_saddr, &ip4->src_address, bibe->fib_index);
-      ste =
-	nat64_db_st_entry_create (&nm->db, bibe, &ip6_saddr, &saddr.ip4, 0);
-    }
-
-  nat64_session_reset_timeout (ste, ctx->vm);
-
-  ip6->src_address.as_u64[0] = ste->in_r_addr.as_u64[0];
-  ip6->src_address.as_u64[1] = ste->in_r_addr.as_u64[1];
-
-  ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
-  ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
-
-  vnet_buffer (ctx->b)->sw_if_index[VLIB_TX] = bibe->fib_index;
-
-  return 0;
-}
-
-static uword
-nat64_out2in_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
-		      vlib_frame_t * frame)
-{
-  u32 n_left_from, *from, *to_next;
-  nat64_out2in_next_t next_index;
-  u32 pkts_processed = 0;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  u32 bi0;
-	  vlib_buffer_t *b0;
-	  u32 next0;
-	  ip4_header_t *ip40;
-	  u32 proto0;
-	  nat64_out2in_set_ctx_t ctx0;
-
-	  /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  ip40 = vlib_buffer_get_current (b0);
-
-	  ctx0.b = b0;
-	  ctx0.vm = vm;
-
-	  next0 = NAT64_OUT2IN_NEXT_LOOKUP;
-
-	  proto0 = ip_proto_to_snat_proto (ip40->protocol);
-
-	  if (proto0 == SNAT_PROTOCOL_ICMP)
-	    {
-	      if (icmp_to_icmp6
-		  (b0, nat64_out2in_icmp_set_cb, &ctx0,
-		   nat64_out2in_inner_icmp_set_cb, &ctx0))
-		{
-		  next0 = NAT64_OUT2IN_NEXT_DROP;
-		  b0->error = node->errors[NAT64_OUT2IN_ERROR_NO_TRANSLATION];
-		  goto trace0;
-		}
-	    }
-	  else if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
-	    {
-	      if (ip4_to_ip6_tcp_udp (b0, nat64_out2in_tcp_udp_set_cb, &ctx0))
-		{
-		  next0 = NAT64_OUT2IN_NEXT_DROP;
-		  b0->error = node->errors[NAT64_OUT2IN_ERROR_NO_TRANSLATION];
-		  goto trace0;
-		}
-	    }
-	  else
-	    {
-	      if (ip4_to_ip6 (b0, nat64_out2in_unk_proto_set_cb, &ctx0))
-		{
-		  next0 = NAT64_OUT2IN_NEXT_DROP;
-		  b0->error = node->errors[NAT64_OUT2IN_ERROR_NO_TRANSLATION];
-		  goto trace0;
-		}
-	    }
-
-	trace0:
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	    {
-	      nat64_out2in_trace_t *t =
-		vlib_add_trace (vm, node, b0, sizeof (*t));
-	      t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-	      t->next_index = next0;
-	    }
-
-	  pkts_processed += next0 != NAT64_OUT2IN_NEXT_DROP;
-
-	  /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-					   n_left_to_next, bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-  vlib_node_increment_counter (vm, nat64_out2in_node.index,
-			       NAT64_OUT2IN_ERROR_OUT2IN_PACKETS,
-			       pkts_processed);
-  return frame->n_vectors;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (nat64_out2in_node) = {
-  .function = nat64_out2in_node_fn,
-  .name = "nat64-out2in",
-  .vector_size = sizeof (u32),
-  .format_trace = format_nat64_out2in_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  .n_errors = ARRAY_LEN (nat64_out2in_error_strings),
-  .error_strings = nat64_out2in_error_strings,.n_next_nodes = 2,
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [NAT64_OUT2IN_NEXT_DROP] = "error-drop",
-    [NAT64_OUT2IN_NEXT_LOOKUP] = "ip6-lookup",
-  },
-};
-/* *INDENT-ON* */
-
-VLIB_NODE_FUNCTION_MULTIARCH (nat64_out2in_node, nat64_out2in_node_fn);
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c
deleted file mode 100644
index 329d67dc..00000000
--- a/src/plugins/snat/out2in.c
+++ /dev/null
@@ -1,2294 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-typedef struct {
-  u32 sw_if_index;
-  u32 next_index;
-  u32 session_index;
-} snat_out2in_trace_t;
-
-typedef struct {
-  u32 next_worker_index;
-  u8 do_handoff;
-} snat_out2in_worker_handoff_trace_t;
-
-/* packet trace format function */
-static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
-  
-  s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
-              t->sw_if_index, t->next_index, t->session_index);
-  return s;
-}
-
-static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
-  
-  s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
-              t->sw_if_index, t->next_index);
-  return s;
-}
-
-static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  snat_out2in_worker_handoff_trace_t * t =
-    va_arg (*args, snat_out2in_worker_handoff_trace_t *);
-  char * m;
-
-  m = t->do_handoff ? "next worker" : "same worker";
-  s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
-
-  return s;
-}
-
-vlib_node_registration_t snat_out2in_node;
-vlib_node_registration_t snat_out2in_fast_node;
-vlib_node_registration_t snat_out2in_worker_handoff_node;
-vlib_node_registration_t snat_det_out2in_node;
-
-#define foreach_snat_out2in_error                       \
-_(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
-_(OUT2IN_PACKETS, "Good out2in packets processed")      \
-_(BAD_ICMP_TYPE, "unsupported ICMP type")               \
-_(NO_TRANSLATION, "No translation")
-  
-typedef enum {
-#define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
-  foreach_snat_out2in_error
-#undef _
-  SNAT_OUT2IN_N_ERROR,
-} snat_out2in_error_t;
-
-static char * snat_out2in_error_strings[] = {
-#define _(sym,string) string,
-  foreach_snat_out2in_error
-#undef _
-};
-
-typedef enum {
-  SNAT_OUT2IN_NEXT_DROP,
-  SNAT_OUT2IN_NEXT_LOOKUP,
-  SNAT_OUT2IN_NEXT_ICMP_ERROR,
-  SNAT_OUT2IN_N_NEXT,
-} snat_out2in_next_t;
-
-/**
- * @brief Create session for static mapping.
- *
- * Create NAT session initiated by host from external network with static
- * mapping.
- *
- * @param sm     SNAT main.
- * @param b0     Vlib buffer.
- * @param in2out In2out SNAT session key.
- * @param out2in Out2in SNAT session key.
- * @param node   Vlib node.
- *
- * @returns SNAT session if successfully created otherwise 0.
- */
-static inline snat_session_t *
-create_session_for_static_mapping (snat_main_t *sm,
-                                   vlib_buffer_t *b0,
-                                   snat_session_key_t in2out,
-                                   snat_session_key_t out2in,
-                                   vlib_node_runtime_t * node,
-                                   u32 thread_index)
-{
-  snat_user_t *u;
-  snat_user_key_t user_key;
-  snat_session_t *s;
-  clib_bihash_kv_8_8_t kv0, value0;
-  dlist_elt_t * per_user_translation_list_elt;
-  dlist_elt_t * per_user_list_head_elt;
-  ip4_header_t *ip0;
-
-  ip0 = vlib_buffer_get_current (b0);
-
-  user_key.addr = in2out.addr;
-  user_key.fib_index = in2out.fib_index;
-  kv0.key = user_key.as_u64;
-
-  /* Ever heard of the "user" = inside ip4 address before? */
-  if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
-    {
-      /* no, make a new one */
-      pool_get (sm->per_thread_data[thread_index].users, u);
-      memset (u, 0, sizeof (*u));
-      u->addr = in2out.addr;
-      u->fib_index = in2out.fib_index;
-
-      pool_get (sm->per_thread_data[thread_index].list_pool,
-                per_user_list_head_elt);
-
-      u->sessions_per_user_list_head_index = per_user_list_head_elt -
-        sm->per_thread_data[thread_index].list_pool;
-
-      clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
-                       u->sessions_per_user_list_head_index);
-
-      kv0.value = u - sm->per_thread_data[thread_index].users;
-
-      /* add user */
-      clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
-
-      /* add non-traslated packets worker lookup */
-      kv0.value = thread_index;
-      clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
-    }
-  else
-    {
-      u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
-                             value0.value);
-    }
-
-  pool_get (sm->per_thread_data[thread_index].sessions, s);
-  memset (s, 0, sizeof (*s));
-
-  s->outside_address_index = ~0;
-  s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
-  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
-  u->nstaticsessions++;
-
-  /* Create list elts */
-  pool_get (sm->per_thread_data[thread_index].list_pool,
-            per_user_translation_list_elt);
-  clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
-                   per_user_translation_list_elt -
-                   sm->per_thread_data[thread_index].list_pool);
-
-  per_user_translation_list_elt->value =
-    s - sm->per_thread_data[thread_index].sessions;
-  s->per_user_index =
-    per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
-  s->per_user_list_head_index = u->sessions_per_user_list_head_index;
-
-  clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                      s->per_user_list_head_index,
-                      per_user_translation_list_elt -
-                      sm->per_thread_data[thread_index].list_pool);
-
-  s->in2out = in2out;
-  s->out2in = out2in;
-  s->in2out.protocol = out2in.protocol;
-
-  /* Add to translation hashes */
-  kv0.key = s->in2out.as_u64;
-  kv0.value = s - sm->per_thread_data[thread_index].sessions;
-  if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
-      clib_warning ("in2out key add failed");
-
-  kv0.key = s->out2in.as_u64;
-  kv0.value = s - sm->per_thread_data[thread_index].sessions;
-
-  if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
-      clib_warning ("out2in key add failed");
-
-  /* log NAT event */
-  snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
-                                      s->out2in.addr.as_u32,
-                                      s->in2out.protocol,
-                                      s->in2out.port,
-                                      s->out2in.port,
-                                      s->in2out.fib_index);
-   return s;
-}
-
-static_always_inline
-snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
-                                 snat_session_key_t *p_key0)
-{
-  icmp46_header_t *icmp0;
-  snat_session_key_t key0;
-  icmp_echo_header_t *echo0, *inner_echo0 = 0;
-  ip4_header_t *inner_ip0;
-  void *l4_header = 0;
-  icmp46_header_t *inner_icmp0;
-
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  echo0 = (icmp_echo_header_t *)(icmp0+1);
-
-  if (!icmp_is_error_message (icmp0))
-    {
-      key0.protocol = SNAT_PROTOCOL_ICMP;
-      key0.addr = ip0->dst_address;
-      key0.port = echo0->identifier;
-    }
-  else
-    {
-      inner_ip0 = (ip4_header_t *)(echo0+1);
-      l4_header = ip4_next_header (inner_ip0);
-      key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
-      key0.addr = inner_ip0->src_address;
-      switch (key0.protocol)
-        {
-        case SNAT_PROTOCOL_ICMP:
-          inner_icmp0 = (icmp46_header_t*)l4_header;
-          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
-          key0.port = inner_echo0->identifier;
-          break;
-        case SNAT_PROTOCOL_UDP:
-        case SNAT_PROTOCOL_TCP:
-          key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
-          break;
-        default:
-          return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
-        }
-    }
-  *p_key0 = key0;
-  return -1; /* success */
-}
-
-/**
- * Get address and port values to be used for packet SNAT translation
- * and create session if needed
- *
- * @param[in,out] sm             SNAT main
- * @param[in,out] node           SNAT node runtime
- * @param[in] thread_index       thread index
- * @param[in,out] b0             buffer containing packet to be translated
- * @param[out] p_proto           protocol used for matching
- * @param[out] p_value           address and port after NAT translation
- * @param[out] p_dont_translate  if packet should not be translated
- * @param d                      optional parameter
- * @param e                      optional parameter
- */
-u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e)
-{
-  ip4_header_t *ip0;
-  icmp46_header_t *icmp0;
-  u32 sw_if_index0;
-  u32 rx_fib_index0;
-  snat_session_key_t key0;
-  snat_session_key_t sm0;
-  snat_session_t *s0 = 0;
-  u8 dont_translate = 0;
-  clib_bihash_kv_8_8_t kv0, value0;
-  u8 is_addr_only;
-  u32 next0 = ~0;
-  int err;
-
-  ip0 = vlib_buffer_get_current (b0);
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
-
-  key0.protocol = 0;
-
-  err = icmp_get_key (ip0, &key0);
-  if (err != -1)
-    {
-      b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out;
-    }
-  key0.fib_index = rx_fib_index0;
-
-  kv0.key = key0.as_u64;
-
-  if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
-    {
-      /* Try to match static mapping by external address and port,
-         destination address and port in packet */
-      if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
-        {
-          /* Don't NAT packet aimed at the intfc address */
-          if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
-                                              ip0->dst_address.as_u32)))
-            {
-              dont_translate = 1;
-              goto out;
-            }
-          b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-          next0 = SNAT_OUT2IN_NEXT_DROP;
-          goto out;
-        }
-
-      if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
-                        (icmp0->type != ICMP4_echo_request || !is_addr_only)))
-        {
-          b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
-          next0 = SNAT_OUT2IN_NEXT_DROP;
-          goto out;
-        }
-
-      /* Create session initiated by host from external network */
-      s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
-                                             node, thread_index);
-
-      if (!s0)
-        {
-          next0 = SNAT_OUT2IN_NEXT_DROP;
-          goto out;
-        }
-    }
-  else
-    {
-      if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
-                        icmp0->type != ICMP4_echo_request &&
-                        !icmp_is_error_message (icmp0)))
-        {
-          b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
-          next0 = SNAT_OUT2IN_NEXT_DROP;
-          goto out;
-        }
-
-      s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                              value0.value);
-    }
-
-out:
-  *p_proto = key0.protocol;
-  if (s0)
-    *p_value = s0->in2out;
-  *p_dont_translate = dont_translate;
-  if (d)
-    *(snat_session_t**)d = s0;
-  return next0;
-}
-
-/**
- * Get address and port values to be used for packet SNAT translation
- *
- * @param[in] sm                 SNAT main
- * @param[in,out] node           SNAT node runtime
- * @param[in] thread_index       thread index
- * @param[in,out] b0             buffer containing packet to be translated
- * @param[out] p_proto           protocol used for matching
- * @param[out] p_value           address and port after NAT translation
- * @param[out] p_dont_translate  if packet should not be translated
- * @param d                      optional parameter
- * @param e                      optional parameter
- */
-u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e)
-{
-  ip4_header_t *ip0;
-  icmp46_header_t *icmp0;
-  u32 sw_if_index0;
-  u32 rx_fib_index0;
-  snat_session_key_t key0;
-  snat_session_key_t sm0;
-  u8 dont_translate = 0;
-  u8 is_addr_only;
-  u32 next0 = ~0;
-  int err;
-
-  ip0 = vlib_buffer_get_current (b0);
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
-
-  err = icmp_get_key (ip0, &key0);
-  if (err != -1)
-    {
-      b0->error = node->errors[err];
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out2;
-    }
-  key0.fib_index = rx_fib_index0;
-
-  if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
-    {
-      /* Don't NAT packet aimed at the intfc address */
-      if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out;
-    }
-
-  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
-                    (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
-                    !icmp_is_error_message (icmp0)))
-    {
-      b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out;
-    }
-
-out:
-  *p_value = sm0;
-out2:
-  *p_proto = key0.protocol;
-  *p_dont_translate = dont_translate;
-  return next0;
-}
-
-static inline u32 icmp_out2in (snat_main_t *sm,
-                               vlib_buffer_t * b0,
-                               ip4_header_t * ip0,
-                               icmp46_header_t * icmp0,
-                               u32 sw_if_index0,
-                               u32 rx_fib_index0,
-                               vlib_node_runtime_t * node,
-                               u32 next0,
-                               u32 thread_index,
-                               void *d,
-                               void *e)
-{
-  snat_session_key_t sm0;
-  u8 protocol;
-  icmp_echo_header_t *echo0, *inner_echo0 = 0;
-  ip4_header_t *inner_ip0 = 0;
-  void *l4_header = 0;
-  icmp46_header_t *inner_icmp0;
-  u8 dont_translate;
-  u32 new_addr0, old_addr0;
-  u16 old_id0, new_id0;
-  ip_csum_t sum0;
-  u16 checksum0;
-  u32 next0_tmp;
-
-  echo0 = (icmp_echo_header_t *)(icmp0+1);
-
-  next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
-                                       &protocol, &sm0, &dont_translate, d, e);
-  if (next0_tmp != ~0)
-    next0 = next0_tmp;
-  if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
-    goto out;
-
-  sum0 = ip_incremental_checksum (0, icmp0,
-                                  ntohs(ip0->length) - ip4_header_bytes (ip0));
-  checksum0 = ~ip_csum_fold (sum0);
-  if (checksum0 != 0 && checksum0 != 0xffff)
-    {
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out;
-    }
-
-  old_addr0 = ip0->dst_address.as_u32;
-  new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
-  vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
-
-  sum0 = ip0->checksum;
-  sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
-                         dst_address /* changed member */);
-  ip0->checksum = ip_csum_fold (sum0);
-
-  if (!icmp_is_error_message (icmp0))
-    {
-      new_id0 = sm0.port;
-      if (PREDICT_FALSE(new_id0 != echo0->identifier))
-        {
-          old_id0 = echo0->identifier;
-          new_id0 = sm0.port;
-          echo0->identifier = new_id0;
-
-          sum0 = icmp0->checksum;
-          sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
-                                 identifier /* changed member */);
-          icmp0->checksum = ip_csum_fold (sum0);
-        }
-    }
-  else
-    {
-      inner_ip0 = (ip4_header_t *)(echo0+1);
-      l4_header = ip4_next_header (inner_ip0);
-
-      if (!ip4_header_checksum_is_valid (inner_ip0))
-        {
-          next0 = SNAT_OUT2IN_NEXT_DROP;
-          goto out;
-        }
-
-      old_addr0 = inner_ip0->src_address.as_u32;
-      inner_ip0->src_address = sm0.addr;
-      new_addr0 = inner_ip0->src_address.as_u32;
-
-      sum0 = icmp0->checksum;
-      sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
-                             src_address /* changed member */);
-      icmp0->checksum = ip_csum_fold (sum0);
-
-      switch (protocol)
-        {
-        case SNAT_PROTOCOL_ICMP:
-          inner_icmp0 = (icmp46_header_t*)l4_header;
-          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
-
-          old_id0 = inner_echo0->identifier;
-          new_id0 = sm0.port;
-          inner_echo0->identifier = new_id0;
-
-          sum0 = icmp0->checksum;
-          sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
-                                 identifier);
-          icmp0->checksum = ip_csum_fold (sum0);
-          break;
-        case SNAT_PROTOCOL_UDP:
-        case SNAT_PROTOCOL_TCP:
-          old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
-          new_id0 = sm0.port;
-          ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
-
-          sum0 = icmp0->checksum;
-          sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
-                                 src_port);
-          icmp0->checksum = ip_csum_fold (sum0);
-          break;
-        default:
-          ASSERT(0);
-        }
-    }
-
-out:
-  return next0;
-}
-
-
-static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
-                                         vlib_buffer_t * b0,
-                                         ip4_header_t * ip0,
-                                         icmp46_header_t * icmp0,
-                                         u32 sw_if_index0,
-                                         u32 rx_fib_index0,
-                                         vlib_node_runtime_t * node,
-                                         u32 next0, f64 now,
-                                         u32 thread_index,
-                                         snat_session_t ** p_s0)
-{
-  next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
-                      next0, thread_index, p_s0, 0);
-  snat_session_t * s0 = *p_s0;
-  if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
-    {
-      /* Accounting */
-      s0->last_heard = now;
-      s0->total_pkts++;
-      s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
-      /* Per-user LRU list maintenance for dynamic translation */
-      if (!snat_is_session_static (s0))
-        {
-          clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                             s0->per_user_index);
-          clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                              s0->per_user_list_head_index,
-                              s0->per_user_index);
-        }
-    }
-  return next0;
-}
-
-static void
-snat_out2in_unknown_proto (snat_main_t *sm,
-                           vlib_buffer_t * b,
-                           ip4_header_t * ip,
-                           u32 rx_fib_index,
-                           u32 thread_index,
-                           f64 now,
-                           vlib_main_t * vm)
-{
-  clib_bihash_kv_8_8_t kv, value;
-  clib_bihash_kv_16_8_t s_kv, s_value;
-  snat_static_mapping_t *m;
-  snat_session_key_t m_key;
-  u32 old_addr, new_addr;
-  ip_csum_t sum;
-  snat_unk_proto_ses_key_t key;
-  snat_session_t * s;
-  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
-  snat_user_key_t u_key;
-  snat_user_t *u;
-  dlist_elt_t *head, *elt;
-
-  old_addr = ip->dst_address.as_u32;
-
-  key.l_addr = ip->dst_address;
-  key.r_addr = ip->src_address;
-  key.fib_index = rx_fib_index;
-  key.proto = ip->protocol;
-  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
-  s_kv.key[0] = key.as_u64[0];
-  s_kv.key[1] = key.as_u64[1];
-
-  if (!clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
-    {
-      s = pool_elt_at_index (tsm->sessions, s_value.value);
-      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
-    }
-  else
-    {
-      m_key.addr = ip->dst_address;
-      m_key.port = 0;
-      m_key.protocol = 0;
-      m_key.fib_index = rx_fib_index;
-      kv.key = m_key.as_u64;
-      if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
-        return;
-
-      m = pool_elt_at_index (sm->static_mappings, value.value);
-
-      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
-
-      u_key.addr = ip->src_address;
-      u_key.fib_index = m->fib_index;
-      kv.key = u_key.as_u64;
-
-      /* Ever heard of the "user" = src ip4 address before? */
-      if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
-        {
-          /* no, make a new one */
-          pool_get (tsm->users, u);
-          memset (u, 0, sizeof (*u));
-          u->addr = ip->src_address;
-          u->fib_index = rx_fib_index;
-
-          pool_get (tsm->list_pool, head);
-          u->sessions_per_user_list_head_index = head - tsm->list_pool;
-
-          clib_dlist_init (tsm->list_pool,
-                           u->sessions_per_user_list_head_index);
-
-          kv.value = u - tsm->users;
-
-          /* add user */
-          clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
-        }
-      else
-        {
-          u = pool_elt_at_index (tsm->users, value.value);
-        }
-
-      /* Create a new session */
-      pool_get (tsm->sessions, s);
-      memset (s, 0, sizeof (*s));
-
-      s->ext_host_addr.as_u32 = ip->src_address.as_u32;
-      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
-      s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
-      s->outside_address_index = ~0;
-      s->out2in.addr.as_u32 = old_addr;
-      s->out2in.fib_index = rx_fib_index;
-      s->in2out.addr.as_u32 = new_addr;
-      s->in2out.fib_index = m->fib_index;
-      s->in2out.port = s->out2in.port = ip->protocol;
-      u->nstaticsessions++;
-
-      /* Create list elts */
-      pool_get (tsm->list_pool, elt);
-      clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
-      elt->value = s - tsm->sessions;
-      s->per_user_index = elt - tsm->list_pool;
-      s->per_user_list_head_index = u->sessions_per_user_list_head_index;
-      clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
-                          s->per_user_index);
-
-      /* Add to lookup tables */
-      s_kv.value = s - tsm->sessions;
-      if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
-        clib_warning ("out2in key add failed");
-
-      key.l_addr = ip->dst_address;
-      key.fib_index = m->fib_index;
-      s_kv.key[0] = key.as_u64[0];
-      s_kv.key[1] = key.as_u64[1];
-      if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
-        clib_warning ("in2out key add failed");
-   }
-
-  /* Update IP checksum */
-  sum = ip->checksum;
-  sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
-  ip->checksum = ip_csum_fold (sum);
-
-  vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
-
-  /* Accounting */
-  s->last_heard = now;
-  s->total_pkts++;
-  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
-  /* Per-user LRU list maintenance */
-  clib_dlist_remove (tsm->list_pool, s->per_user_index);
-  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
-                      s->per_user_index);
-}
-
-static uword
-snat_out2in_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_out2in_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-  f64 now = vlib_time_now (vm);
-  u32 thread_index = vlib_get_thread_index ();
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
-          u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
-          u32 sw_if_index0, sw_if_index1;
-          ip4_header_t * ip0, *ip1;
-          ip_csum_t sum0, sum1;
-          u32 new_addr0, old_addr0;
-          u16 new_port0, old_port0;
-          u32 new_addr1, old_addr1;
-          u16 new_port1, old_port1;
-          udp_header_t * udp0, * udp1;
-          tcp_header_t * tcp0, * tcp1;
-          icmp46_header_t * icmp0, * icmp1;
-          snat_session_key_t key0, key1, sm0, sm1;
-          u32 rx_fib_index0, rx_fib_index1;
-          u32 proto0, proto1;
-          snat_session_t * s0 = 0, * s1 = 0;
-          clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
-          
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-            
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-            
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          vnet_buffer (b0)->snat.flags = 0;
-          vnet_buffer (b1)->snat.flags = 0;
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-          icmp0 = (icmp46_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, 
-                                   sw_if_index0);
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace0;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE (proto0 == ~0))
-            {
-              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
-                                        thread_index, now, vm);
-              goto trace0;
-            }
-
-          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              next0 = icmp_out2in_slow_path 
-                (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, 
-                 next0, now, thread_index, &s0);
-              goto trace0;
-            }
-
-          key0.addr = ip0->dst_address;
-          key0.port = udp0->dst_port;
-          key0.protocol = proto0;
-          key0.fib_index = rx_fib_index0;
-          
-          kv0.key = key0.as_u64;
-
-          if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
-            {
-              /* Try to match static mapping by external address and port,
-                 destination address and port in packet */
-              if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
-                {
-                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-                  /* 
-                   * Send DHCP packets to the ipv4 stack, or we won't
-                   * be able to use dhcp client on the outside interface
-                   */
-                  if (proto0 != SNAT_PROTOCOL_UDP 
-                      || (udp0->dst_port 
-                          != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
-                    next0 = SNAT_OUT2IN_NEXT_DROP;
-                  goto trace0;
-                }
-
-              /* Create session initiated by host from external network */
-              s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
-                                                     thread_index);
-              if (!s0)
-                {
-                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-                  next0 = SNAT_OUT2IN_NEXT_DROP;
-                  goto trace0;
-                }
-            }
-          else
-            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                    value0.value);
-
-          old_addr0 = ip0->dst_address.as_u32;
-          ip0->dst_address = s0->in2out.addr;
-          new_addr0 = ip0->dst_address.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              old_port0 = tcp0->dst_port;
-              tcp0->dst_port = s0->in2out.port;
-              new_port0 = tcp0->dst_port;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              old_port0 = udp0->dst_port;
-              udp0->dst_port = s0->in2out.port;
-              udp0->checksum = 0;
-            }
-
-          /* Accounting */
-          s0->last_heard = now;
-          s0->total_pkts++;
-          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
-          /* Per-user LRU list maintenance for dynamic translation */
-          if (!snat_is_session_static (s0))
-            {
-              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                                 s0->per_user_index);
-              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                                  s0->per_user_list_head_index,
-                                  s0->per_user_index);
-            }
-        trace0:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              snat_out2in_trace_t *t = 
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-              t->session_index = ~0;
-              if (s0)
-                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
-            }
-
-          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
-
-
-          ip1 = vlib_buffer_get_current (b1);
-          udp1 = ip4_next_header (ip1);
-          tcp1 = (tcp_header_t *) udp1;
-          icmp1 = (icmp46_header_t *) udp1;
-
-          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-	  rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, 
-                                   sw_if_index1);
-
-          if (PREDICT_FALSE(ip1->ttl == 1))
-            {
-              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace1;
-            }
-
-          proto1 = ip_proto_to_snat_proto (ip1->protocol);
-
-          if (PREDICT_FALSE (proto1 == ~0))
-            {
-              snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
-                                        thread_index, now, vm);
-              goto trace1;
-            }
-
-          if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
-            {
-              next1 = icmp_out2in_slow_path 
-                (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, 
-                 next1, now, thread_index, &s1);
-              goto trace1;
-            }
-
-          key1.addr = ip1->dst_address;
-          key1.port = udp1->dst_port;
-          key1.protocol = proto1;
-          key1.fib_index = rx_fib_index1;
-          
-          kv1.key = key1.as_u64;
-
-          if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
-            {
-              /* Try to match static mapping by external address and port,
-                 destination address and port in packet */
-              if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
-                {
-                  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-                  /* 
-                   * Send DHCP packets to the ipv4 stack, or we won't
-                   * be able to use dhcp client on the outside interface
-                   */
-                  if (proto1 != SNAT_PROTOCOL_UDP 
-                      || (udp1->dst_port 
-                          != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
-                    next1 = SNAT_OUT2IN_NEXT_DROP;
-                  goto trace1;
-                }
-
-              /* Create session initiated by host from external network */
-              s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
-                                                     thread_index);
-              if (!s1)
-                {
-                  b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-                  next1 = SNAT_OUT2IN_NEXT_DROP;
-                  goto trace1;
-                }
-            }
-          else
-            s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                    value1.value);
-
-          old_addr1 = ip1->dst_address.as_u32;
-          ip1->dst_address = s1->in2out.addr;
-          new_addr1 = ip1->dst_address.as_u32;
-          vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
-
-          sum1 = ip1->checksum;
-          sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip1->checksum = ip_csum_fold (sum1);
-
-          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
-            {
-              old_port1 = tcp1->dst_port;
-              tcp1->dst_port = s1->in2out.port;
-              new_port1 = tcp1->dst_port;
-
-              sum1 = tcp1->checksum;
-              sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-
-              sum1 = ip_csum_update (sum1, old_port1, new_port1,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp1->checksum = ip_csum_fold(sum1);
-            }
-          else
-            {
-              old_port1 = udp1->dst_port;
-              udp1->dst_port = s1->in2out.port;
-              udp1->checksum = 0;
-            }
-
-          /* Accounting */
-          s1->last_heard = now;
-          s1->total_pkts++;
-          s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
-          /* Per-user LRU list maintenance for dynamic translation */
-          if (!snat_is_session_static (s1))
-            {
-              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                                 s1->per_user_index);
-              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                                  s1->per_user_list_head_index,
-                                  s1->per_user_index);
-            }
-        trace1:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b1->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              snat_out2in_trace_t *t = 
-                 vlib_add_trace (vm, node, b1, sizeof (*t));
-              t->sw_if_index = sw_if_index1;
-              t->next_index = next1;
-              t->session_index = ~0;
-              if (s1)
-                t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
-            }
-
-          pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
-
-          /* verify speculative enqueues, maybe switch current next frame */
-          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                           to_next, n_left_to_next,
-                                           bi0, bi1, next0, next1);
-        }
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
-          u32 sw_if_index0;
-          ip4_header_t * ip0;
-          ip_csum_t sum0;
-          u32 new_addr0, old_addr0;
-          u16 new_port0, old_port0;
-          udp_header_t * udp0;
-          tcp_header_t * tcp0;
-          icmp46_header_t * icmp0;
-          snat_session_key_t key0, sm0;
-          u32 rx_fib_index0;
-          u32 proto0;
-          snat_session_t * s0 = 0;
-          clib_bihash_kv_8_8_t kv0, value0;
-          
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-          vnet_buffer (b0)->snat.flags = 0;
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-          icmp0 = (icmp46_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-	  rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index, 
-                                   sw_if_index0);
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE (proto0 == ~0))
-            {
-              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
-                                        thread_index, now, vm);
-              goto trace00;
-            }
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace00;
-            }
-
-          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              next0 = icmp_out2in_slow_path 
-                (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, 
-                 next0, now, thread_index, &s0);
-              goto trace00;
-            }
-
-          key0.addr = ip0->dst_address;
-          key0.port = udp0->dst_port;
-          key0.protocol = proto0;
-          key0.fib_index = rx_fib_index0;
-          
-          kv0.key = key0.as_u64;
-
-          if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
-            {
-              /* Try to match static mapping by external address and port,
-                 destination address and port in packet */
-              if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
-                {
-                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-                  /* 
-                   * Send DHCP packets to the ipv4 stack, or we won't
-                   * be able to use dhcp client on the outside interface
-                   */
-                  if (proto0 != SNAT_PROTOCOL_UDP 
-                      || (udp0->dst_port 
-                          != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
-
-                    next0 = SNAT_OUT2IN_NEXT_DROP;
-                  goto trace00;
-                }
-
-              /* Create session initiated by host from external network */
-              s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
-                                                     thread_index);
-              if (!s0)
-                {
-                  b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-                    next0 = SNAT_OUT2IN_NEXT_DROP;
-                  goto trace00;
-                }
-            }
-          else
-            s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
-                                    value0.value);
-
-          old_addr0 = ip0->dst_address.as_u32;
-          ip0->dst_address = s0->in2out.addr;
-          new_addr0 = ip0->dst_address.as_u32;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              old_port0 = tcp0->dst_port;
-              tcp0->dst_port = s0->in2out.port;
-              new_port0 = tcp0->dst_port;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              old_port0 = udp0->dst_port;
-              udp0->dst_port = s0->in2out.port;
-              udp0->checksum = 0;
-            }
-
-          /* Accounting */
-          s0->last_heard = now;
-          s0->total_pkts++;
-          s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
-          /* Per-user LRU list maintenance for dynamic translation */
-          if (!snat_is_session_static (s0))
-            {
-              clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
-                                 s0->per_user_index);
-              clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
-                                  s0->per_user_list_head_index,
-                                  s0->per_user_index);
-            }
-        trace00:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) 
-            {
-              snat_out2in_trace_t *t = 
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-              t->session_index = ~0;
-              if (s0)
-                t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
-            }
-
-          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, snat_out2in_node.index, 
-                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS, 
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_out2in_node) = {
-  .function = snat_out2in_node_fn,
-  .name = "snat-out2in",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_out2in_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
-  .error_strings = snat_out2in_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-  
-  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
-    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
-
-/**************************/
-/*** deterministic mode ***/
-/**************************/
-static uword
-snat_det_out2in_node_fn (vlib_main_t * vm,
-                         vlib_node_runtime_t * node,
-                         vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_out2in_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-  u32 thread_index = vlib_get_thread_index ();
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-        {
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
-          u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
-          u32 sw_if_index0, sw_if_index1;
-          ip4_header_t * ip0, * ip1;
-          ip_csum_t sum0, sum1;
-          ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
-          u16 new_port0, old_port0, old_port1, new_port1;
-          udp_header_t * udp0, * udp1;
-          tcp_header_t * tcp0, * tcp1;
-          u32 proto0, proto1;
-          snat_det_out_key_t key0, key1;
-          snat_det_map_t * dm0, * dm1;
-          snat_det_session_t * ses0 = 0, * ses1 = 0;
-          u32 rx_fib_index0, rx_fib_index1;
-          icmp46_header_t * icmp0, * icmp1;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace0;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-              icmp0 = (icmp46_header_t *) udp0;
-
-              next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
-                                  rx_fib_index0, node, next0, thread_index,
-                                  &ses0, &dm0);
-              goto trace0;
-            }
-
-          key0.ext_host_addr = ip0->src_address;
-          key0.ext_host_port = tcp0->src;
-          key0.out_port = tcp0->dst;
-
-          dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
-          if (PREDICT_FALSE(!dm0))
-            {
-              clib_warning("unknown dst address:  %U",
-                           format_ip4_address, &ip0->dst_address);
-              next0 = SNAT_OUT2IN_NEXT_DROP;
-              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace0;
-            }
-
-          snat_det_reverse(dm0, &ip0->dst_address,
-                           clib_net_to_host_u16(tcp0->dst), &new_addr0);
-
-          ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
-          if (PREDICT_FALSE(!ses0))
-            {
-              clib_warning("no match src %U:%d dst %U:%d for user %U",
-                           format_ip4_address, &ip0->src_address,
-                           clib_net_to_host_u16 (tcp0->src),
-                           format_ip4_address, &ip0->dst_address,
-                           clib_net_to_host_u16 (tcp0->dst),
-                           format_ip4_address, &new_addr0);
-              next0 = SNAT_OUT2IN_NEXT_DROP;
-              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace0;
-            }
-          new_port0 = ses0->in_port;
-
-          old_addr0 = ip0->dst_address;
-          ip0->dst_address = new_addr0;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
-                ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
-              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
-                snat_det_ses_close(dm0, ses0);
-
-              old_port0 = tcp0->dst;
-              tcp0->dst = new_port0;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              old_port0 = udp0->dst_port;
-              udp0->dst_port = new_port0;
-              udp0->checksum = 0;
-            }
-
-        trace0:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_out2in_trace_t *t =
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-              t->session_index = ~0;
-              if (ses0)
-                t->session_index = ses0 - dm0->sessions;
-            }
-
-          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
-
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          ip1 = vlib_buffer_get_current (b1);
-          udp1 = ip4_next_header (ip1);
-          tcp1 = (tcp_header_t *) udp1;
-
-          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-
-          if (PREDICT_FALSE(ip1->ttl == 1))
-            {
-              vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace1;
-            }
-
-          proto1 = ip_proto_to_snat_proto (ip1->protocol);
-
-          if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
-            {
-              rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
-              icmp1 = (icmp46_header_t *) udp1;
-
-              next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
-                                  rx_fib_index1, node, next1, thread_index,
-                                  &ses1, &dm1);
-              goto trace1;
-            }
-
-          key1.ext_host_addr = ip1->src_address;
-          key1.ext_host_port = tcp1->src;
-          key1.out_port = tcp1->dst;
-
-          dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
-          if (PREDICT_FALSE(!dm1))
-            {
-              clib_warning("unknown dst address:  %U",
-                           format_ip4_address, &ip1->dst_address);
-              next1 = SNAT_OUT2IN_NEXT_DROP;
-              b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace1;
-            }
-
-          snat_det_reverse(dm1, &ip1->dst_address,
-                           clib_net_to_host_u16(tcp1->dst), &new_addr1);
-
-          ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
-          if (PREDICT_FALSE(!ses1))
-            {
-              clib_warning("no match src %U:%d dst %U:%d for user %U",
-                           format_ip4_address, &ip1->src_address,
-                           clib_net_to_host_u16 (tcp1->src),
-                           format_ip4_address, &ip1->dst_address,
-                           clib_net_to_host_u16 (tcp1->dst),
-                           format_ip4_address, &new_addr1);
-              next1 = SNAT_OUT2IN_NEXT_DROP;
-              b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace1;
-            }
-          new_port1 = ses1->in_port;
-
-          old_addr1 = ip1->dst_address;
-          ip1->dst_address = new_addr1;
-          vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
-
-          sum1 = ip1->checksum;
-          sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip1->checksum = ip_csum_fold (sum1);
-
-          if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
-            {
-              if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
-                ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
-              else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
-                snat_det_ses_close(dm1, ses1);
-
-              old_port1 = tcp1->dst;
-              tcp1->dst = new_port1;
-
-              sum1 = tcp1->checksum;
-              sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-
-              sum1 = ip_csum_update (sum1, old_port1, new_port1,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp1->checksum = ip_csum_fold(sum1);
-            }
-          else
-            {
-              old_port1 = udp1->dst_port;
-              udp1->dst_port = new_port1;
-              udp1->checksum = 0;
-            }
-
-        trace1:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b1->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_out2in_trace_t *t =
-                 vlib_add_trace (vm, node, b1, sizeof (*t));
-              t->sw_if_index = sw_if_index1;
-              t->next_index = next1;
-              t->session_index = ~0;
-              if (ses1)
-                t->session_index = ses1 - dm1->sessions;
-            }
-
-          pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
-
-          /* verify speculative enqueues, maybe switch current next frame */
-          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                           to_next, n_left_to_next,
-                                           bi0, bi1, next0, next1);
-         }
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
-          u32 sw_if_index0;
-          ip4_header_t * ip0;
-          ip_csum_t sum0;
-          ip4_address_t new_addr0, old_addr0;
-          u16 new_port0, old_port0;
-          udp_header_t * udp0;
-          tcp_header_t * tcp0;
-          u32 proto0;
-          snat_det_out_key_t key0;
-          snat_det_map_t * dm0;
-          snat_det_session_t * ses0 = 0;
-          u32 rx_fib_index0;
-          icmp46_header_t * icmp0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace00;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-              icmp0 = (icmp46_header_t *) udp0;
-
-              next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
-                                  rx_fib_index0, node, next0, thread_index,
-                                  &ses0, &dm0);
-              goto trace00;
-            }
-
-          key0.ext_host_addr = ip0->src_address;
-          key0.ext_host_port = tcp0->src;
-          key0.out_port = tcp0->dst;
-
-          dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
-          if (PREDICT_FALSE(!dm0))
-            {
-              clib_warning("unknown dst address:  %U",
-                           format_ip4_address, &ip0->dst_address);
-              next0 = SNAT_OUT2IN_NEXT_DROP;
-              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace00;
-            }
-
-          snat_det_reverse(dm0, &ip0->dst_address,
-                           clib_net_to_host_u16(tcp0->dst), &new_addr0);
-
-          ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
-          if (PREDICT_FALSE(!ses0))
-            {
-              clib_warning("no match src %U:%d dst %U:%d for user %U",
-                           format_ip4_address, &ip0->src_address,
-                           clib_net_to_host_u16 (tcp0->src),
-                           format_ip4_address, &ip0->dst_address,
-                           clib_net_to_host_u16 (tcp0->dst),
-                           format_ip4_address, &new_addr0);
-              next0 = SNAT_OUT2IN_NEXT_DROP;
-              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace00;
-            }
-          new_port0 = ses0->in_port;
-
-          old_addr0 = ip0->dst_address;
-          ip0->dst_address = new_addr0;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-            {
-              if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
-                ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
-              else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
-                snat_det_ses_close(dm0, ses0);
-
-              old_port0 = tcp0->dst;
-              tcp0->dst = new_port0;
-
-              sum0 = tcp0->checksum;
-              sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
-                                     ip4_header_t,
-                                     dst_address /* changed member */);
-
-              sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                     ip4_header_t /* cheat */,
-                                     length /* changed member */);
-              tcp0->checksum = ip_csum_fold(sum0);
-            }
-          else
-            {
-              old_port0 = udp0->dst_port;
-              udp0->dst_port = new_port0;
-              udp0->checksum = 0;
-            }
-
-        trace00:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_out2in_trace_t *t =
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-              t->session_index = ~0;
-              if (ses0)
-                t->session_index = ses0 - dm0->sessions;
-            }
-
-          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, snat_det_out2in_node.index,
-                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_det_out2in_node) = {
-  .function = snat_det_out2in_node_fn,
-  .name = "snat-det-out2in",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_out2in_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-
-  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
-  .error_strings = snat_out2in_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-
-  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
-    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
-
-/**
- * Get address and port values to be used for packet SNAT translation
- * and create session if needed
- *
- * @param[in,out] sm             SNAT main
- * @param[in,out] node           SNAT node runtime
- * @param[in] thread_index       thread index
- * @param[in,out] b0             buffer containing packet to be translated
- * @param[out] p_proto           protocol used for matching
- * @param[out] p_value           address and port after NAT translation
- * @param[out] p_dont_translate  if packet should not be translated
- * @param d                      optional parameter
- * @param e                      optional parameter
- */
-u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
-                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                          snat_session_key_t *p_value,
-                          u8 *p_dont_translate, void *d, void *e)
-{
-  ip4_header_t *ip0;
-  icmp46_header_t *icmp0;
-  u32 sw_if_index0;
-  u8 protocol;
-  snat_det_out_key_t key0;
-  u8 dont_translate = 0;
-  u32 next0 = ~0;
-  icmp_echo_header_t *echo0, *inner_echo0 = 0;
-  ip4_header_t *inner_ip0;
-  void *l4_header = 0;
-  icmp46_header_t *inner_icmp0;
-  snat_det_map_t * dm0 = 0;
-  ip4_address_t new_addr0 = {{0}};
-  snat_det_session_t * ses0 = 0;
-  ip4_address_t out_addr;
-
-  ip0 = vlib_buffer_get_current (b0);
-  icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
-  echo0 = (icmp_echo_header_t *)(icmp0+1);
-  sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-
-  if (!icmp_is_error_message (icmp0))
-    {
-      protocol = SNAT_PROTOCOL_ICMP;
-      key0.ext_host_addr = ip0->src_address;
-      key0.ext_host_port = 0;
-      key0.out_port = echo0->identifier;
-      out_addr = ip0->dst_address;
-    }
-  else
-    {
-      inner_ip0 = (ip4_header_t *)(echo0+1);
-      l4_header = ip4_next_header (inner_ip0);
-      protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
-      key0.ext_host_addr = inner_ip0->dst_address;
-      out_addr = inner_ip0->src_address;
-      switch (protocol)
-        {
-        case SNAT_PROTOCOL_ICMP:
-          inner_icmp0 = (icmp46_header_t*)l4_header;
-          inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
-          key0.ext_host_port = 0;
-          key0.out_port = inner_echo0->identifier;
-          break;
-        case SNAT_PROTOCOL_UDP:
-        case SNAT_PROTOCOL_TCP:
-          key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
-          key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
-          break;
-        default:
-          b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
-          next0 = SNAT_OUT2IN_NEXT_DROP;
-          goto out;
-        }
-    }
-
-  dm0 = snat_det_map_by_out(sm, &out_addr);
-  if (PREDICT_FALSE(!dm0))
-    {
-      /* Don't NAT packet aimed at the intfc address */
-      if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
-                                          ip0->dst_address.as_u32)))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-      clib_warning("unknown dst address:  %U",
-                   format_ip4_address, &ip0->dst_address);
-      goto out;
-    }
-
-  snat_det_reverse(dm0, &ip0->dst_address,
-                   clib_net_to_host_u16(key0.out_port), &new_addr0);
-
-  ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
-  if (PREDICT_FALSE(!ses0))
-    {
-      /* Don't NAT packet aimed at the intfc address */
-      if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
-                                          ip0->dst_address.as_u32)))
-        {
-          dont_translate = 1;
-          goto out;
-        }
-      clib_warning("no match src %U:%d dst %U:%d for user %U",
-                   format_ip4_address, &key0.ext_host_addr,
-                   clib_net_to_host_u16 (key0.ext_host_port),
-                   format_ip4_address, &out_addr,
-                   clib_net_to_host_u16 (key0.out_port),
-                   format_ip4_address, &new_addr0);
-      b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out;
-    }
-
-  if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
-                    !icmp_is_error_message (icmp0)))
-    {
-      b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
-      next0 = SNAT_OUT2IN_NEXT_DROP;
-      goto out;
-    }
-
-  goto out;
-
-out:
-  *p_proto = protocol;
-  if (ses0)
-    {
-      p_value->addr = new_addr0;
-      p_value->fib_index = sm->inside_fib_index;
-      p_value->port = ses0->in_port;
-    }
-  *p_dont_translate = dont_translate;
-  if (d)
-    *(snat_det_session_t**)d = ses0;
-  if (e)
-    *(snat_det_map_t**)e = dm0;
-  return next0;
-}
-
-/**********************/
-/*** worker handoff ***/
-/**********************/
-static uword
-snat_out2in_worker_handoff_fn (vlib_main_t * vm,
-                               vlib_node_runtime_t * node,
-                               vlib_frame_t * frame)
-{
-  snat_main_t *sm = &snat_main;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  u32 n_left_from, *from, *to_next = 0;
-  static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
-  static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
-    = 0;
-  vlib_frame_queue_elt_t *hf = 0;
-  vlib_frame_t *f = 0;
-  int i;
-  u32 n_left_to_next_worker = 0, *to_next_worker = 0;
-  u32 next_worker_index = 0;
-  u32 current_worker_index = ~0;
-  u32 thread_index = vlib_get_thread_index ();
-
-  ASSERT (vec_len (sm->workers));
-
-  if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
-    {
-      vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
-
-      vec_validate_init_empty (congested_handoff_queue_by_worker_index,
-			       sm->first_worker_index + sm->num_workers - 1,
-			       (vlib_frame_queue_t *) (~0));
-    }
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-
-  while (n_left_from > 0)
-    {
-      u32 bi0;
-      vlib_buffer_t *b0;
-      u32 sw_if_index0;
-      u32 rx_fib_index0;
-      ip4_header_t * ip0;
-      u8 do_handoff;
-
-      bi0 = from[0];
-      from += 1;
-      n_left_from -= 1;
-
-      b0 = vlib_get_buffer (vm, bi0);
-
-      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
-      rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-
-      ip0 = vlib_buffer_get_current (b0);
-
-      next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
-
-      if (PREDICT_FALSE (next_worker_index != thread_index))
-        {
-          do_handoff = 1;
-
-          if (next_worker_index != current_worker_index)
-            {
-              if (hf)
-                hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
-
-              hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
-                                                      next_worker_index,
-                                                      handoff_queue_elt_by_worker_index);
-
-              n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
-              to_next_worker = &hf->buffer_index[hf->n_vectors];
-              current_worker_index = next_worker_index;
-            }
-
-          /* enqueue to correct worker thread */
-          to_next_worker[0] = bi0;
-          to_next_worker++;
-          n_left_to_next_worker--;
-
-          if (n_left_to_next_worker == 0)
-            {
-              hf->n_vectors = VLIB_FRAME_SIZE;
-              vlib_put_frame_queue_elt (hf);
-              current_worker_index = ~0;
-              handoff_queue_elt_by_worker_index[next_worker_index] = 0;
-              hf = 0;
-            }
-        }
-      else
-        {
-          do_handoff = 0;
-          /* if this is 1st frame */
-          if (!f)
-            {
-              f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
-              to_next = vlib_frame_vector_args (f);
-            }
-
-          to_next[0] = bi0;
-          to_next += 1;
-          f->n_vectors++;
-        }
-
-      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-	{
-          snat_out2in_worker_handoff_trace_t *t =
-            vlib_add_trace (vm, node, b0, sizeof (*t));
-          t->next_worker_index = next_worker_index;
-          t->do_handoff = do_handoff;
-        }
-    }
-
-  if (f)
-    vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
-
-  if (hf)
-    hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
-
-  /* Ship frames to the worker nodes */
-  for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
-    {
-      if (handoff_queue_elt_by_worker_index[i])
-	{
-	  hf = handoff_queue_elt_by_worker_index[i];
-	  /*
-	   * It works better to let the handoff node
-	   * rate-adapt, always ship the handoff queue element.
-	   */
-	  if (1 || hf->n_vectors == hf->last_n_vectors)
-	    {
-	      vlib_put_frame_queue_elt (hf);
-	      handoff_queue_elt_by_worker_index[i] = 0;
-	    }
-	  else
-	    hf->last_n_vectors = hf->n_vectors;
-	}
-      congested_handoff_queue_by_worker_index[i] =
-	(vlib_frame_queue_t *) (~0);
-    }
-  hf = 0;
-  current_worker_index = ~0;
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
-  .function = snat_out2in_worker_handoff_fn,
-  .name = "snat-out2in-worker-handoff",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_out2in_worker_handoff_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_next_nodes = 1,
-
-  .next_nodes = {
-    [0] = "error-drop",
-  },
-};
-
-VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
-
-static uword
-snat_out2in_fast_node_fn (vlib_main_t * vm,
-	        	  vlib_node_runtime_t * node,
-		          vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  snat_out2in_next_t next_index;
-  u32 pkts_processed = 0;
-  snat_main_t * sm = &snat_main;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0 = SNAT_OUT2IN_NEXT_DROP;
-          u32 sw_if_index0;
-          ip4_header_t * ip0;
-          ip_csum_t sum0;
-          u32 new_addr0, old_addr0;
-          u16 new_port0, old_port0;
-          udp_header_t * udp0;
-          tcp_header_t * tcp0;
-          icmp46_header_t * icmp0;
-          snat_session_key_t key0, sm0;
-          u32 proto0;
-          u32 rx_fib_index0;
-
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-          ip0 = vlib_buffer_get_current (b0);
-          udp0 = ip4_next_header (ip0);
-          tcp0 = (tcp_header_t *) udp0;
-          icmp0 = (icmp46_header_t *) udp0;
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-	  rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
-
-	  vnet_feature_next (sw_if_index0, &next0, b0);
-
-          if (PREDICT_FALSE(ip0->ttl == 1))
-            {
-              vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
-              icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
-                                           ICMP4_time_exceeded_ttl_exceeded_in_transit,
-                                           0);
-              next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
-              goto trace00;
-            }
-
-          proto0 = ip_proto_to_snat_proto (ip0->protocol);
-
-          if (PREDICT_FALSE (proto0 == ~0))
-              goto trace00;
-
-          if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
-            {
-              next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
-                                  rx_fib_index0, node, next0, ~0, 0, 0);
-              goto trace00;
-            }
-
-          key0.addr = ip0->dst_address;
-          key0.port = udp0->dst_port;
-          key0.fib_index = rx_fib_index0;
-
-          if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
-            {
-              b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
-              goto trace00;
-            }
-
-          new_addr0 = sm0.addr.as_u32;
-          new_port0 = sm0.port;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
-          old_addr0 = ip0->dst_address.as_u32;
-          ip0->dst_address.as_u32 = new_addr0;
-
-          sum0 = ip0->checksum;
-          sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                 ip4_header_t,
-                                 dst_address /* changed member */);
-          ip0->checksum = ip_csum_fold (sum0);
-
-          if (PREDICT_FALSE(new_port0 != udp0->dst_port))
-            {
-               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-                {
-                  old_port0 = tcp0->dst_port;
-                  tcp0->dst_port = new_port0;
-
-                  sum0 = tcp0->checksum;
-                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                         ip4_header_t,
-                                         dst_address /* changed member */);
-
-                  sum0 = ip_csum_update (sum0, old_port0, new_port0,
-                                         ip4_header_t /* cheat */,
-                                         length /* changed member */);
-                  tcp0->checksum = ip_csum_fold(sum0);
-                }
-              else
-                {
-                  old_port0 = udp0->dst_port;
-                  udp0->dst_port = new_port0;
-                  udp0->checksum = 0;
-                }
-            }
-          else
-            {
-              if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
-                {
-                  sum0 = tcp0->checksum;
-                  sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
-                                         ip4_header_t,
-                                         dst_address /* changed member */);
-
-                  tcp0->checksum = ip_csum_fold(sum0);
-                }
-            }
-
-        trace00:
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED)))
-            {
-              snat_out2in_trace_t *t =
-                 vlib_add_trace (vm, node, b0, sizeof (*t));
-              t->sw_if_index = sw_if_index0;
-              t->next_index = next0;
-            }
-
-          pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
-
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
-                               SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
-                               pkts_processed);
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
-  .function = snat_out2in_fast_node_fn,
-  .name = "snat-out2in-fast",
-  .vector_size = sizeof (u32),
-  .format_trace = format_snat_out2in_fast_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_errors = ARRAY_LEN(snat_out2in_error_strings),
-  .error_strings = snat_out2in_error_strings,
-
-  .runtime_data_bytes = sizeof (snat_runtime_t),
-  
-  .n_next_nodes = SNAT_OUT2IN_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-    [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
-    [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
-    [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
-  },
-};
-VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);
diff --git a/src/plugins/snat/snat.api b/src/plugins/snat/snat.api
deleted file mode 100644
index 3c493dda..00000000
--- a/src/plugins/snat/snat.api
+++ /dev/null
@@ -1,897 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file snat.api
- * @brief VPP control-plane API messages.
- *
- * This file defines VPP control-plane API messages which are generally
- * called through a shared memory interface.
- */
-
-/** \brief Add/del S-NAT address range
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param first_ip_address - first IP address
-    @param last_ip_address - last IP address
-    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
-    @param is_add - 1 if add, 0 if delete
-*/
-autoreply define snat_add_address_range {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u8 first_ip_address[16];
-  u8 last_ip_address[16];
-  u32 vrf_id;
-  u8 is_add;
-};
-
-/** \brief Dump S-NAT addresses
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_address_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT address details response
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param ip_address - IP address
-    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
-*/
-define snat_address_details {
-  u32 context;
-  u8 is_ip4;
-  u8 ip_address[16];
-  u32 vrf_id;
-};
-
-/** \brief Enable/disable S-NAT feature on the interface
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_add - 1 if add, 0 if delete
-    @param is_inside - 1 if inside, 0 if outside
-    @param sw_if_index - software index of the interface
-*/
-autoreply define snat_interface_add_del_feature {
-  u32 client_index;
-  u32 context;
-  u8 is_add;
-  u8 is_inside;
-  u32 sw_if_index;
-};
-
-/** \brief Dump interfaces with S-NAT feature
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_interface_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT interface details response
-    @param context - sender context, to match reply w/ request
-    @param is_inside - 1 if inside, 0 if outside
-    @param sw_if_index - software index of the interface
-*/
-define snat_interface_details {
-  u32 context;
-  u8 is_inside;
-  u32 sw_if_index;
-};
-
-/** \brief Enable/disbale S-NAT as an interface output feature (postrouting
-           in2out translation)
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_add - 1 if add, 0 if delete
-    @param is_inside - 1 if inside, 0 if outside
-    @param sw_if_index - software index of the interface
-*/
-autoreply define snat_interface_add_del_output_feature {
-  u32 client_index;
-  u32 context;
-  u8 is_add;
-  u8 is_inside;
-  u32 sw_if_index;
-};
-
-/** \brief Dump interfaces with S-NAT output feature
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_interface_output_feature_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT interface with output feature details response
-    @param context - sender context, to match reply w/ request
-    @param is_inside - 1 if inside, 0 if outside
-    @param sw_if_index - software index of the interface
-*/
-define snat_interface_output_feature_details {
-  u32 context;
-  u8 is_inside;
-  u32 sw_if_index;
-};
-
-/** \brief Add/delete S-NAT static mapping
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_add - 1 if add, 0 if delete
-    @param is_ip4 - 1 if address type is IPv4
-    @param addr_only - 1 if address only mapping
-    @param local_ip_address - local IP address
-    @param external_ip_address - external IP address
-    @param protocol - IP protocol
-    @param local_port - local port number
-    @param external_port - external port number
-    @param external_sw_if_index - external interface (if set
-                                  external_ip_address is ignored, ~0 means not
-                                  used)
-    @param vfr_id - VRF ID
-*/
-autoreply define snat_add_static_mapping {
-  u32 client_index;
-  u32 context;
-  u8 is_add;
-  u8 is_ip4;
-  u8 addr_only;
-  u8 local_ip_address[16];
-  u8 external_ip_address[16];
-  u8 protocol;
-  u16 local_port;
-  u16 external_port;
-  u32 external_sw_if_index;
-  u32 vrf_id;
-};
-
-/** \brief Dump S-NAT static mappings
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_static_mapping_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT static mapping details response
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param addr_only - 1 if address only mapping
-    @param local_ip_address - local IP address
-    @param external_ip_address - external IP address
-    @param protocol - IP protocol
-    @param local_port - local port number
-    @param external_port - external port number
-    @param external_sw_if_index - external interface
-    @param vfr_id - VRF ID
-*/
-define snat_static_mapping_details {
-  u32 context;
-  u8 is_ip4;
-  u8 addr_only;
-  u8 local_ip_address[16];
-  u8 external_ip_address[16];
-  u8 protocol;
-  u16 local_port;
-  u16 external_port;
-  u32 external_sw_if_index;
-  u32 vrf_id;
-};
-
-/** \brief Control ping from client to api server request
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_control_ping
-{
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief Control ping from the client to the server response
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param retval - return code for the request
-    @param vpe_pid - the pid of the vpe, returned by the server
-*/
-define snat_control_ping_reply
-{
-  u32 context;
-  i32 retval;
-  u32 client_index;
-  u32 vpe_pid;
-};
-
-/** \brief Show S-NAT plugin startup config
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_show_config
-{
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief Show S-NAT plugin startup config reply
-    @param context - sender context, to match reply w/ request
-    @param retval - return code for the request
-    @param static_mapping_only - if 1 dynamic translations disabled
-    @param static_mapping_connection_tracking - if 1 create session data
-    @param deterministic - if 1 deterministic mapping
-    @param translation_buckets - number of translation hash buckets
-    @param translation_memory_size - translation hash memory size
-    @param user_buckets - number of user hash buckets
-    @param user_memory_size - user hash memory size
-    @param max_translations_per_user - maximum number of translations per user
-    @param outside_vrf_id - outside VRF id
-    @param inside_vrf_id - default inside VRF id
-*/
-define snat_show_config_reply
-{
-  u32 context;
-  i32 retval;
-  u8 static_mapping_only;
-  u8 static_mapping_connection_tracking;
-  u8 deterministic;
-  u32 translation_buckets;
-  u32 translation_memory_size;
-  u32 user_buckets;
-  u32 user_memory_size;
-  u32 max_translations_per_user;
-  u32 outside_vrf_id;
-  u32 inside_vrf_id;
-};
-
-/** \brief Set S-NAT workers
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param worker_mask - S-NAT workers mask
-*/
-autoreply define snat_set_workers {
-  u32 client_index;
-  u32 context;
-  u64 worker_mask;
-};
-
-/** \brief Dump S-NAT workers
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_worker_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT workers details response
-    @param context - sender context, to match reply w/ request
-    @param worker_index - worker index
-    @param lcore_id - lcore ID
-    @param name - worker name
-*/
-define snat_worker_details {
-  u32 context;
-  u32 worker_index;
-  u32 lcore_id;
-  u8 name[64];
-};
-
-/** \brief Add/delete S-NAT pool address from specific interfce
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_add - 1 if add, 0 if delete
-    @param sw_if_index - software index of the interface
-*/
-autoreply define snat_add_del_interface_addr {
-  u32 client_index;
-  u32 context;
-  u8 is_add;
-  u8 is_inside;
-  u32 sw_if_index;
-};
-
-/** \brief Dump S-NAT pool addresses interfaces
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_interface_addr_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT pool addresses interfaces details response
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - software index of the interface
-*/
-define snat_interface_addr_details {
-  u32 context;
-  u32 sw_if_index;
-};
-
-/** \brief Enable/disable S-NAT IPFIX logging
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param domain_id - observation domain ID
-    @param src_port - source port number
-    @param enable - 1 if enable, 0 if disable
-*/
-autoreply define snat_ipfix_enable_disable {
-  u32 client_index;
-  u32 context;
-  u32 domain_id;
-  u16 src_port;
-  u8 enable;
-};
-
-/** \brief Dump S-NAT users
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_user_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT users response
-    @param context - sender context, to match reply w/ request
-    @vrf_id - VRF ID
-    @param is_ip4 - 1 if address type is IPv4
-    @param ip_adress - IP address
-    @param nsessions - number of dynamic sessions
-    @param nstaticsessions - number of static sessions
-*/
-define snat_user_details {
-  u32 context;
-  u32 vrf_id;
-  u8 is_ip4;
-  u8 ip_address[16];
-  u32 nsessions;
-  u32 nstaticsessions;
-};
-
-/** \brief S-NAT user's sessions
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param user_ip - IP address of the user to dump
-    @param vrf_id - VRF_ID
-*/
-define snat_user_session_dump {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u8 ip_address[16];
-  u32 vrf_id;
-};
-
-/** \brief S-NAT user's sessions response
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param outside_ip_address - outside IP address
-    @param outside_port - outside port
-    @param inside_ip_address - inside IP address
-    @param inside_port - inside port
-    @param protocol - protocol
-    @param is_static - 1 if session is static
-    @param last_heard - last heard timer
-    @param total_bytes - count of bytes sent through session
-    @param total_pkts - count of pakets sent through session
-*/
-define snat_user_session_details {
-  u32 context;
-  u8 is_ip4;
-  u8 outside_ip_address[16];
-  u16 outside_port;
-  u8 inside_ip_address[16];
-  u16 inside_port;
-  u16 protocol;
-  u8 is_static;
-  u64 last_heard;
-  u64 total_bytes;
-  u32 total_pkts;
-};
-
-/** \brief Add/delete S-NAT deterministic mapping
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_add - 1 if add, 0 if delete
-    @param is_ip4 - 1 if address type is IPv4
-    @param in_addr - inside IP address
-    @param in_plen - inside IP address prefix length
-    @param out_addr - outside IP address
-    @param out_addr - outside IP address prefix length
-*/
-autoreply define snat_add_det_map {
-  u32 client_index;
-  u32 context;
-  u8 is_add;
-  u8 is_ip4;
-  u8 addr_only;
-  u8 in_addr[16];
-  u8 in_plen;
-  u8 out_addr[16];
-  u8 out_plen;
-};
-
-/** \brief Get outside address and port range from inside address
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param in_addr - inside IP address
-*/
-define snat_det_forward {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u8 in_addr[16];
-};
-
-/** \brief Get outside address and port range from inside address
-    @param context - sender context, to match reply w/ request
-    @param retval - return code
-    @param out_port_lo - outside port range start
-    @param out_port_hi - outside port range end
-    @param is_ip4 - 1 if address type is IPv4
-    @param out_addr - outside IP address
-*/
-define snat_det_forward_reply {
-  u32 context;
-  i32 retval;
-  u16 out_port_lo;
-  u16 out_port_hi;
-  u8 is_ip4;
-  u8 out_addr[16];
-};
-
-/** \brief Get inside address from outside address and port
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param out_port - outside port
-    @param is_ip4 - 1 if address type is IPv4
-    @param out_addr - outside IP address
-*/
-define snat_det_reverse {
-  u32 client_index;
-  u32 context;
-  u16 out_port;
-  u8 is_ip4;
-  u8 out_addr[16];
-};
-
-/** \brief Get inside address from outside address and port reply
-    @param context - sender context, to match reply w/ request
-    @param retval - return code
-    @param is_ip4 - 1 if address type is IPv4
-    @param in_addr - inside IP address
-*/
-define snat_det_reverse_reply {
-  u32 context;
-  i32 retval;
-  u8 is_ip4;
-  u8 in_addr[16];
-};
-
-/** \brief Dump S-NAT deterministic mappings
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_det_map_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief S-NAT users response
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param in_addr - inside IP address
-    @param in_plen - inside IP address prefix length
-    @param out_addr - outside IP address
-    @param out_plen - outside IP address prefix length
-    @param sharing_ratio - outside to inside address sharing ratio
-    @param ports_per_host - number of ports available to a host
-    @param ses_num - number of sessions belonging to this mapping
-*/
-define snat_det_map_details {
-  u32 context;
-  u8 is_ip4;
-  u8 in_addr[16];
-  u8 in_plen;
-  u8 out_addr[16];
-  u8 out_plen;
-  u32 sharing_ratio;
-  u16 ports_per_host;
-  u32 ses_num;
-};
-
-/** \brief Set values of timeouts for deterministic NAT (seconds, 0 = default)
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param udp - UDP timeout (default 300sec)
-    @param tcp_established - TCP established timeout (default 7440sec)
-    @param tcp_transitory - TCP transitory timeout (default 240sec)
-    @param icmp - ICMP timeout (default 60sec)
-*/
-autoreply define snat_det_set_timeouts {
-  u32 client_index;
-  u32 context;
-  u32 udp;
-  u32 tcp_established;
-  u32 tcp_transitory;
-  u32 icmp;
-};
-
-/** \brief Get values of timeouts for deterministic NAT (seconds)
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define snat_det_get_timeouts {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief Get values of timeouts for deterministic NAT reply
-    @param context - sender context, to match reply w/ request
-    @param retval - return code
-    @param udp - UDP timeout (default 300sec)
-    @param tcp_established - TCP established timeout (default 7440sec)
-    @param tcp_transitory - TCP transitory timeout (default 240sec)
-    @param icmp - ICMP timeout (default 60sec)
-*/
-define snat_det_get_timeouts_reply {
-  u32 context;
-  i32 retval;
-  u32 udp;
-  u32 tcp_established;
-  u32 tcp_transitory;
-  u32 icmp;
-};
-
-/** \brief Close CGNAT session by outside address and port
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param out_addr - outside IP address
-    @param out_port - outside port
-    @param ext_addr - external host address
-    @param ext_port - external host port
-*/
-autoreply define snat_det_close_session_out {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u8 out_addr[16];
-  u16 out_port;
-  u8 ext_addr[16];
-  u16 ext_port;
-};
-
-/** \brief Close CGNAT session by inside address and port
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param in_addr - inside IP address
-    @param in_port - inside port
-    @param ext_addr - external host address
-    @param ext_port - external host port
-*/
-autoreply define snat_det_close_session_in {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u8 in_addr[16];
-  u16 in_port;
-  u8 ext_addr[16];
-  u16 ext_port;
-};
-
-/** \brief Dump S-NAT deterministic sessions
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param user_addr - address of an inside user whose sessions to dump
-*/
-define snat_det_session_dump {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u8 user_addr[16];
-};
-
-/** \brief S-NAT deterministic sessions reply
-    @param context - sender context, to match reply w/ request
-    @param is_ip4 - 1 if address type is IPv4
-    @param in_port - inside port
-    @param ext_addr - external host address
-    @param ext_port - external host port
-    @param out_port - outside NAT port
-    @param state - session state
-    @param expire - session expiration timestamp
-*/
-define snat_det_session_details {
-  u32 client_index;
-  u32 context;
-  u8 is_ip4;
-  u16 in_port;
-  u8 ext_addr[16];
-  u16 ext_port;
-  u16 out_port;
-  u8 state;
-  u32 expire;
-};
-
-/** \brief Add/delete address range to NAT64 pool
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param start_addr - start address of the range
-    @param end_addr - end address of the range
-    @param vrf_id - VRF id of tenant, ~0 means independent of VRF
-    @param is_add - 1 if add, 0 if delete
-*/
-autoreply define nat64_add_del_pool_addr_range {
-  u32 client_index;
-  u32 context;
-  u8 start_addr[4];
-  u8 end_addr[4];
-  u32 vrf_id;
-  u8 is_add;
-};
-
-/** \brief Dump NAT64 pool addresses
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define nat64_pool_addr_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief NAT64 pool address details response
-    @param context - sender context, to match reply w/ request
-    @param address - IPv4 address
-    @param vfr_id - VRF id of tenant, ~0 means independent of VRF
-*/
-define nat64_pool_addr_details {
-  u32 context;
-  u8 address[4];
-  u32 vrf_id;
-};
-
-/** \brief Enable/disable NAT64 feature on the interface
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param sw_if_index - index of the interface
-    @param is_inside - 1 if inside, 0 if outside
-    @param is_add - 1 if add, 0 if delete
-*/
-autoreply define nat64_add_del_interface {
-  u32 client_index;
-  u32 context;
-  u32 sw_if_index;
-  u8 is_inside;
-  u8 is_add;
-};
-
-/** \brief Dump interfaces with NAT64 feature
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define nat64_interface_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief NAT64 interface details response
-    @param context - sender context, to match reply w/ request
-    @param is_inside - 1 if inside, 0 if outside
-    @param sw_if_index - index of the interface
-*/
-define nat64_interface_details {
-  u32 context;
-  u8 is_inside;
-  u32 sw_if_index;
-};
-
-/** \brief Add/delete NAT64 static BIB entry
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param i_addr - inside IPv6 address
-    @param o_addr - outside IPv4 address
-    @param i_port - inside port number
-    @param o_port - outside port number
-    @param vrf_id - VRF id of tenant
-    @param proto - protocol number
-    @param is_add - 1 if add, 0 if delete
-*/
- autoreply define nat64_add_del_static_bib {
-  u32 client_index;
-  u32 context;
-  u8 i_addr[16];
-  u8 o_addr[4];
-  u16 i_port;
-  u16 o_port;
-  u32 vrf_id;
-  u8 proto;
-  u8 is_add;
-};
-
-/** \brief Dump NAT64 BIB
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param proto - protocol of the BIB: 255 - all BIBs
-                                        6 - TCP BIB
-                                        17 - UDP BIB
-                                        1/58 - ICMP BIB
-                                        otherwise - "unknown" protocol BIB
-*/
-define nat64_bib_dump {
-  u32 client_index;
-  u32 context;
-  u8 proto;
-};
-
-/** \brief NAT64 BIB details response
-    @param context - sender context, to match reply w/ request
-    @param i_addr - inside IPv6 address
-    @param o_addr - outside IPv4 address
-    @param i_port - inside port number
-    @param o_port - outside port number
-    @param vrf_id - VRF id of tenant
-    @param proto - protocol number
-    @param is_static - 1 if static BIB entry, 0 if dynamic
-    @param ses_num - number of sessions associated with the BIB entry
-*/
-define nat64_bib_details {
-  u32 context;
-  u8 i_addr[16];
-  u8 o_addr[4];
-  u16 i_port;
-  u16 o_port;
-  u32 vrf_id;
-  u8 proto;
-  u8 is_static;
-  u32 ses_num;
-};
-
-/** \brief Set values of timeouts for NAT64 (seconds, 0 = default)
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param udp - UDP timeout (default 300sec)
-    @param icmp - ICMP timeout (default 60sec)
-    @param tcp_trans - TCP transitory timeout (default 240sec)
-    @param tcp_est - TCP established timeout (default 7440sec)
-    @param tcp_incoming_syn - TCP incoming SYN timeout (default 6sec)
-*/
-autoreply define nat64_set_timeouts {
-  u32 client_index;
-  u32 context;
-  u32 udp;
-  u32 icmp;
-  u32 tcp_trans;
-  u32 tcp_est;
-  u32 tcp_incoming_syn;
-};
-
-/** \brief Get values of timeouts for NAT64 (seconds)
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define nat64_get_timeouts {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief Get values of timeouts for NAT64 reply
-    @param context - sender context, to match reply w/ request
-    @param retval - return code
-    @param udp - UDP timeout
-    @param icmp - ICMP timeout
-    @param tcp_trans - TCP transitory timeout
-    @param tcp_est - TCP established timeout
-    @param tcp_incoming_syn - TCP incoming SYN timeout
-*/
-define nat64_get_timeouts_reply {
-  u32 context;
-  i32 retval;
-  u32 udp;
-  u32 icmp;
-  u32 tcp_trans;
-  u32 tcp_est;
-  u32 tcp_incoming_syn;
-};
-
-/** \brief Dump NAT64 session table
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param proto - protocol of the session table: 255 - all STs
-                                                  6 - TCP ST
-                                                  17 - UDP ST
-                                                  1/58 - ICMP ST
-                                                  otherwise - "unknown" proto ST
-*/
-define nat64_st_dump {
-  u32 client_index;
-  u32 context;
-  u8 proto;
-};
-
-/** \brief NAT64 session table details response
-    @param context - sender context, to match reply w/ request
-    @param il_addr - inside IPv6 address of the local host
-    @param ol_addr - outside IPv4 address of the local host
-    @param il_port - inside port number id of the local host/inside ICMP id
-    @param ol_port - outside port number of the local host/outside ICMP id
-    @param il_addr - inside IPv6 address of the remote host
-    @param ol_addr - outside IPv4 address of the remote host
-    @param l_port - port number of the remote host (not used for ICMP)
-    @param vrf_id - VRF id of tenant
-    @param proto - protocol number
-*/
-define nat64_st_details {
-  u32 context;
-  u8 il_addr[16];
-  u8 ol_addr[4];
-  u16 il_port;
-  u16 ol_port;
-  u8 ir_addr[16];
-  u8 or_addr[4];
-  u16 r_port;
-  u32 vrf_id;
-  u8 proto;
-};
-
-/** \brief Add/del NAT64 prefix
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-    @param prefix - NAT64 prefix
-    @param prefix - NAT64 prefix length
-    @param vrf_id - VRF id of tenant
-    @param is_add - 1 if add, 0 if delete
-*/
-autoreply define nat64_add_del_prefix {
-  u32 client_index;
-  u32 context;
-  u8 prefix[16];
-  u8 prefix_len;
-  u32 vrf_id;
-  u8 is_add;
-};
-
-/** \brief Dump NAT64 prefix
-    @param client_index - opaque cookie to identify the sender
-    @param context - sender context, to match reply w/ request
-*/
-define nat64_prefix_dump {
-  u32 client_index;
-  u32 context;
-};
-
-/** \brief Dump NAT64 prefix details response
-    @param context - sender context, to match reply w/ request
-    @param prefix - NAT64 prefix
-    @param prefix - NAT64 prefix length
-    @param vrf_id - VRF id of tenant
-*/
-define nat64_prefix_details {
-  u32 context;
-  u8 prefix[16];
-  u8 prefix_len;
-  u32 vrf_id;
-};
diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c
deleted file mode 100644
index 315cec8a..00000000
--- a/src/plugins/snat/snat.c
+++ /dev/null
@@ -1,2842 +0,0 @@
-/*
- * snat.c - simple nat plugin
- *
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-
-snat_main_t snat_main;
-
-
-/* Hook up input features */
-VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-in2out",
-  .runs_before = VNET_FEATURES ("snat-out2in"),
-};
-VNET_FEATURE_INIT (ip4_snat_out2in, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-out2in",
-  .runs_before = VNET_FEATURES ("ip4-lookup"),
-};
-VNET_FEATURE_INIT (ip4_snat_det_in2out, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-det-in2out",
-  .runs_before = VNET_FEATURES ("snat-det-out2in"),
-};
-VNET_FEATURE_INIT (ip4_snat_det_out2in, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-det-out2in",
-  .runs_before = VNET_FEATURES ("ip4-lookup"),
-};
-VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-in2out-worker-handoff",
-  .runs_before = VNET_FEATURES ("snat-out2in-worker-handoff"),
-};
-VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-out2in-worker-handoff",
-  .runs_before = VNET_FEATURES ("ip4-lookup"),
-};
-VNET_FEATURE_INIT (ip4_snat_in2out_fast, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-in2out-fast",
-  .runs_before = VNET_FEATURES ("snat-out2in-fast"),
-};
-VNET_FEATURE_INIT (ip4_snat_out2in_fast, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-out2in-fast",
-  .runs_before = VNET_FEATURES ("ip4-lookup"),
-};
-VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "snat-hairpin-dst",
-  .runs_before = VNET_FEATURES ("ip4-lookup"),
-};
-
-/* Hook up output features */
-VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
-  .arc_name = "ip4-output",
-  .node_name = "snat-in2out-output",
-  .runs_before = VNET_FEATURES ("interface-output"),
-};
-VNET_FEATURE_INIT (ip4_snat_in2out_output_worker_handoff, static) = {
-  .arc_name = "ip4-output",
-  .node_name = "snat-in2out-output-worker-handoff",
-  .runs_before = VNET_FEATURES ("interface-output"),
-};
-VNET_FEATURE_INIT (ip4_snat_hairpin_src, static) = {
-  .arc_name = "ip4-output",
-  .node_name = "snat-hairpin-src",
-  .runs_before = VNET_FEATURES ("interface-output"),
-};
-
-
-/* *INDENT-OFF* */
-VLIB_PLUGIN_REGISTER () = {
-    .version = VPP_BUILD_VER,
-    .description = "Network Address Translation",
-};
-/* *INDENT-ON* */
-
-/**
- * @brief Add/del NAT address to FIB.
- *
- * Add the external NAT address to the FIB as receive entries. This ensures
- * that VPP will reply to ARP for this address and we don't need to enable
- * proxy ARP on the outside interface.
- *
- * @param addr IPv4 address.
- * @param plen address prefix length
- * @param sw_if_index Interface.
- * @param is_add If 0 delete, otherwise add.
- */
-void
-snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
-                          int is_add)
-{
-  fib_prefix_t prefix = {
-    .fp_len = p_len,
-    .fp_proto = FIB_PROTOCOL_IP4,
-    .fp_addr = {
-        .ip4.as_u32 = addr->as_u32,
-    },
-  };
-  u32 fib_index = ip4_fib_table_get_index_for_sw_if_index(sw_if_index);
-
-  if (is_add)
-    fib_table_entry_update_one_path(fib_index,
-                                    &prefix,
-                                    FIB_SOURCE_PLUGIN_HI,
-                                    (FIB_ENTRY_FLAG_CONNECTED |
-                                     FIB_ENTRY_FLAG_LOCAL |
-                                     FIB_ENTRY_FLAG_EXCLUSIVE),
-                                    DPO_PROTO_IP4,
-                                    NULL,
-                                    sw_if_index,
-                                    ~0,
-                                    1,
-                                    NULL,
-                                    FIB_ROUTE_PATH_FLAG_NONE);
-  else
-    fib_table_entry_delete(fib_index,
-                           &prefix,
-                           FIB_SOURCE_PLUGIN_HI);
-}
-
-void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id)
-{
-  snat_address_t * ap;
-  snat_interface_t *i;
-
-  if (vrf_id != ~0)
-    sm->vrf_mode = 1;
-
-  /* Check if address already exists */
-  vec_foreach (ap, sm->addresses)
-    {
-      if (ap->addr.as_u32 == addr->as_u32)
-        return;
-    }
-
-  vec_add2 (sm->addresses, ap, 1);
-  ap->addr = *addr;
-  if (vrf_id != ~0)
-    ap->fib_index =
-      fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id);
-  else
-    ap->fib_index = ~0;
-#define _(N, i, n, s) \
-  clib_bitmap_alloc (ap->busy_##n##_port_bitmap, 65535);
-  foreach_snat_protocol
-#undef _
-
-  /* Add external address to FIB */
-  pool_foreach (i, sm->interfaces,
-  ({
-    if (i->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
-    break;
-  }));
-  pool_foreach (i, sm->output_feature_interfaces,
-  ({
-    if (i->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(addr, 32, i->sw_if_index, 1);
-    break;
-  }));
-}
-
-static int is_snat_address_used_in_static_mapping (snat_main_t *sm,
-                                                   ip4_address_t addr)
-{
-  snat_static_mapping_t *m;
-  pool_foreach (m, sm->static_mappings,
-  ({
-      if (m->external_addr.as_u32 == addr.as_u32)
-        return 1;
-  }));
-
-  return 0;
-}
-
-void increment_v4_address (ip4_address_t * a)
-{
-  u32 v;
-  
-  v = clib_net_to_host_u32(a->as_u32) + 1;
-  a->as_u32 = clib_host_to_net_u32(v);
-}
-
-static void 
-snat_add_static_mapping_when_resolved (snat_main_t * sm, 
-                                       ip4_address_t l_addr, 
-                                       u16 l_port, 
-                                       u32 sw_if_index, 
-                                       u16 e_port, 
-                                       u32 vrf_id,
-                                       snat_protocol_t proto,
-                                       int addr_only,  
-                                       int is_add)
-{
-  snat_static_map_resolve_t *rp;
-
-  vec_add2 (sm->to_resolve, rp, 1);
-  rp->l_addr.as_u32 = l_addr.as_u32;
-  rp->l_port = l_port;
-  rp->sw_if_index = sw_if_index;
-  rp->e_port = e_port;
-  rp->vrf_id = vrf_id;
-  rp->proto = proto;
-  rp->addr_only = addr_only;
-  rp->is_add = is_add;
-}
-
-/**
- * @brief Add static mapping.
- *
- * Create static mapping between local addr+port and external addr+port.
- *
- * @param l_addr Local IPv4 address.
- * @param e_addr External IPv4 address.
- * @param l_port Local port number.
- * @param e_port External port number.
- * @param vrf_id VRF ID.
- * @param addr_only If 0 address port and pair mapping, otherwise address only.
- * @param sw_if_index External port instead of specific IP address.
- * @param is_add If 0 delete static mapping, otherwise add.
- *
- * @returns
- */
-int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
-                            u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
-                            u32 sw_if_index, snat_protocol_t proto, int is_add)
-{
-  snat_main_t * sm = &snat_main;
-  snat_static_mapping_t *m;
-  snat_session_key_t m_key;
-  clib_bihash_kv_8_8_t kv, value;
-  snat_address_t *a = 0;
-  u32 fib_index = ~0;
-  uword * p;
-  snat_interface_t *interface;
-  int i;
-
-  /* If the external address is a specific interface address */
-  if (sw_if_index != ~0)
-    {
-      ip4_address_t * first_int_addr;
-
-      /* Might be already set... */
-      first_int_addr = ip4_interface_first_address 
-        (sm->ip4_main, sw_if_index, 0 /* just want the address*/);
-
-      /* DHCP resolution required? */
-      if (first_int_addr == 0)
-        {
-          snat_add_static_mapping_when_resolved 
-            (sm, l_addr, l_port, sw_if_index, e_port, vrf_id, proto,
-             addr_only,  is_add);
-          return 0;
-        }
-        else
-          e_addr.as_u32 = first_int_addr->as_u32;
-    }
-
-  m_key.addr = e_addr;
-  m_key.port = addr_only ? 0 : e_port;
-  m_key.protocol = addr_only ? 0 : proto;
-  m_key.fib_index = sm->outside_fib_index;
-  kv.key = m_key.as_u64;
-  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
-    m = 0;
-  else
-    m = pool_elt_at_index (sm->static_mappings, value.value);
-
-  if (is_add)
-    {
-      if (m)
-        return VNET_API_ERROR_VALUE_EXIST;
-
-      /* Convert VRF id to FIB index */
-      if (vrf_id != ~0)
-        {
-          p = hash_get (sm->ip4_main->fib_index_by_table_id, vrf_id);
-          if (!p)
-            return VNET_API_ERROR_NO_SUCH_FIB;
-          fib_index = p[0];
-        }
-      /* If not specified use inside VRF id from SNAT plugin startup config */
-      else
-        {
-          fib_index = sm->inside_fib_index;
-          vrf_id = sm->inside_vrf_id;
-        }
-
-      /* Find external address in allocated addresses and reserve port for
-         address and port pair mapping when dynamic translations enabled */
-      if (!addr_only && !(sm->static_mapping_only))
-        {
-          for (i = 0; i < vec_len (sm->addresses); i++)
-            {
-              if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
-                {
-                  a = sm->addresses + i;
-                  /* External port must be unused */
-                  switch (proto)
-                    {
-#define _(N, j, n, s) \
-                    case SNAT_PROTOCOL_##N: \
-                      if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, e_port)) \
-                        return VNET_API_ERROR_INVALID_VALUE; \
-                      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 1); \
-                      if (e_port > 1024) \
-                        a->busy_##n##_ports++; \
-                      break;
-                      foreach_snat_protocol
-#undef _
-                    default:
-                      clib_warning("unknown_protocol");
-                      return VNET_API_ERROR_INVALID_VALUE_2;
-                    }
-                  break;
-                }
-            }
-          /* External address must be allocated */
-          if (!a)
-            return VNET_API_ERROR_NO_SUCH_ENTRY;
-        }
-
-      pool_get (sm->static_mappings, m);
-      memset (m, 0, sizeof (*m));
-      m->local_addr = l_addr;
-      m->external_addr = e_addr;
-      m->addr_only = addr_only;
-      m->vrf_id = vrf_id;
-      m->fib_index = fib_index;
-      if (!addr_only)
-        {
-          m->local_port = l_port;
-          m->external_port = e_port;
-          m->proto = proto;
-        }
-
-      m_key.addr = m->local_addr;
-      m_key.port = m->local_port;
-      m_key.protocol = m->proto;
-      m_key.fib_index = m->fib_index;
-      kv.key = m_key.as_u64;
-      kv.value = m - sm->static_mappings;
-      clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 1);
-
-      m_key.addr = m->external_addr;
-      m_key.port = m->external_port;
-      m_key.fib_index = sm->outside_fib_index;
-      kv.key = m_key.as_u64;
-      kv.value = m - sm->static_mappings;
-      clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
-
-      /* Assign worker */
-      if (sm->workers)
-        {
-          snat_user_key_t w_key0;
-          snat_worker_key_t w_key1;
-
-          w_key0.addr = m->local_addr;
-          w_key0.fib_index = m->fib_index;
-          kv.key = w_key0.as_u64;
-
-          if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value))
-            {
-              kv.value = sm->first_worker_index +
-                sm->workers[sm->next_worker++ % vec_len (sm->workers)];
-
-              clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv, 1);
-            }
-          else
-            {
-              kv.value = value.value;
-            }
-
-          w_key1.addr = m->external_addr;
-          w_key1.port = clib_host_to_net_u16 (m->external_port);
-          w_key1.fib_index = sm->outside_fib_index;
-          kv.key = w_key1.as_u64;
-          clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv, 1);
-        }
-    }
-  else
-    {
-      if (!m)
-        return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-      /* Free external address port */
-      if (!addr_only && !(sm->static_mapping_only))
-        {
-          for (i = 0; i < vec_len (sm->addresses); i++)
-            {
-              if (sm->addresses[i].addr.as_u32 == e_addr.as_u32)
-                {
-                  a = sm->addresses + i;
-                  switch (proto)
-                    {
-#define _(N, j, n, s) \
-                    case SNAT_PROTOCOL_##N: \
-                      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, e_port, 0); \
-                      if (e_port > 1024) \
-                        a->busy_##n##_ports--; \
-                      break;
-                      foreach_snat_protocol
-#undef _
-                    default:
-                      clib_warning("unknown_protocol");
-                      return VNET_API_ERROR_INVALID_VALUE_2;
-                    }
-                  break;
-                }
-            }
-        }
-
-      m_key.addr = m->local_addr;
-      m_key.port = m->local_port;
-      m_key.protocol = m->proto;
-      m_key.fib_index = m->fib_index;
-      kv.key = m_key.as_u64;
-      clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0);
-
-      m_key.addr = m->external_addr;
-      m_key.port = m->external_port;
-      m_key.fib_index = sm->outside_fib_index;
-      kv.key = m_key.as_u64;
-      clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
-
-      /* Delete session(s) for static mapping if exist */
-      if (!(sm->static_mapping_only) ||
-          (sm->static_mapping_only && sm->static_mapping_connection_tracking))
-        {
-          snat_user_key_t u_key;
-          snat_user_t *u;
-          dlist_elt_t * head, * elt;
-          u32 elt_index, head_index, del_elt_index;
-          u32 ses_index;
-          u64 user_index;
-          snat_session_t * s;
-          snat_main_per_thread_data_t *tsm;
-
-          u_key.addr = m->local_addr;
-          u_key.fib_index = m->fib_index;
-          kv.key = u_key.as_u64;
-          if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
-            {
-              user_index = value.value;
-              if (!clib_bihash_search_8_8 (&sm->worker_by_in, &kv, &value))
-                tsm = vec_elt_at_index (sm->per_thread_data, value.value);
-              else
-                tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
-              u = pool_elt_at_index (tsm->users, user_index);
-              if (u->nstaticsessions)
-                {
-                  head_index = u->sessions_per_user_list_head_index;
-                  head = pool_elt_at_index (tsm->list_pool, head_index);
-                  elt_index = head->next;
-                  elt = pool_elt_at_index (tsm->list_pool, elt_index);
-                  ses_index = elt->value;
-                  while (ses_index != ~0)
-                    {
-                      s =  pool_elt_at_index (tsm->sessions, ses_index);
-                      del_elt_index = elt_index;
-                      elt_index = elt->next;
-                      elt = pool_elt_at_index (tsm->list_pool, elt_index);
-                      ses_index = elt->value;
-
-                      if (!addr_only)
-                        {
-                          if ((s->out2in.addr.as_u32 != e_addr.as_u32) &&
-                              (clib_net_to_host_u16 (s->out2in.port) != e_port))
-                            continue;
-                        }
-
-                      if (snat_is_unk_proto_session (s))
-                        {
-                          clib_bihash_kv_16_8_t up_kv;
-                          snat_unk_proto_ses_key_t up_key;
-                          up_key.l_addr = s->in2out.addr;
-                          up_key.r_addr = s->ext_host_addr;
-                          up_key.fib_index = s->in2out.fib_index;
-                          up_key.proto = s->in2out.port;
-                          up_key.rsvd[0] = up_key.rsvd[1] = up_key.rsvd[2] = 0;
-                          up_kv.key[0] = up_key.as_u64[0];
-                          up_kv.key[1] = up_key.as_u64[1];
-                          if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto,
-                                                        &up_kv, 0))
-                            clib_warning ("in2out key del failed");
-
-                          up_key.l_addr = s->out2in.addr;
-                          up_key.fib_index = s->out2in.fib_index;
-                          up_kv.key[0] = up_key.as_u64[0];
-                          up_kv.key[1] = up_key.as_u64[1];
-                          if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto,
-                                                        &up_kv, 0))
-                            clib_warning ("out2in key del failed");
-
-                          goto delete;
-                        }
-                      /* log NAT event */
-                      snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
-                                                          s->out2in.addr.as_u32,
-                                                          s->in2out.protocol,
-                                                          s->in2out.port,
-                                                          s->out2in.port,
-                                                          s->in2out.fib_index);
-
-                      value.key = s->in2out.as_u64;
-                      if (clib_bihash_add_del_8_8 (&sm->in2out, &value, 0))
-                        clib_warning ("in2out key del failed");
-                      value.key = s->out2in.as_u64;
-                      if (clib_bihash_add_del_8_8 (&sm->out2in, &value, 0))
-                        clib_warning ("out2in key del failed");
-delete:
-                      pool_put (tsm->sessions, s);
-
-                      clib_dlist_remove (tsm->list_pool, del_elt_index);
-                      pool_put_index (tsm->list_pool, del_elt_index);
-                      u->nstaticsessions--;
-
-                      if (!addr_only)
-                        break;
-                    }
-                  if (addr_only)
-                    {
-                      pool_put (tsm->users, u);
-                      clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 0);
-                    }
-                }
-            }
-        }
-
-      /* Delete static mapping from pool */
-      pool_put (sm->static_mappings, m);
-    }
-
-  if (!addr_only)
-    return 0;
-
-  /* Add/delete external address to FIB */
-  pool_foreach (interface, sm->interfaces,
-  ({
-    if (interface->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
-    break;
-  }));
-  pool_foreach (interface, sm->output_feature_interfaces,
-  ({
-    if (interface->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(&e_addr, 32, interface->sw_if_index, is_add);
-    break;
-  }));
-
-  return 0;
-}
-
-int snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm)
-{
-  snat_address_t *a = 0;
-  snat_session_t *ses;
-  u32 *ses_to_be_removed = 0, *ses_index;
-  clib_bihash_kv_8_8_t kv, value;
-  snat_user_key_t user_key;
-  snat_user_t *u;
-  snat_main_per_thread_data_t *tsm;
-  snat_static_mapping_t *m;
-  snat_interface_t *interface;
-  int i;
-
-  /* Find SNAT address */
-  for (i=0; i < vec_len (sm->addresses); i++)
-    {
-      if (sm->addresses[i].addr.as_u32 == addr.as_u32)
-        {
-          a = sm->addresses + i;
-          break;
-        }
-    }
-  if (!a)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  if (delete_sm)
-    {
-      pool_foreach (m, sm->static_mappings,
-      ({
-          if (m->external_addr.as_u32 == addr.as_u32)
-            (void) snat_add_static_mapping (m->local_addr, m->external_addr,
-                                            m->local_port, m->external_port,
-                                            m->vrf_id, m->addr_only, ~0,
-                                            m->proto, 0);
-      }));
-    }
-  else
-    {
-      /* Check if address is used in some static mapping */
-      if (is_snat_address_used_in_static_mapping(sm, addr))
-        {
-          clib_warning ("address used in static mapping");
-          return VNET_API_ERROR_UNSPECIFIED;
-        }
-    }
-
-  if (a->fib_index != ~0)
-    fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4);
-
-  /* Delete sessions using address */
-  if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
-    {
-      vec_foreach (tsm, sm->per_thread_data)
-        {
-          pool_foreach (ses, tsm->sessions, ({
-            if (ses->out2in.addr.as_u32 == addr.as_u32)
-              {
-                if (snat_is_unk_proto_session (ses))
-                  {
-                    clib_bihash_kv_16_8_t up_kv;
-                    snat_unk_proto_ses_key_t up_key;
-                    up_key.l_addr = ses->in2out.addr;
-                    up_key.r_addr = ses->ext_host_addr;
-                    up_key.fib_index = ses->in2out.fib_index;
-                    up_key.proto = ses->in2out.port;
-                    up_key.rsvd[0] = up_key.rsvd[1] = up_key.rsvd[2] = 0;
-                    up_kv.key[0] = up_key.as_u64[0];
-                    up_kv.key[1] = up_key.as_u64[1];
-                    if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto,
-                                                  &up_kv, 0))
-                      clib_warning ("in2out key del failed");
-
-                    up_key.l_addr = ses->out2in.addr;
-                    up_key.fib_index = ses->out2in.fib_index;
-                    up_kv.key[0] = up_key.as_u64[0];
-                    up_kv.key[1] = up_key.as_u64[1];
-                    if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto,
-                                                  &up_kv, 0))
-                      clib_warning ("out2in key del failed");
-                  }
-                else
-                  {
-                    /* log NAT event */
-                    snat_ipfix_logging_nat44_ses_delete(ses->in2out.addr.as_u32,
-                                                        ses->out2in.addr.as_u32,
-                                                        ses->in2out.protocol,
-                                                        ses->in2out.port,
-                                                        ses->out2in.port,
-                                                        ses->in2out.fib_index);
-                    kv.key = ses->in2out.as_u64;
-                    clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0);
-                    kv.key = ses->out2in.as_u64;
-                    clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0);
-                  }
-                vec_add1 (ses_to_be_removed, ses - tsm->sessions);
-                clib_dlist_remove (tsm->list_pool, ses->per_user_index);
-                user_key.addr = ses->in2out.addr;
-                user_key.fib_index = ses->in2out.fib_index;
-                kv.key = user_key.as_u64;
-                if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
-                  {
-                    u = pool_elt_at_index (tsm->users, value.value);
-                    u->nsessions--;
-                  }
-              }
-          }));
-
-          vec_foreach (ses_index, ses_to_be_removed)
-            pool_put_index (tsm->sessions, ses_index[0]);
-
-          vec_free (ses_to_be_removed);
-       }
-    }
-
-  vec_del1 (sm->addresses, i);
-
-  /* Delete external address from FIB */
-  pool_foreach (interface, sm->interfaces,
-  ({
-    if (interface->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
-    break;
-  }));
-  pool_foreach (interface, sm->output_feature_interfaces,
-  ({
-    if (interface->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(&addr, 32, interface->sw_if_index, 0);
-    break;
-  }));
-
-  return 0;
-}
-
-int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
-{
-  snat_main_t *sm = &snat_main;
-  snat_interface_t *i;
-  const char * feature_name;
-  snat_address_t * ap;
-  snat_static_mapping_t * m;
-  snat_det_map_t * dm;
-
-  if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
-    feature_name = is_inside ?  "snat-in2out-fast" : "snat-out2in-fast";
-  else
-    {
-      if (sm->num_workers > 1 && !sm->deterministic)
-        feature_name = is_inside ?  "snat-in2out-worker-handoff" : "snat-out2in-worker-handoff";
-      else if (sm->deterministic)
-        feature_name = is_inside ?  "snat-det-in2out" : "snat-det-out2in";
-      else
-        feature_name = is_inside ?  "snat-in2out" : "snat-out2in";
-    }
-
-  vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index,
-			       !is_del, 0, 0);
-
-  if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
-    sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index, 0);
-
-  if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
-    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
-
-  pool_foreach (i, sm->interfaces,
-  ({
-    if (i->sw_if_index == sw_if_index)
-      {
-        if (is_del)
-          pool_put (sm->interfaces, i);
-        else
-          return VNET_API_ERROR_VALUE_EXIST;
-
-        goto fib;
-      }
-  }));
-
-  if (is_del)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  pool_get (sm->interfaces, i);
-  i->sw_if_index = sw_if_index;
-  i->is_inside = is_inside;
-
-  /* Add/delete external addresses to FIB */
-fib:
-  if (is_inside)
-    return 0;
-
-  vec_foreach (ap, sm->addresses)
-    snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
-
-  pool_foreach (m, sm->static_mappings,
-  ({
-    if (!(m->addr_only))
-      continue;
-
-    snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
-  }));
-
-  pool_foreach (dm, sm->det_maps,
-  ({
-    snat_add_del_addr_to_fib(&dm->out_addr, dm->out_plen, sw_if_index, !is_del);
-  }));
-
-  return 0;
-}
-
-int snat_interface_add_del_output_feature (u32 sw_if_index,
-                                           u8 is_inside,
-                                           int is_del)
-{
-  snat_main_t *sm = &snat_main;
-  snat_interface_t *i;
-  snat_address_t * ap;
-  snat_static_mapping_t * m;
-
-  if (sm->deterministic ||
-      (sm->static_mapping_only && !(sm->static_mapping_connection_tracking)))
-    return VNET_API_ERROR_UNSUPPORTED;
-
-  if (is_inside)
-    {
-      vnet_feature_enable_disable ("ip4-unicast", "snat-hairpin-dst",
-                                   sw_if_index, !is_del, 0, 0);
-      vnet_feature_enable_disable ("ip4-output", "snat-hairpin-src",
-                                   sw_if_index, !is_del, 0, 0);
-      goto fq;
-    }
-
-  if (sm->num_workers > 1)
-    {
-      vnet_feature_enable_disable ("ip4-unicast", "snat-out2in-worker-handoff",
-                                   sw_if_index, !is_del, 0, 0);
-      vnet_feature_enable_disable ("ip4-output",
-                                   "snat-in2out-output-worker-handoff",
-                                   sw_if_index, !is_del, 0, 0);
-    }
-  else
-    {
-      vnet_feature_enable_disable ("ip4-unicast", "snat-out2in", sw_if_index,
-                                   !is_del, 0, 0);
-      vnet_feature_enable_disable ("ip4-output", "snat-in2out-output",
-                                   sw_if_index, !is_del, 0, 0);
-    }
-
-fq:
-  if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
-    sm->fq_in2out_output_index =
-      vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
-
-  if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
-    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index, 0);
-
-  pool_foreach (i, sm->output_feature_interfaces,
-  ({
-    if (i->sw_if_index == sw_if_index)
-      {
-        if (is_del)
-          pool_put (sm->output_feature_interfaces, i);
-        else
-          return VNET_API_ERROR_VALUE_EXIST;
-
-        goto fib;
-      }
-  }));
-
-  if (is_del)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  pool_get (sm->output_feature_interfaces, i);
-  i->sw_if_index = sw_if_index;
-  i->is_inside = is_inside;
-
-  /* Add/delete external addresses to FIB */
-fib:
-  if (is_inside)
-    return 0;
-
-  vec_foreach (ap, sm->addresses)
-    snat_add_del_addr_to_fib(&ap->addr, 32, sw_if_index, !is_del);
-
-  pool_foreach (m, sm->static_mappings,
-  ({
-    if (!(m->addr_only))
-      continue;
-
-    snat_add_del_addr_to_fib(&m->external_addr, 32, sw_if_index, !is_del);
-  }));
-
-  return 0;
-}
-
-int snat_set_workers (uword * bitmap)
-{
-  snat_main_t *sm = &snat_main;
-  int i, j = 0;
-
-  if (sm->num_workers < 2)
-    return VNET_API_ERROR_FEATURE_DISABLED;
-
-  if (clib_bitmap_last_set (bitmap) >= sm->num_workers)
-    return VNET_API_ERROR_INVALID_WORKER;
-
-  vec_free (sm->workers);
-  clib_bitmap_foreach (i, bitmap,
-    ({
-      vec_add1(sm->workers, i);
-      sm->per_thread_data[i].snat_thread_index = j;
-      j++;
-    }));
-
-  sm->port_per_thread = (0xffff - 1024) / _vec_len (sm->workers);
-  sm->num_snat_thread = _vec_len (sm->workers);
-
-  return 0;
-}
-
-
-static void
-snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
-                                       uword opaque,
-                                       u32 sw_if_index,
-                                       ip4_address_t * address,
-                                       u32 address_length,
-                                       u32 if_address_index,
-                                       u32 is_delete);
-
-static clib_error_t * snat_init (vlib_main_t * vm)
-{
-  snat_main_t * sm = &snat_main;
-  clib_error_t * error = 0;
-  ip4_main_t * im = &ip4_main;
-  ip_lookup_main_t * lm = &im->lookup_main;
-  uword *p;
-  vlib_thread_registration_t *tr;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  uword *bitmap = 0;
-  u32 i;
-  ip4_add_del_interface_address_callback_t cb4;
-
-  sm->vlib_main = vm;
-  sm->vnet_main = vnet_get_main();
-  sm->ip4_main = im;
-  sm->ip4_lookup_main = lm;
-  sm->api_main = &api_main;
-  sm->first_worker_index = 0;
-  sm->next_worker = 0;
-  sm->num_workers = 0;
-  sm->num_snat_thread = 1;
-  sm->workers = 0;
-  sm->port_per_thread = 0xffff - 1024;
-  sm->fq_in2out_index = ~0;
-  sm->fq_out2in_index = ~0;
-  sm->udp_timeout = SNAT_UDP_TIMEOUT;
-  sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
-  sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
-  sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
-
-  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-  if (p)
-    {
-      tr = (vlib_thread_registration_t *) p[0];
-      if (tr)
-        {
-          sm->num_workers = tr->count;
-          sm->first_worker_index = tr->first_index;
-        }
-    }
-
-  vec_validate (sm->per_thread_data, tm->n_vlib_mains - 1);
-
-  /* Use all available workers by default */
-  if (sm->num_workers > 1)
-    {
-      for (i=0; i < sm->num_workers; i++)
-        bitmap = clib_bitmap_set (bitmap, i, 1);
-      snat_set_workers(bitmap);
-      clib_bitmap_free (bitmap);
-    }
-  else
-    {
-      sm->per_thread_data[0].snat_thread_index = 0;
-    }
-
-  error = snat_api_init(vm, sm);
-  if (error)
-    return error;
-
-  /* Set up the interface address add/del callback */
-  cb4.function = snat_ip4_add_del_interface_address_cb;
-  cb4.function_opaque = 0;
-
-  vec_add1 (im->add_del_interface_address_callbacks, cb4);
-
-  /* Init IPFIX logging */
-  snat_ipfix_logging_init(vm);
-
-  error = nat64_init(vm);
-
-  return error;
-}
-
-VLIB_INIT_FUNCTION (snat_init);
-
-void snat_free_outside_address_and_port (snat_main_t * sm, 
-                                         snat_session_key_t * k, 
-                                         u32 address_index)
-{
-  snat_address_t *a;
-  u16 port_host_byte_order = clib_net_to_host_u16 (k->port);
-  
-  ASSERT (address_index < vec_len (sm->addresses));
-
-  a = sm->addresses + address_index;
-
-  switch (k->protocol)
-    {
-#define _(N, i, n, s) \
-    case SNAT_PROTOCOL_##N: \
-      ASSERT (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, \
-        port_host_byte_order) == 1); \
-      clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, \
-        port_host_byte_order, 0); \
-      a->busy_##n##_ports--; \
-      break;
-      foreach_snat_protocol
-#undef _
-    default:
-      clib_warning("unknown_protocol");
-      return;
-    }
-}  
-
-/**
- * @brief Match SNAT static mapping.
- *
- * @param sm          SNAT main.
- * @param match       Address and port to match.
- * @param mapping     External or local address and port of the matched mapping.
- * @param by_external If 0 match by local address otherwise match by external
- *                    address.
- * @param is_addr_only If matched mapping is address only
- *
- * @returns 0 if match found otherwise 1.
- */
-int snat_static_mapping_match (snat_main_t * sm,
-                               snat_session_key_t match,
-                               snat_session_key_t * mapping,
-                               u8 by_external,
-                               u8 *is_addr_only)
-{
-  clib_bihash_kv_8_8_t kv, value;
-  snat_static_mapping_t *m;
-  snat_session_key_t m_key;
-  clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
-
-  if (by_external)
-    mapping_hash = &sm->static_mapping_by_external;
-
-  m_key.addr = match.addr;
-  m_key.port = clib_net_to_host_u16 (match.port);
-  m_key.protocol = match.protocol;
-  m_key.fib_index = match.fib_index;
-
-  kv.key = m_key.as_u64;
-
-  if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
-    {
-      /* Try address only mapping */
-      m_key.port = 0;
-      m_key.protocol = 0;
-      kv.key = m_key.as_u64;
-      if (clib_bihash_search_8_8 (mapping_hash, &kv, &value))
-        return 1;
-    }
-
-  m = pool_elt_at_index (sm->static_mappings, value.value);
-
-  if (by_external)
-    {
-      mapping->addr = m->local_addr;
-      /* Address only mapping doesn't change port */
-      mapping->port = m->addr_only ? match.port
-        : clib_host_to_net_u16 (m->local_port);
-      mapping->fib_index = m->fib_index;
-    }
-  else
-    {
-      mapping->addr = m->external_addr;
-      /* Address only mapping doesn't change port */
-      mapping->port = m->addr_only ? match.port
-        : clib_host_to_net_u16 (m->external_port);
-      mapping->fib_index = sm->outside_fib_index;
-    }
-
-  if (PREDICT_FALSE(is_addr_only != 0))
-    *is_addr_only = m->addr_only;
-
-  return 0;
-}
-
-static_always_inline u16
-snat_random_port (snat_main_t * sm, u16 min, u16 max)
-{
-  return min + random_u32 (&sm->random_seed) /
-    (random_u32_max() / (max - min + 1) + 1);
-}
-
-int snat_alloc_outside_address_and_port (snat_main_t * sm,
-                                         u32 fib_index,
-                                         u32 thread_index,
-                                         snat_session_key_t * k,
-                                         u32 * address_indexp)
-{
-  int i;
-  snat_address_t *a;
-  u32 portnum;
-
-  for (i = 0; i < vec_len (sm->addresses); i++)
-    {
-      a = sm->addresses + i;
-      if (sm->vrf_mode && a->fib_index != ~0 && a->fib_index != fib_index)
-        continue;
-      switch (k->protocol)
-        {
-#define _(N, j, n, s) \
-        case SNAT_PROTOCOL_##N: \
-          if (a->busy_##n##_ports < (sm->port_per_thread * sm->num_snat_thread)) \
-            { \
-              while (1) \
-                { \
-                  portnum = (sm->port_per_thread * \
-                    sm->per_thread_data[thread_index].snat_thread_index) + \
-                    snat_random_port(sm, 0, sm->port_per_thread) + 1024; \
-                  if (clib_bitmap_get_no_check (a->busy_##n##_port_bitmap, portnum)) \
-                    continue; \
-                  clib_bitmap_set_no_check (a->busy_##n##_port_bitmap, portnum, 1); \
-                  a->busy_##n##_ports++; \
-                  k->addr = a->addr; \
-                  k->port = clib_host_to_net_u16(portnum); \
-                  *address_indexp = i; \
-                  return 0; \
-                } \
-            } \
-          break;
-          foreach_snat_protocol
-#undef _
-        default:
-          clib_warning("unknown protocol");
-          return 1;
-        }
-
-    }
-  /* Totally out of translations to use... */
-  snat_ipfix_logging_addresses_exhausted(0);
-  return 1;
-}
-
-
-static clib_error_t *
-add_address_command_fn (vlib_main_t * vm,
-                        unformat_input_t * input,
-                        vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  snat_main_t * sm = &snat_main;
-  ip4_address_t start_addr, end_addr, this_addr;
-  u32 start_host_order, end_host_order;
-  u32 vrf_id = ~0;
-  int i, count;
-  int is_add = 1;
-  int rv = 0;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U - %U",
-                    unformat_ip4_address, &start_addr,
-                    unformat_ip4_address, &end_addr))
-        ;
-      else if (unformat (line_input, "tenant-vrf %u", &vrf_id))
-        ;
-      else if (unformat (line_input, "%U", unformat_ip4_address, &start_addr))
-        end_addr = start_addr;
-      else if (unformat (line_input, "del"))
-        is_add = 0;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-            format_unformat_error, line_input);
-          goto done;
-        }
-     }
-
-  if (sm->static_mapping_only)
-    {
-      error = clib_error_return (0, "static mapping only mode");
-      goto done;
-    }
-
-  start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
-  end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
-  
-  if (end_host_order < start_host_order)
-    {
-      error = clib_error_return (0, "end address less than start address");
-      goto done;
-    }
-
-  count = (end_host_order - start_host_order) + 1;
-
-  if (count > 1024)
-    clib_warning ("%U - %U, %d addresses...",
-                  format_ip4_address, &start_addr,
-                  format_ip4_address, &end_addr,
-                  count);
-  
-  this_addr = start_addr;
-
-  for (i = 0; i < count; i++)
-    {
-      if (is_add)
-        snat_add_address (sm, &this_addr, vrf_id);
-      else
-        rv = snat_del_address (sm, this_addr, 0);
-
-      switch (rv)
-        {
-        case VNET_API_ERROR_NO_SUCH_ENTRY:
-          error = clib_error_return (0, "S-NAT address not exist.");
-          goto done;
-        case VNET_API_ERROR_UNSPECIFIED:
-          error = clib_error_return (0, "S-NAT address used in static mapping.");
-          goto done;
-        default:
-          break;
-        }
-
-      increment_v4_address (&this_addr);
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-VLIB_CLI_COMMAND (add_address_command, static) = {
-  .path = "snat add address",
-  .short_help = "snat add addresses  [- ] "
-                "[tenant-vrf ] [del]",
-  .function = add_address_command_fn,
-};
-
-static clib_error_t *
-snat_feature_command_fn (vlib_main_t * vm,
-                          unformat_input_t * input,
-                          vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  vnet_main_t * vnm = vnet_get_main();
-  clib_error_t * error = 0;
-  u32 sw_if_index;
-  u32 * inside_sw_if_indices = 0;
-  u32 * outside_sw_if_indices = 0;
-  u8 is_output_feature = 0;
-  int is_del = 0;
-  int i;
-
-  sw_if_index = ~0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "in %U", unformat_vnet_sw_interface,
-                    vnm, &sw_if_index))
-        vec_add1 (inside_sw_if_indices, sw_if_index);
-      else if (unformat (line_input, "out %U", unformat_vnet_sw_interface,
-                         vnm, &sw_if_index))
-        vec_add1 (outside_sw_if_indices, sw_if_index);
-      else if (unformat (line_input, "output-feature"))
-        is_output_feature = 1;
-      else if (unformat (line_input, "del"))
-        is_del = 1;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-            format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  if (vec_len (inside_sw_if_indices))
-    {
-      for (i = 0; i < vec_len(inside_sw_if_indices); i++)
-        {
-          sw_if_index = inside_sw_if_indices[i];
-          if (is_output_feature)
-            {
-              if (snat_interface_add_del_output_feature (sw_if_index, 1, is_del))
-                {
-                  error = clib_error_return (0, "%s %U failed",
-                                             is_del ? "del" : "add",
-                                             format_vnet_sw_interface_name, vnm,
-                                             vnet_get_sw_interface (vnm,
-                                                                    sw_if_index));
-                  goto done;
-                }
-            }
-          else
-            {
-              if (snat_interface_add_del (sw_if_index, 1, is_del))
-                {
-                  error = clib_error_return (0, "%s %U failed",
-                                             is_del ? "del" : "add",
-                                             format_vnet_sw_interface_name, vnm,
-                                             vnet_get_sw_interface (vnm,
-                                                                    sw_if_index));
-                  goto done;
-                }
-            }
-        }
-    }
-
-  if (vec_len (outside_sw_if_indices))
-    {
-      for (i = 0; i < vec_len(outside_sw_if_indices); i++)
-        {
-          sw_if_index = outside_sw_if_indices[i];
-          if (is_output_feature)
-            {
-              if (snat_interface_add_del_output_feature (sw_if_index, 0, is_del))
-                {
-                  error = clib_error_return (0, "%s %U failed",
-                                             is_del ? "del" : "add",
-                                             format_vnet_sw_interface_name, vnm,
-                                             vnet_get_sw_interface (vnm,
-                                                                    sw_if_index));
-                  goto done;
-                }
-            }
-          else
-            {
-              if (snat_interface_add_del (sw_if_index, 0, is_del))
-                {
-                  error = clib_error_return (0, "%s %U failed",
-                                             is_del ? "del" : "add",
-                                             format_vnet_sw_interface_name, vnm,
-                                             vnet_get_sw_interface (vnm,
-                                                                    sw_if_index));
-                  goto done;
-                }
-            }
-        }
-    }
-
-done:
-  unformat_free (line_input);
-  vec_free (inside_sw_if_indices);
-  vec_free (outside_sw_if_indices);
-
-  return error;
-}
-
-VLIB_CLI_COMMAND (set_interface_snat_command, static) = {
-  .path = "set interface snat",
-  .function = snat_feature_command_fn,
-  .short_help = "set interface snat in  out  [output-feature] "
-                "[del]",
-};
-
-uword
-unformat_snat_protocol (unformat_input_t * input, va_list * args)
-{
-  u32 *r = va_arg (*args, u32 *);
-
-  if (0);
-#define _(N, i, n, s) else if (unformat (input, s)) *r = SNAT_PROTOCOL_##N;
-  foreach_snat_protocol
-#undef _
-  else
-    return 0;
-  return 1;
-}
-
-u8 *
-format_snat_protocol (u8 * s, va_list * args)
-{
-  u32 i = va_arg (*args, u32);
-  u8 *t = 0;
-
-  switch (i)
-    {
-#define _(N, j, n, str) case SNAT_PROTOCOL_##N: t = (u8 *) str; break;
-      foreach_snat_protocol
-#undef _
-    default:
-      s = format (s, "unknown");
-      return s;
-    }
-  s = format (s, "%s", t);
-  return s;
-}
-
-static clib_error_t *
-add_static_mapping_command_fn (vlib_main_t * vm,
-                               unformat_input_t * input,
-                               vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t * error = 0;
-  ip4_address_t l_addr, e_addr;
-  u32 l_port = 0, e_port = 0, vrf_id = ~0;
-  int is_add = 1;
-  int addr_only = 1;
-  u32 sw_if_index = ~0;
-  vnet_main_t * vnm = vnet_get_main();
-  int rv;
-  snat_protocol_t proto;
-  u8 proto_set = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "local %U %u", unformat_ip4_address, &l_addr,
-                    &l_port))
-        addr_only = 0;
-      else if (unformat (line_input, "local %U", unformat_ip4_address, &l_addr))
-        ;
-      else if (unformat (line_input, "external %U %u", unformat_ip4_address,
-                         &e_addr, &e_port))
-        addr_only = 0;
-      else if (unformat (line_input, "external %U", unformat_ip4_address,
-                         &e_addr))
-        ;
-      else if (unformat (line_input, "external %U %u",
-                         unformat_vnet_sw_interface, vnm, &sw_if_index,
-                         &e_port))
-        addr_only = 0;
-
-      else if (unformat (line_input, "external %U",
-                         unformat_vnet_sw_interface, vnm, &sw_if_index))
-        ;
-      else if (unformat (line_input, "vrf %u", &vrf_id))
-        ;
-      else if (unformat (line_input, "%U", unformat_snat_protocol, &proto))
-        proto_set = 1;
-      else if (unformat (line_input, "del"))
-        is_add = 0;
-      else
-        {
-          error = clib_error_return (0, "unknown input: '%U'",
-            format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  if (!addr_only && !proto_set)
-    {
-      error = clib_error_return (0, "missing protocol");
-      goto done;
-    }
-
-  rv = snat_add_static_mapping(l_addr, e_addr, (u16) l_port, (u16) e_port,
-                               vrf_id, addr_only, sw_if_index, proto, is_add);
-
-  switch (rv)
-    {
-    case VNET_API_ERROR_INVALID_VALUE:
-      error = clib_error_return (0, "External port already in use.");
-      goto done;
-    case VNET_API_ERROR_NO_SUCH_ENTRY:
-      if (is_add)
-        error = clib_error_return (0, "External addres must be allocated.");
-      else
-        error = clib_error_return (0, "Mapping not exist.");
-      goto done;
-    case VNET_API_ERROR_NO_SUCH_FIB:
-      error = clib_error_return (0, "No such VRF id.");
-      goto done;
-    case VNET_API_ERROR_VALUE_EXIST:
-      error = clib_error_return (0, "Mapping already exist.");
-      goto done;
-    default:
-      break;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat add static mapping}
- * Static mapping allows hosts on the external network to initiate connection
- * to to the local network host.
- * To create static mapping between local host address 10.0.0.3 port 6303 and
- * external address 4.4.4.4 port 3606 for TCP protocol use:
- *  vpp# snat add static mapping local tcp 10.0.0.3 6303 external 4.4.4.4 3606
- * If not runnig "static mapping only" S-NAT plugin mode use before:
- *  vpp# snat add address 4.4.4.4
- * To create static mapping between local and external address use:
- *  vpp# snat add static mapping local 10.0.0.3 external 4.4.4.4
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (add_static_mapping_command, static) = {
-  .path = "snat add static mapping",
-  .function = add_static_mapping_command_fn,
-  .short_help =
-    "snat add static mapping local tcp|udp|icmp  [] external  [] [vrf ] [del]",
-};
-
-static clib_error_t *
-set_workers_command_fn (vlib_main_t * vm,
-                        unformat_input_t * input,
-                        vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  uword *bitmap = 0;
-  int rv = 0;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U", unformat_bitmap_list, &bitmap))
-        ;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-            format_unformat_error, line_input);
-          goto done;
-        }
-     }
-
-  if (bitmap == 0)
-    {
-      error = clib_error_return (0, "List of workers must be specified.");
-      goto done;
-    }
-
-  rv = snat_set_workers(bitmap);
-
-  clib_bitmap_free (bitmap);
-
-  switch (rv)
-    {
-    case VNET_API_ERROR_INVALID_WORKER:
-      error = clib_error_return (0, "Invalid worker(s).");
-      goto done;
-    case VNET_API_ERROR_FEATURE_DISABLED:
-      error = clib_error_return (0,
-        "Supported only if 2 or more workes available.");
-      goto done;
-    default:
-      break;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{set snat workers}
- * Set SNAT workers if 2 or more workers available, use:
- *  vpp# set snat workers 0-2,5
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (set_workers_command, static) = {
-  .path = "set snat workers",
-  .function = set_workers_command_fn,
-  .short_help =
-    "set snat workers ",
-};
-
-static clib_error_t *
-snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm,
-                                              unformat_input_t * input,
-                                              vlib_cli_command_t * cmd)
-{
-  unformat_input_t _line_input, *line_input = &_line_input;
-  u32 domain_id = 0;
-  u32 src_port = 0;
-  u8 enable = 1;
-  int rv = 0;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "domain %d", &domain_id))
-        ;
-      else if (unformat (line_input, "src-port %d", &src_port))
-        ;
-      else if (unformat (line_input, "disable"))
-        enable = 0;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-            format_unformat_error, line_input);
-          goto done;
-        }
-     }
-
-  rv = snat_ipfix_logging_enable_disable (enable, domain_id, (u16) src_port);
-
-  if (rv)
-    {
-      error = clib_error_return (0, "ipfix logging enable failed");
-      goto done;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat ipfix logging}
- * To enable SNAT IPFIX logging use:
- *  vpp# snat ipfix logging
- * To set IPFIX exporter use:
- *  vpp# set ipfix exporter collector 10.10.10.3 src 10.10.10.1
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (snat_ipfix_logging_enable_disable_command, static) = {
-  .path = "snat ipfix logging",
-  .function = snat_ipfix_logging_enable_disable_command_fn,
-  .short_help = "snat ipfix logging [domain ] [src-port ] [disable]",
-};
-
-static u32
-snat_get_worker_in2out_cb (ip4_header_t * ip0, u32 rx_fib_index0)
-{
-  snat_main_t *sm = &snat_main;
-  snat_user_key_t key0;
-  clib_bihash_kv_8_8_t kv0, value0;
-  u32 next_worker_index = 0;
-
-  key0.addr = ip0->src_address;
-  key0.fib_index = rx_fib_index0;
-
-  kv0.key = key0.as_u64;
-
-  /* Ever heard of of the "user" before? */
-  if (clib_bihash_search_8_8 (&sm->worker_by_in, &kv0, &value0))
-    {
-      /* No, assign next available worker (RR) */
-      next_worker_index = sm->first_worker_index;
-      if (vec_len (sm->workers))
-        {
-          next_worker_index +=
-            sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
-        }
-
-      /* add non-traslated packets worker lookup */
-      kv0.value = next_worker_index;
-      clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
-    }
-  else
-    next_worker_index = value0.value;
-
-  return next_worker_index;
-}
-
-static u32
-snat_get_worker_out2in_cb (ip4_header_t * ip0, u32 rx_fib_index0)
-{
-  snat_main_t *sm = &snat_main;
-  snat_worker_key_t key0;
-  clib_bihash_kv_8_8_t kv0, value0;
-  udp_header_t * udp0;
-  u32 next_worker_index = 0;
-
-  udp0 = ip4_next_header (ip0);
-
-  key0.addr = ip0->dst_address;
-  key0.port = udp0->dst_port;
-  key0.fib_index = rx_fib_index0;
-
-  if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_ICMP))
-    {
-      icmp46_header_t * icmp0 = (icmp46_header_t *) udp0;
-      icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
-      key0.port = echo0->identifier;
-    }
-
-  kv0.key = key0.as_u64;
-
-  /* Ever heard of of the "user" before? */
-  if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
-    {
-      key0.port = 0;
-      kv0.key = key0.as_u64;
-
-      if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
-        {
-          /* No, assign next available worker (RR) */
-          next_worker_index = sm->first_worker_index;
-          if (vec_len (sm->workers))
-            {
-              next_worker_index +=
-                sm->workers[sm->next_worker++ % _vec_len (sm->workers)];
-            }
-        }
-      else
-        {
-          /* Static mapping without port */
-          next_worker_index = value0.value;
-        }
-
-      /* Add to translated packets worker lookup */
-      kv0.value = next_worker_index;
-      clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
-    }
-  else
-    next_worker_index = value0.value;
-
-  return next_worker_index;
-}
-
-static clib_error_t *
-snat_config (vlib_main_t * vm, unformat_input_t * input)
-{
-  snat_main_t * sm = &snat_main;
-  u32 translation_buckets = 1024;
-  u32 translation_memory_size = 128<<20;
-  u32 user_buckets = 128;
-  u32 user_memory_size = 64<<20;
-  u32 max_translations_per_user = 100;
-  u32 outside_vrf_id = 0;
-  u32 inside_vrf_id = 0;
-  u32 static_mapping_buckets = 1024;
-  u32 static_mapping_memory_size = 64<<20;
-  u8 static_mapping_only = 0;
-  u8 static_mapping_connection_tracking = 0;
-
-  sm->deterministic = 0;
-
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (input, "translation hash buckets %d", &translation_buckets))
-        ;
-      else if (unformat (input, "translation hash memory %d",
-                         &translation_memory_size));
-      else if (unformat (input, "user hash buckets %d", &user_buckets))
-        ;
-      else if (unformat (input, "user hash memory %d",
-                         &user_memory_size))
-        ;
-      else if (unformat (input, "max translations per user %d",
-                         &max_translations_per_user))
-        ;
-      else if (unformat (input, "outside VRF id %d",
-                         &outside_vrf_id))
-        ;
-      else if (unformat (input, "inside VRF id %d",
-                         &inside_vrf_id))
-        ;
-      else if (unformat (input, "static mapping only"))
-        {
-          static_mapping_only = 1;
-          if (unformat (input, "connection tracking"))
-            static_mapping_connection_tracking = 1;
-        }
-      else if (unformat (input, "deterministic"))
-        sm->deterministic = 1;
-      else
-	return clib_error_return (0, "unknown input '%U'",
-				  format_unformat_error, input);
-    }
-
-  /* for show commands, etc. */
-  sm->translation_buckets = translation_buckets;
-  sm->translation_memory_size = translation_memory_size;
-  sm->user_buckets = user_buckets;
-  sm->user_memory_size = user_memory_size;
-  sm->max_translations_per_user = max_translations_per_user;
-  sm->outside_vrf_id = outside_vrf_id;
-  sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-                                                             outside_vrf_id);
-  sm->inside_vrf_id = inside_vrf_id;
-  sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-                                                            inside_vrf_id);
-  sm->static_mapping_only = static_mapping_only;
-  sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
-
-  if (sm->deterministic)
-    {
-      sm->in2out_node_index = snat_det_in2out_node.index;
-      sm->in2out_output_node_index = ~0;
-      sm->out2in_node_index = snat_det_out2in_node.index;
-      sm->icmp_match_in2out_cb = icmp_match_in2out_det;
-      sm->icmp_match_out2in_cb = icmp_match_out2in_det;
-    }
-  else
-    {
-      sm->worker_in2out_cb = snat_get_worker_in2out_cb;
-      sm->worker_out2in_cb = snat_get_worker_out2in_cb;
-      sm->in2out_node_index = snat_in2out_node.index;
-      sm->in2out_output_node_index = snat_in2out_output_node.index;
-      sm->out2in_node_index = snat_out2in_node.index;
-      if (!static_mapping_only ||
-          (static_mapping_only && static_mapping_connection_tracking))
-        {
-          sm->icmp_match_in2out_cb = icmp_match_in2out_slow;
-          sm->icmp_match_out2in_cb = icmp_match_out2in_slow;
-
-          clib_bihash_init_8_8 (&sm->worker_by_in, "worker-by-in", user_buckets,
-                                user_memory_size);
-
-          clib_bihash_init_8_8 (&sm->worker_by_out, "worker-by-out", user_buckets,
-                                user_memory_size);
-
-          clib_bihash_init_8_8 (&sm->in2out, "in2out", translation_buckets,
-                                translation_memory_size);
-
-          clib_bihash_init_8_8 (&sm->out2in, "out2in", translation_buckets,
-                                translation_memory_size);
-
-          clib_bihash_init_8_8 (&sm->user_hash, "users", user_buckets,
-                                user_memory_size);
-
-          clib_bihash_init_16_8 (&sm->in2out_unk_proto, "in2out-unk-proto",
-                                 translation_buckets, translation_memory_size);
-
-          clib_bihash_init_16_8 (&sm->out2in_unk_proto, "out2in-unk-proto",
-                                 translation_buckets, translation_memory_size);
-        }
-      else
-        {
-          sm->icmp_match_in2out_cb = icmp_match_in2out_fast;
-          sm->icmp_match_out2in_cb = icmp_match_out2in_fast;
-        }
-      clib_bihash_init_8_8 (&sm->static_mapping_by_local,
-                            "static_mapping_by_local", static_mapping_buckets,
-                            static_mapping_memory_size);
-
-      clib_bihash_init_8_8 (&sm->static_mapping_by_external,
-                            "static_mapping_by_external", static_mapping_buckets,
-                            static_mapping_memory_size);
-    }
-
-  return 0;
-}
-
-VLIB_CONFIG_FUNCTION (snat_config, "snat");
-
-u8 * format_snat_session_state (u8 * s, va_list * args)
-{
-  u32 i = va_arg (*args, u32);
-  u8 *t = 0;
-
-  switch (i)
-    {
-#define _(v, N, str) case SNAT_SESSION_##N: t = (u8 *) str; break;
-    foreach_snat_session_state
-#undef _
-    default:
-      t = format (t, "unknown");
-    }
-  s = format (s, "%s", t);
-  return s;
-}
-
-u8 * format_snat_key (u8 * s, va_list * args)
-{
-  snat_session_key_t * key = va_arg (*args, snat_session_key_t *);
-  char * protocol_string = "unknown";
-  static char *protocol_strings[] = {
-      "UDP",
-      "TCP",
-      "ICMP",
-  };
-
-  if (key->protocol < ARRAY_LEN(protocol_strings))
-      protocol_string = protocol_strings[key->protocol];
-
-  s = format (s, "%U proto %s port %d fib %d",
-              format_ip4_address, &key->addr, protocol_string,
-              clib_net_to_host_u16 (key->port), key->fib_index);
-  return s;
-}
-
-u8 * format_snat_session (u8 * s, va_list * args)
-{
-  snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *);
-  snat_session_t * sess = va_arg (*args, snat_session_t *);
-
-  if (snat_is_unk_proto_session (sess))
-    {
-      s = format (s, "  i2o %U proto %u fib %u\n",
-                  format_ip4_address, &sess->in2out.addr, sess->in2out.port,
-                  sess->in2out.fib_index);
-      s = format (s, "    o2i %U proto %u fib %u\n",
-                  format_ip4_address, &sess->out2in.addr, sess->out2in.port,
-                  sess->out2in.fib_index);
-    }
-  else
-    {
-      s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
-      s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
-    }
-  s = format (s, "       last heard %.2f\n", sess->last_heard);
-  s = format (s, "       total pkts %d, total bytes %lld\n",
-              sess->total_pkts, sess->total_bytes);
-  if (snat_is_session_static (sess))
-    s = format (s, "       static translation\n");
-  else
-    s = format (s, "       dynamic translation\n");
-
-  return s;
-}
-
-u8 * format_snat_user (u8 * s, va_list * args)
-{
-  snat_main_per_thread_data_t * sm = va_arg (*args, snat_main_per_thread_data_t *);
-  snat_user_t * u = va_arg (*args, snat_user_t *);
-  int verbose = va_arg (*args, int);
-  dlist_elt_t * head, * elt;
-  u32 elt_index, head_index;
-  u32 session_index;
-  snat_session_t * sess;
-
-  s = format (s, "%U: %d dynamic translations, %d static translations\n",
-              format_ip4_address, &u->addr, u->nsessions, u->nstaticsessions);
-
-  if (verbose == 0)
-    return s;
-
-  if (u->nsessions || u->nstaticsessions)
-    {
-      head_index = u->sessions_per_user_list_head_index;
-      head = pool_elt_at_index (sm->list_pool, head_index);
-
-      elt_index = head->next;
-      elt = pool_elt_at_index (sm->list_pool, elt_index);
-      session_index = elt->value;
-
-      while (session_index != ~0)
-        {
-          sess = pool_elt_at_index (sm->sessions, session_index);
-
-          s = format (s, "  %U\n", format_snat_session, sm, sess);
-
-          elt_index = elt->next;
-          elt = pool_elt_at_index (sm->list_pool, elt_index);
-          session_index = elt->value;
-        }
-    }
-
-  return s;
-}
-
-u8 * format_snat_static_mapping (u8 * s, va_list * args)
-{
-  snat_static_mapping_t *m = va_arg (*args, snat_static_mapping_t *);
-
-  if (m->addr_only)
-      s = format (s, "local %U external %U vrf %d",
-                  format_ip4_address, &m->local_addr,
-                  format_ip4_address, &m->external_addr,
-                  m->vrf_id);
-  else
-      s = format (s, "%U local %U:%d external %U:%d vrf %d",
-                  format_snat_protocol, m->proto,
-                  format_ip4_address, &m->local_addr, m->local_port,
-                  format_ip4_address, &m->external_addr, m->external_port,
-                  m->vrf_id);
-
-  return s;
-}
-
-u8 * format_snat_static_map_to_resolve (u8 * s, va_list * args)
-{
-  snat_static_map_resolve_t *m = va_arg (*args, snat_static_map_resolve_t *);
-  vnet_main_t *vnm = vnet_get_main();
-
-  if (m->addr_only)
-      s = format (s, "local %U external %U vrf %d",
-                  format_ip4_address, &m->l_addr,
-                  format_vnet_sw_interface_name, vnm,
-                  vnet_get_sw_interface (vnm, m->sw_if_index),
-                  m->vrf_id);
-  else
-      s = format (s, "%U local %U:%d external %U:%d vrf %d",
-                  format_snat_protocol, m->proto,
-                  format_ip4_address, &m->l_addr, m->l_port,
-                  format_vnet_sw_interface_name, vnm,
-                  vnet_get_sw_interface (vnm, m->sw_if_index), m->e_port,
-                  m->vrf_id);
-
-  return s;
-}
-
-u8 * format_det_map_ses (u8 * s, va_list * args)
-{
-  snat_det_map_t * det_map = va_arg (*args, snat_det_map_t *);
-  ip4_address_t in_addr, out_addr;
-  u32 in_offset, out_offset;
-  snat_det_session_t * ses = va_arg (*args, snat_det_session_t *);
-  u32 * i = va_arg (*args, u32 *);
-
-  u32 user_index = *i / SNAT_DET_SES_PER_USER;
-  in_addr.as_u32 = clib_host_to_net_u32 (
-    clib_net_to_host_u32(det_map->in_addr.as_u32) + user_index);
-  in_offset = clib_net_to_host_u32(in_addr.as_u32) -
-    clib_net_to_host_u32(det_map->in_addr.as_u32);
-  out_offset = in_offset / det_map->sharing_ratio;
-  out_addr.as_u32 = clib_host_to_net_u32(
-    clib_net_to_host_u32(det_map->out_addr.as_u32) + out_offset);
-  s = format (s, "in %U:%d out %U:%d external host %U:%d state: %U expire: %d\n",
-              format_ip4_address, &in_addr,
-              clib_net_to_host_u16 (ses->in_port),
-              format_ip4_address, &out_addr,
-              clib_net_to_host_u16 (ses->out.out_port),
-              format_ip4_address, &ses->out.ext_host_addr,
-              clib_net_to_host_u16 (ses->out.ext_host_port),
-              format_snat_session_state, ses->state,
-              ses->expire);
-
-  return s;
-}
-
-static clib_error_t *
-show_snat_command_fn (vlib_main_t * vm,
-		 unformat_input_t * input,
-		 vlib_cli_command_t * cmd)
-{
-  int verbose = 0;
-  snat_main_t * sm = &snat_main;
-  snat_user_t * u;
-  snat_static_mapping_t *m;
-  snat_interface_t *i;
-  snat_address_t * ap;
-  vnet_main_t *vnm = vnet_get_main();
-  snat_main_per_thread_data_t *tsm;
-  u32 users_num = 0, sessions_num = 0, *worker, *sw_if_index;
-  uword j = 0;
-  snat_static_map_resolve_t *rp;
-  snat_det_map_t * dm;
-  snat_det_session_t * ses;
-
-  if (unformat (input, "detail"))
-    verbose = 1;
-  else if (unformat (input, "verbose"))
-    verbose = 2;
-
-  if (sm->static_mapping_only)
-    {
-      if (sm->static_mapping_connection_tracking)
-        vlib_cli_output (vm, "SNAT mode: static mapping only connection "
-                         "tracking");
-      else
-        vlib_cli_output (vm, "SNAT mode: static mapping only");
-    }
-  else if (sm->deterministic)
-    {
-      vlib_cli_output (vm, "SNAT mode: deterministic mapping");
-    }
-  else
-    {
-      vlib_cli_output (vm, "SNAT mode: dynamic translations enabled");
-    }
-
-  if (verbose > 0)
-    {
-      pool_foreach (i, sm->interfaces,
-      ({
-        vlib_cli_output (vm, "%U %s", format_vnet_sw_interface_name, vnm,
-                         vnet_get_sw_interface (vnm, i->sw_if_index),
-                         i->is_inside ? "in" : "out");
-      }));
-
-      pool_foreach (i, sm->output_feature_interfaces,
-      ({
-        vlib_cli_output (vm, "%U output-feature %s",
-                         format_vnet_sw_interface_name, vnm,
-                         vnet_get_sw_interface (vnm, i->sw_if_index),
-                         i->is_inside ? "in" : "out");
-      }));
-
-      if (vec_len (sm->auto_add_sw_if_indices))
-        {
-          vlib_cli_output (vm, "SNAT pool addresses interfaces:");
-          vec_foreach (sw_if_index, sm->auto_add_sw_if_indices)
-            {
-              vlib_cli_output (vm, "%U", format_vnet_sw_interface_name, vnm,
-                               vnet_get_sw_interface (vnm, *sw_if_index));
-            }
-        }
-
-      vec_foreach (ap, sm->addresses)
-        {
-          vlib_cli_output (vm, "%U", format_ip4_address, &ap->addr);
-          if (ap->fib_index != ~0)
-              vlib_cli_output (vm, "  tenant VRF: %u",
-                               ip4_fib_get(ap->fib_index)->table_id);
-          else
-            vlib_cli_output (vm, "  tenant VRF independent");
-#define _(N, i, n, s) \
-          vlib_cli_output (vm, "  %d busy %s ports", ap->busy_##n##_ports, s);
-          foreach_snat_protocol
-#undef _
-        }
-    }
-
-  if (sm->num_workers > 1)
-    {
-      vlib_cli_output (vm, "%d workers", vec_len (sm->workers));
-      if (verbose > 0)
-        {
-          vec_foreach (worker, sm->workers)
-            {
-              vlib_worker_thread_t *w =
-                vlib_worker_threads + *worker + sm->first_worker_index;
-              vlib_cli_output (vm, "  %s", w->name);
-            }
-        }
-    }
-
-  if (sm->deterministic)
-    {
-      vlib_cli_output (vm, "udp timeout: %dsec", sm->udp_timeout);
-      vlib_cli_output (vm, "tcp-established timeout: %dsec",
-                       sm->tcp_established_timeout);
-      vlib_cli_output (vm, "tcp-transitory timeout: %dsec",
-                       sm->tcp_transitory_timeout);
-      vlib_cli_output (vm, "icmp timeout: %dsec", sm->icmp_timeout);
-      vlib_cli_output (vm, "%d deterministic mappings",
-                       pool_elts (sm->det_maps));
-      if (verbose > 0)
-        {
-          pool_foreach (dm, sm->det_maps,
-          ({
-            vlib_cli_output (vm, "in %U/%d out %U/%d\n",
-                             format_ip4_address, &dm->in_addr, dm->in_plen,
-                             format_ip4_address, &dm->out_addr, dm->out_plen);
-            vlib_cli_output (vm, " outside address sharing ratio: %d\n",
-                             dm->sharing_ratio);
-            vlib_cli_output (vm, " number of ports per inside host: %d\n",
-                             dm->ports_per_host);
-            vlib_cli_output (vm, " sessions number: %d\n", dm->ses_num);
-            if (verbose > 1)
-              {
-                vec_foreach_index (j, dm->sessions)
-                  {
-                    ses = vec_elt_at_index (dm->sessions, j);
-                    if (ses->in_port)
-                      vlib_cli_output (vm, "  %U", format_det_map_ses, dm, ses,
-                                       &j);
-                  }
-              }
-          }));
-        }
-    }
-  else
-    {
-      if (sm->static_mapping_only && !(sm->static_mapping_connection_tracking))
-        {
-          vlib_cli_output (vm, "%d static mappings",
-                           pool_elts (sm->static_mappings));
-
-          if (verbose > 0)
-            {
-              pool_foreach (m, sm->static_mappings,
-              ({
-                vlib_cli_output (vm, "%U", format_snat_static_mapping, m);
-              }));
-            }
-        }
-      else
-        {
-          vec_foreach (tsm, sm->per_thread_data)
-            {
-              users_num += pool_elts (tsm->users);
-              sessions_num += pool_elts (tsm->sessions);
-            }
-
-          vlib_cli_output (vm, "%d users, %d outside addresses, %d active sessions,"
-                           " %d static mappings",
-                           users_num,
-                           vec_len (sm->addresses),
-                           sessions_num,
-                           pool_elts (sm->static_mappings));
-
-          if (verbose > 0)
-            {
-              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->in2out,
-                               verbose - 1);
-              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->out2in,
-                               verbose - 1);
-              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_in,
-                               verbose - 1);
-              vlib_cli_output (vm, "%U", format_bihash_8_8, &sm->worker_by_out,
-                               verbose - 1);
-              vec_foreach_index (j, sm->per_thread_data)
-                {
-                  tsm = vec_elt_at_index (sm->per_thread_data, j);
-
-                  if (pool_elts (tsm->users) == 0)
-                    continue;
-
-                  vlib_worker_thread_t *w = vlib_worker_threads + j;
-                  vlib_cli_output (vm, "Thread %d (%s at lcore %u):", j, w->name,
-                                   w->lcore_id);
-                  vlib_cli_output (vm, "  %d list pool elements",
-                                   pool_elts (tsm->list_pool));
-
-                  pool_foreach (u, tsm->users,
-                  ({
-                    vlib_cli_output (vm, "  %U", format_snat_user, tsm, u,
-                                     verbose - 1);
-                  }));
-                }
-
-              if (pool_elts (sm->static_mappings))
-                {
-                  vlib_cli_output (vm, "static mappings:");
-                  pool_foreach (m, sm->static_mappings,
-                  ({
-                    vlib_cli_output (vm, "%U", format_snat_static_mapping, m);
-                  }));
-                  for (j = 0; j < vec_len (sm->to_resolve); j++)
-                    {
-                      rp = sm->to_resolve + j;
-                      vlib_cli_output (vm, "%U",
-                                       format_snat_static_map_to_resolve, rp);
-                    }
-                }
-            }
-        }
-    }
-  return 0;
-}
-
-VLIB_CLI_COMMAND (show_snat_command, static) = {
-    .path = "show snat",
-    .short_help = "show snat",
-    .function = show_snat_command_fn,
-};
-
-
-static void
-snat_ip4_add_del_interface_address_cb (ip4_main_t * im,
-                                       uword opaque,
-                                       u32 sw_if_index,
-                                       ip4_address_t * address,
-                                       u32 address_length,
-                                       u32 if_address_index,
-                                       u32 is_delete)
-{
-  snat_main_t *sm = &snat_main;
-  snat_static_map_resolve_t *rp;
-  u32 *indices_to_delete = 0;
-  int i, j;
-  int rv;
-
-  for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
-    {
-      if (sw_if_index == sm->auto_add_sw_if_indices[i])
-        {
-          if (!is_delete)
-            {
-              /* Don't trip over lease renewal, static config */
-              for (j = 0; j < vec_len(sm->addresses); j++)
-                if (sm->addresses[j].addr.as_u32 == address->as_u32)
-                  return;
-
-              snat_add_address (sm, address, ~0);
-              /* Scan static map resolution vector */
-              for (j = 0; j < vec_len (sm->to_resolve); j++)
-                {
-                  rp = sm->to_resolve + j;
-                  /* On this interface? */
-                  if (rp->sw_if_index == sw_if_index)
-                    {
-                      /* Add the static mapping */
-                      rv = snat_add_static_mapping (rp->l_addr,
-                                                    address[0],
-                                                    rp->l_port,
-                                                    rp->e_port,
-                                                    rp->vrf_id,
-                                                    rp->addr_only,
-                                                    ~0 /* sw_if_index */,
-                                                    rp->proto,
-                                                    rp->is_add);
-                      if (rv)
-                        clib_warning ("snat_add_static_mapping returned %d", 
-                                      rv);
-                      vec_add1 (indices_to_delete, j);
-                    }
-                }
-              /* If we resolved any of the outstanding static mappings */
-              if (vec_len(indices_to_delete))
-                {
-                  /* Delete them */
-                  for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
-                    vec_delete(sm->to_resolve, 1, j);
-                  vec_free(indices_to_delete);
-                }
-              return;
-            }
-          else
-            {
-              (void) snat_del_address(sm, address[0], 1);
-              return;
-            }
-        }
-    }
-}
-
-
-int snat_add_interface_address (snat_main_t *sm, u32 sw_if_index, int is_del)
-{
-  ip4_main_t * ip4_main = sm->ip4_main;
-  ip4_address_t * first_int_addr;
-  snat_static_map_resolve_t *rp;
-  u32 *indices_to_delete = 0;
-  int i, j;
-
-  first_int_addr = ip4_interface_first_address (ip4_main, sw_if_index,
-                                                0 /* just want the address*/);
-
-  for (i = 0; i < vec_len(sm->auto_add_sw_if_indices); i++)
-    {
-      if (sm->auto_add_sw_if_indices[i] == sw_if_index)
-        {
-          if (is_del)
-            {
-              /* if have address remove it */
-              if (first_int_addr)
-                  (void) snat_del_address (sm, first_int_addr[0], 1);
-              else
-                {
-                  for (j = 0; j < vec_len (sm->to_resolve); j++)
-                    {
-                      rp = sm->to_resolve + j;
-                      if (rp->sw_if_index == sw_if_index)
-                        vec_add1 (indices_to_delete, j);
-                    }
-                  if (vec_len(indices_to_delete))
-                    {
-                      for (j = vec_len(indices_to_delete)-1; j >= 0; j--)
-                        vec_del1(sm->to_resolve, j);
-                      vec_free(indices_to_delete);
-                    }
-                }
-              vec_del1(sm->auto_add_sw_if_indices, i);
-            }
-          else
-            return VNET_API_ERROR_VALUE_EXIST;
-
-          return 0;
-        }
-    }
-  
-  if (is_del)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  /* add to the auto-address list */
-  vec_add1(sm->auto_add_sw_if_indices, sw_if_index);
-
-  /* If the address is already bound - or static - add it now */
-  if (first_int_addr)
-      snat_add_address (sm, first_int_addr, ~0);
-
-  return 0;
-}
-
-static clib_error_t *
-snat_add_interface_address_command_fn (vlib_main_t * vm,
-                                       unformat_input_t * input,
-                                       vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  u32 sw_if_index;
-  int rv;
-  int is_del = 0;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U", unformat_vnet_sw_interface,
-                    sm->vnet_main, &sw_if_index))
-        ;
-      else if (unformat (line_input, "del"))
-        is_del = 1;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-				     format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  rv = snat_add_interface_address (sm, sw_if_index, is_del);
-
-  switch (rv)
-    {
-    case 0:
-      break;
-
-    default:
-      error = clib_error_return (0, "snat_add_interface_address returned %d",
-                                 rv);
-      goto done;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-VLIB_CLI_COMMAND (snat_add_interface_address_command, static) = {
-    .path = "snat add interface address",
-    .short_help = "snat add interface address  [del]",
-    .function = snat_add_interface_address_command_fn,
-};
-
-static clib_error_t *
-snat_det_map_command_fn (vlib_main_t * vm,
-                         unformat_input_t * input,
-                         vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  ip4_address_t in_addr, out_addr;
-  u32 in_plen, out_plen;
-  int is_add = 1, rv;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "in %U/%u", unformat_ip4_address, &in_addr, &in_plen))
-        ;
-      else if (unformat (line_input, "out %U/%u", unformat_ip4_address, &out_addr, &out_plen))
-        ;
-      else if (unformat (line_input, "del"))
-        is_add = 0;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-                                     format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  unformat_free (line_input);
-
-  rv = snat_det_add_map(sm, &in_addr, (u8) in_plen, &out_addr, (u8)out_plen,
-                        is_add);
-
-  if (rv)
-    {
-      error = clib_error_return (0, "snat_det_add_map return %d", rv);
-      goto done;
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat deterministic add}
- * Create bijective mapping of inside address to outside address and port range
- * pairs, with the purpose of enabling deterministic NAT to reduce logging in
- * CGN deployments.
- * To create deterministic mapping between inside network 10.0.0.0/18 and
- * outside network 1.1.1.0/30 use:
- * # vpp# snat deterministic add in 10.0.0.0/18 out 1.1.1.0/30
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (snat_det_map_command, static) = {
-    .path = "snat deterministic add",
-    .short_help = "snat deterministic add in / out / [del]",
-    .function = snat_det_map_command_fn,
-};
-
-static clib_error_t *
-snat_det_forward_command_fn (vlib_main_t * vm,
-                             unformat_input_t * input,
-                             vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  ip4_address_t in_addr, out_addr;
-  u16 lo_port;
-  snat_det_map_t * dm;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U", unformat_ip4_address, &in_addr))
-        ;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-                                     format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  unformat_free (line_input);
-
-  dm = snat_det_map_by_user(sm, &in_addr);
-  if (!dm)
-    vlib_cli_output (vm, "no match");
-  else
-    {
-      snat_det_forward (dm, &in_addr, &out_addr, &lo_port);
-      vlib_cli_output (vm, "%U:<%d-%d>", format_ip4_address, &out_addr,
-                       lo_port, lo_port + dm->ports_per_host - 1);
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat deterministic forward}
- * Return outside address and port range from inside address for deterministic
- * NAT.
- * To obtain outside address and port of inside host use:
- *  vpp# snat deterministic forward 10.0.0.2
- *  1.1.1.0:<1054-1068>
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (snat_det_forward_command, static) = {
-    .path = "snat deterministic forward",
-    .short_help = "snat deterministic forward ",
-    .function = snat_det_forward_command_fn,
-};
-
-static clib_error_t *
-snat_det_reverse_command_fn (vlib_main_t * vm,
-                             unformat_input_t * input,
-                             vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  ip4_address_t in_addr, out_addr;
-  u32 out_port;
-  snat_det_map_t * dm;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U:%d", unformat_ip4_address, &out_addr, &out_port))
-        ;
-      else
-        {
-          error =  clib_error_return (0, "unknown input '%U'",
-                                      format_unformat_error, line_input);
-        }
-    }
-
-  unformat_free (line_input);
-
-  if (out_port < 1024 || out_port > 65535)
-    {
-      error = clib_error_return (0, "wrong port, must be <1024-65535>");
-      goto done;
-    }
-
-  dm = snat_det_map_by_out(sm, &out_addr);
-  if (!dm)
-    vlib_cli_output (vm, "no match");
-  else
-    {
-      snat_det_reverse (dm, &out_addr, (u16) out_port, &in_addr);
-      vlib_cli_output (vm, "%U", format_ip4_address, &in_addr);
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat deterministic reverse}
- * Return inside address from outside address and port for deterministic NAT.
- * To obtain inside host address from outside address and port use:
- *  #vpp snat deterministic reverse 1.1.1.1:1276
- *  10.0.16.16
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (snat_det_reverse_command, static) = {
-    .path = "snat deterministic reverse",
-    .short_help = "snat deterministic reverse :",
-    .function = snat_det_reverse_command_fn,
-};
-
-static clib_error_t *
-set_timeout_command_fn (vlib_main_t * vm,
-                        unformat_input_t * input,
-                        vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "udp %u", &sm->udp_timeout))
-        ;
-      else if (unformat (line_input, "tcp-established %u",
-               &sm->tcp_established_timeout))
-        ;
-      else if (unformat (line_input, "tcp-transitory %u",
-               &sm->tcp_transitory_timeout))
-        ;
-      else if (unformat (line_input, "icmp %u", &sm->icmp_timeout))
-        ;
-      else if (unformat (line_input, "reset"))
-        {
-          sm->udp_timeout = SNAT_UDP_TIMEOUT;
-          sm->tcp_established_timeout = SNAT_TCP_ESTABLISHED_TIMEOUT;
-          sm->tcp_transitory_timeout = SNAT_TCP_TRANSITORY_TIMEOUT;
-          sm->icmp_timeout = SNAT_ICMP_TIMEOUT;
-        }
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-                                     format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  unformat_free (line_input);
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{set snat deterministic timeout}
- * Set values of timeouts for deterministic NAT (in seconds), use:
- *  vpp# set snat deterministic timeout udp 120 tcp-established 7500
- *  tcp-transitory 250 icmp 90
- * To reset default values use:
- *  vpp# set snat deterministic timeout reset
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (set_timeout_command, static) = {
-  .path = "set snat deterministic timeout",
-  .function = set_timeout_command_fn,
-  .short_help =
-    "set snat deterministic timeout [udp  | tcp-established  "
-    "tcp-transitory  | icmp  | reset]",
-};
-
-static clib_error_t *
-snat_det_close_session_out_fn (vlib_main_t *vm,
-                               unformat_input_t * input,
-                               vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  ip4_address_t out_addr, ext_addr, in_addr;
-  u16 out_port, ext_port;
-  snat_det_map_t * dm;
-  snat_det_session_t * ses;
-  snat_det_out_key_t key;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U:%d %U:%d",
-                    unformat_ip4_address, &out_addr, &out_port,
-                    unformat_ip4_address, &ext_addr, &ext_port))
-        ;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-                                     format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  unformat_free (line_input);
-
-  dm = snat_det_map_by_out(sm, &out_addr);
-  if (!dm)
-    vlib_cli_output (vm, "no match");
-  else
-    {
-      snat_det_reverse(dm, &ext_addr, out_port, &in_addr);
-      key.ext_host_addr = out_addr;
-      key.ext_host_port = ntohs(ext_port);
-      key.out_port = ntohs(out_port);
-      ses = snat_det_get_ses_by_out(dm, &out_addr, key.as_u64);
-      if (!ses)
-        vlib_cli_output (vm, "no match");
-      else
-       snat_det_ses_close(dm, ses);
-    }
-
-done:
-  unformat_free (line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat deterministic close session out}
- * Close session using outside ip address and port
- * and external ip address and port, use:
- *  vpp# snat deterministic close session out 1.1.1.1:1276 2.2.2.2:2387
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (snat_det_close_sesion_out_command, static) = {
-  .path = "snat deterministic close session out",
-  .short_help = "snat deterministic close session out "
-                ": :",
-  .function = snat_det_close_session_out_fn,
-};
-
-static clib_error_t *
-snat_det_close_session_in_fn (vlib_main_t *vm,
-                              unformat_input_t * input,
-                              vlib_cli_command_t * cmd)
-{
-  snat_main_t *sm = &snat_main;
-  unformat_input_t _line_input, *line_input = &_line_input;
-  ip4_address_t in_addr, ext_addr;
-  u16 in_port, ext_port;
-  snat_det_map_t * dm;
-  snat_det_session_t * ses;
-  snat_det_out_key_t key;
-  clib_error_t *error = 0;
-
-  /* Get a line of input. */
-  if (!unformat_user (input, unformat_line_input, line_input))
-    return 0;
-
-  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (line_input, "%U:%d %U:%d",
-                    unformat_ip4_address, &in_addr, &in_port,
-                    unformat_ip4_address, &ext_addr, &ext_port))
-        ;
-      else
-        {
-          error = clib_error_return (0, "unknown input '%U'",
-                                     format_unformat_error, line_input);
-          goto done;
-        }
-    }
-
-  unformat_free (line_input);
-
-  dm = snat_det_map_by_user (sm, &in_addr);
-  if (!dm)
-    vlib_cli_output (vm, "no match");
-  else
-    {
-      key.ext_host_addr = ext_addr;
-      key.ext_host_port = ntohs (ext_port);
-      ses = snat_det_find_ses_by_in (dm, &in_addr, ntohs(in_port), key);
-      if (!ses)
-        vlib_cli_output (vm, "no match");
-      else
-        snat_det_ses_close(dm, ses);
-    }
-
-done:
-  unformat_free(line_input);
-
-  return error;
-}
-
-/*?
- * @cliexpar
- * @cliexstart{snat deterministic close_session_in}
- * Close session using inside ip address and port
- * and external ip address and port, use:
- *  vpp# snat deterministic close session in 3.3.3.3:3487 2.2.2.2:2387
- * @cliexend
-?*/
-VLIB_CLI_COMMAND (snat_det_close_session_in_command, static) = {
-  .path = "snat deterministic close session in",
-  .short_help = "snat deterministic close session in "
-                ": :",
-  .function = snat_det_close_session_in_fn,
-};
diff --git a/src/plugins/snat/snat.h b/src/plugins/snat/snat.h
deleted file mode 100644
index aa0f82fc..00000000
--- a/src/plugins/snat/snat.h
+++ /dev/null
@@ -1,541 +0,0 @@
-
-/*
- * snat.h - simple nat definitions
- *
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_snat_h__
-#define __included_snat_h__
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-
-#define SNAT_UDP_TIMEOUT 300
-#define SNAT_UDP_TIMEOUT_MIN 120
-#define SNAT_TCP_TRANSITORY_TIMEOUT 240
-#define SNAT_TCP_ESTABLISHED_TIMEOUT 7440
-#define SNAT_TCP_INCOMING_SYN 6
-#define SNAT_ICMP_TIMEOUT 60
-
-#define SNAT_FLAG_HAIRPINNING (1 << 0)
-
-/* Key */
-typedef struct {
-  union 
-  {
-    struct 
-    {
-      ip4_address_t addr;
-      u16 port;
-      u16 protocol:3,
-        fib_index:13;
-    };
-    u64 as_u64;
-  };
-} snat_session_key_t;
-
-typedef struct {
-  union
-  {
-    struct
-    {
-      ip4_address_t l_addr;
-      ip4_address_t r_addr;
-      u32 fib_index;
-      u8 proto;
-      u8 rsvd[3];
-    };
-    u64 as_u64[2];
-  };
-} snat_unk_proto_ses_key_t;
-
-typedef struct {
-  union
-  {
-    struct
-    {
-      ip4_address_t ext_host_addr;
-      u16 ext_host_port;
-      u16 out_port;
-    };
-    u64 as_u64;
-  };
-} snat_det_out_key_t;
-
-typedef struct {
-  union
-  {
-    struct
-    {
-      ip4_address_t addr;
-      u32 fib_index;
-    };
-    u64 as_u64;
-  };
-} snat_user_key_t;
-
-typedef struct {
-  union
-  {
-    struct
-    {
-      ip4_address_t addr;
-      u16 port;
-      u16 fib_index;
-    };
-    u64 as_u64;
-  };
-} snat_worker_key_t;
-
-
-#define foreach_snat_protocol \
-  _(UDP, 0, udp, "udp")       \
-  _(TCP, 1, tcp, "tcp")       \
-  _(ICMP, 2, icmp, "icmp")
-
-typedef enum {
-#define _(N, i, n, s) SNAT_PROTOCOL_##N = i,
-  foreach_snat_protocol
-#undef _
-} snat_protocol_t;
-
-
-#define foreach_snat_session_state          \
-  _(0, UNKNOWN, "unknown")                 \
-  _(1, UDP_ACTIVE, "udp-active")           \
-  _(2, TCP_SYN_SENT, "tcp-syn-sent")       \
-  _(3, TCP_ESTABLISHED, "tcp-established") \
-  _(4, TCP_FIN_WAIT, "tcp-fin-wait")       \
-  _(5, TCP_CLOSE_WAIT, "tcp-close-wait")   \
-  _(6, TCP_LAST_ACK, "tcp-last-ack")       \
-  _(7, ICMP_ACTIVE, "icmp-active")
-
-typedef enum {
-#define _(v, N, s) SNAT_SESSION_##N = v,
-  foreach_snat_session_state
-#undef _
-} snat_session_state_t;
-
-
-#define SNAT_SESSION_FLAG_STATIC_MAPPING 1
-#define SNAT_SESSION_FLAG_UNKNOWN_PROTO  2
-
-typedef CLIB_PACKED(struct {
-  snat_session_key_t out2in;    /* 0-15 */
-
-  snat_session_key_t in2out;    /* 16-31 */
-
-  u32 flags;                    /* 32-35 */
-
-  /* per-user translations */
-  u32 per_user_index;           /* 36-39 */
-
-  u32 per_user_list_head_index; /* 40-43 */
-
-  /* Last heard timer */
-  f64 last_heard;               /* 44-51 */
-
-  u64 total_bytes;              /* 52-59 */
-  
-  u32 total_pkts;               /* 60-63 */
-
-  /* Outside address */
-  u32 outside_address_index;    /* 64-67 */
-
-  /* External host address */
-  ip4_address_t ext_host_addr;  /* 68-71 */
-
-}) snat_session_t;
-
-
-typedef struct {
-  ip4_address_t addr;
-  u32 fib_index;
-  u32 sessions_per_user_list_head_index;
-  u32 nsessions;
-  u32 nstaticsessions;
-} snat_user_t;
-
-typedef struct {
-  ip4_address_t addr;
-  u32 fib_index;
-#define _(N, i, n, s) \
-  u32 busy_##n##_ports; \
-  uword * busy_##n##_port_bitmap;
-  foreach_snat_protocol
-#undef _
-} snat_address_t;
-
-typedef struct {
-  u16 in_port;
-  snat_det_out_key_t out;
-  u8 state;
-  u32 expire;
-} snat_det_session_t;
-
-typedef struct {
-  ip4_address_t in_addr;
-  u8 in_plen;
-  ip4_address_t out_addr;
-  u8 out_plen;
-  u32 sharing_ratio;
-  u16 ports_per_host;
-  u32 ses_num;
-  /* vector of sessions */
-  snat_det_session_t * sessions;
-} snat_det_map_t;
-
-typedef struct {
-  ip4_address_t local_addr;
-  ip4_address_t external_addr;
-  u16 local_port;
-  u16 external_port;
-  u8 addr_only;
-  u32 vrf_id;
-  u32 fib_index;
-  snat_protocol_t proto;
-} snat_static_mapping_t;
-
-typedef struct {
-  u32 sw_if_index;
-  u8 is_inside;
-} snat_interface_t;
-
-typedef struct {
-  ip4_address_t l_addr;
-  u16 l_port;
-  u16 e_port;
-  u32 sw_if_index;
-  u32 vrf_id;
-  snat_protocol_t proto;
-  int addr_only;
-  int is_add;
-} snat_static_map_resolve_t;
-
-typedef struct {
-  /* User pool */
-  snat_user_t * users;
-
-  /* Session pool */
-  snat_session_t * sessions;
-
-  /* Pool of doubly-linked list elements */
-  dlist_elt_t * list_pool;
-
-  u32 snat_thread_index;
-} snat_main_per_thread_data_t;
-
-struct snat_main_s;
-
-typedef u32 snat_icmp_match_function_t (struct snat_main_s *sm,
-                                        vlib_node_runtime_t *node,
-                                        u32 thread_index,
-                                        vlib_buffer_t *b0,
-                                        u8 *p_proto,
-                                        snat_session_key_t *p_value,
-                                        u8 *p_dont_translate,
-                                        void *d,
-                                        void *e);
-
-typedef u32 (snat_get_worker_function_t) (ip4_header_t * ip, u32 rx_fib_index);
-
-typedef struct snat_main_s {
-  /* Main lookup tables */
-  clib_bihash_8_8_t out2in;
-  clib_bihash_8_8_t in2out;
-
-  /* Unknown protocol sessions lookup tables */
-  clib_bihash_16_8_t out2in_unk_proto;
-  clib_bihash_16_8_t in2out_unk_proto;
-
-  /* Find-a-user => src address lookup */
-  clib_bihash_8_8_t user_hash;
-
-  /* Non-translated packets worker lookup => src address + VRF */
-  clib_bihash_8_8_t worker_by_in;
-
-  /* Translated packets worker lookup => IP address + port number */
-  clib_bihash_8_8_t worker_by_out;
-
-  snat_icmp_match_function_t * icmp_match_in2out_cb;
-  snat_icmp_match_function_t * icmp_match_out2in_cb;
-
-  u32 num_workers;
-  u32 first_worker_index;
-  u32 next_worker;
-  u32 * workers;
-  snat_get_worker_function_t * worker_in2out_cb;
-  snat_get_worker_function_t * worker_out2in_cb;
-  u16 port_per_thread;
-  u32 num_snat_thread;
-
-  /* Per thread data */
-  snat_main_per_thread_data_t * per_thread_data;
-
-  /* Find a static mapping by local */
-  clib_bihash_8_8_t static_mapping_by_local;
-
-  /* Find a static mapping by external */
-  clib_bihash_8_8_t static_mapping_by_external;
-
-  /* Static mapping pool */
-  snat_static_mapping_t * static_mappings;
-
-  /* Interface pool */
-  snat_interface_t * interfaces;
-  snat_interface_t * output_feature_interfaces;
-
-  /* Vector of outside addresses */
-  snat_address_t * addresses;
-
-  /* sw_if_indices whose intfc addresses should be auto-added */
-  u32 * auto_add_sw_if_indices;
-
-  /* vector of interface address static mappings to resolve. */
-  snat_static_map_resolve_t *to_resolve;
-
-  /* Randomize port allocation order */
-  u32 random_seed;
-
-  /* Worker handoff index */
-  u32 fq_in2out_index;
-  u32 fq_in2out_output_index;
-  u32 fq_out2in_index;
-
-  /* in2out and out2in node index */
-  u32 in2out_node_index;
-  u32 in2out_output_node_index;
-  u32 out2in_node_index;
-
-  /* Deterministic NAT */
-  snat_det_map_t * det_maps;
-
-  /* Config parameters */
-  u8 static_mapping_only;
-  u8 static_mapping_connection_tracking;
-  u8 deterministic;
-  u32 translation_buckets;
-  u32 translation_memory_size;
-  u32 user_buckets;
-  u32 user_memory_size;
-  u32 max_translations_per_user;
-  u32 outside_vrf_id;
-  u32 outside_fib_index;
-  u32 inside_vrf_id;
-  u32 inside_fib_index;
-
-  /* tenant VRF aware address pool activation flag */
-  u8 vrf_mode;
-
-  /* values of various timeouts */
-  u32 udp_timeout;
-  u32 tcp_established_timeout;
-  u32 tcp_transitory_timeout;
-  u32 icmp_timeout;
-
-  /* API message ID base */
-  u16 msg_id_base;
-
-  /* convenience */
-  vlib_main_t * vlib_main;
-  vnet_main_t * vnet_main;
-  ip4_main_t * ip4_main;
-  ip_lookup_main_t * ip4_lookup_main;
-  api_main_t * api_main;
-} snat_main_t;
-
-extern snat_main_t snat_main;
-extern vlib_node_registration_t snat_in2out_node;
-extern vlib_node_registration_t snat_in2out_output_node;
-extern vlib_node_registration_t snat_out2in_node;
-extern vlib_node_registration_t snat_in2out_fast_node;
-extern vlib_node_registration_t snat_out2in_fast_node;
-extern vlib_node_registration_t snat_in2out_worker_handoff_node;
-extern vlib_node_registration_t snat_in2out_output_worker_handoff_node;
-extern vlib_node_registration_t snat_out2in_worker_handoff_node;
-extern vlib_node_registration_t snat_det_in2out_node;
-extern vlib_node_registration_t snat_det_out2in_node;
-extern vlib_node_registration_t snat_hairpin_dst_node;
-extern vlib_node_registration_t snat_hairpin_src_node;
-
-void snat_free_outside_address_and_port (snat_main_t * sm,
-                                         snat_session_key_t * k,
-                                         u32 address_index);
-
-int snat_alloc_outside_address_and_port (snat_main_t * sm,
-                                         u32 fib_index,
-                                         u32 thread_index,
-                                         snat_session_key_t * k,
-                                         u32 * address_indexp);
-
-int snat_static_mapping_match (snat_main_t * sm,
-                               snat_session_key_t match,
-                               snat_session_key_t * mapping,
-                               u8 by_external,
-                               u8 *is_addr_only);
-
-void snat_add_del_addr_to_fib (ip4_address_t * addr,
-                               u8 p_len,
-                               u32 sw_if_index,
-                               int is_add);
-
-format_function_t format_snat_user;
-
-typedef struct {
-  u32 cached_sw_if_index;
-  u32 cached_ip4_address;
-} snat_runtime_t;
-
-/** \brief Check if SNAT session is created from static mapping.
-    @param s SNAT session
-    @return 1 if SNAT session is created from static mapping otherwise 0
-*/
-#define snat_is_session_static(s) s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING
-
-/** \brief Check if SNAT session for unknown protocol.
-    @param s SNAT session
-    @return 1 if SNAT session for unknown protocol otherwise 0
-*/
-#define snat_is_unk_proto_session(s) s->flags & SNAT_SESSION_FLAG_UNKNOWN_PROTO
-
-/* 
- * Why is this here? Because we don't need to touch this layer to
- * simply reply to an icmp. We need to change id to a unique
- * value to NAT an echo request/reply.
- */
-   
-typedef struct {
-  u16 identifier;
-  u16 sequence;
-} icmp_echo_header_t;
-
-always_inline u32
-ip_proto_to_snat_proto (u8 ip_proto)
-{
-  u32 snat_proto = ~0;
-
-  snat_proto = (ip_proto == IP_PROTOCOL_UDP) ? SNAT_PROTOCOL_UDP : snat_proto;
-  snat_proto = (ip_proto == IP_PROTOCOL_TCP) ? SNAT_PROTOCOL_TCP : snat_proto;
-  snat_proto = (ip_proto == IP_PROTOCOL_ICMP) ? SNAT_PROTOCOL_ICMP : snat_proto;
-  snat_proto = (ip_proto == IP_PROTOCOL_ICMP6) ? SNAT_PROTOCOL_ICMP : snat_proto;
-
-  return snat_proto;
-}
-
-always_inline u8
-snat_proto_to_ip_proto (snat_protocol_t snat_proto)
-{
-  u8 ip_proto = ~0;
-
-  ip_proto = (snat_proto == SNAT_PROTOCOL_UDP) ? IP_PROTOCOL_UDP : ip_proto;
-  ip_proto = (snat_proto == SNAT_PROTOCOL_TCP) ? IP_PROTOCOL_TCP : ip_proto;
-  ip_proto = (snat_proto == SNAT_PROTOCOL_ICMP) ? IP_PROTOCOL_ICMP : ip_proto;
-
-  return ip_proto;
-}
-
-typedef struct {
-  u16 src_port, dst_port;
-} tcp_udp_header_t;
-
-u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e);
-u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e);
-u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
-                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                          snat_session_key_t *p_value,
-                          u8 *p_dont_translate, void *d, void *e);
-u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e);
-u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
-                           u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                           snat_session_key_t *p_value,
-                           u8 *p_dont_translate, void *d, void *e);
-u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
-                          u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
-                          snat_session_key_t *p_value,
-                          u8 *p_dont_translate, void *d, void *e);
-void increment_v4_address(ip4_address_t * a);
-void snat_add_address(snat_main_t *sm, ip4_address_t *addr, u32 vrf_id);
-int snat_del_address(snat_main_t *sm, ip4_address_t addr, u8 delete_sm);
-int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
-                            u16 l_port, u16 e_port, u32 vrf_id, int addr_only,
-                            u32 sw_if_index, snat_protocol_t proto, int is_add);
-clib_error_t * snat_api_init(vlib_main_t * vm, snat_main_t * sm);
-int snat_set_workers (uword * bitmap);
-int snat_interface_add_del(u32 sw_if_index, u8 is_inside, int is_del);
-int snat_interface_add_del_output_feature(u32 sw_if_index, u8 is_inside,
-                                          int is_del);
-int snat_add_interface_address(snat_main_t *sm, u32 sw_if_index, int is_del);
-uword unformat_snat_protocol(unformat_input_t * input, va_list * args);
-u8 * format_snat_protocol(u8 * s, va_list * args);
-
-static_always_inline u8
-icmp_is_error_message (icmp46_header_t * icmp)
-{
-  switch(icmp->type)
-    {
-    case ICMP4_destination_unreachable:
-    case ICMP4_time_exceeded:
-    case ICMP4_parameter_problem:
-    case ICMP4_source_quench:
-    case ICMP4_redirect:
-    case ICMP4_alternate_host_address:
-      return 1;
-    }
-  return 0;
-}
-
-static_always_inline u8
-is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0,
-                  u32 ip4_addr)
-{
-  snat_runtime_t *rt = (snat_runtime_t *) node->runtime_data;
-  ip4_address_t * first_int_addr;
-
-  if (PREDICT_FALSE(rt->cached_sw_if_index != sw_if_index0))
-    {
-      first_int_addr =
-        ip4_interface_first_address (sm->ip4_main, sw_if_index0,
-                                     0 /* just want the address */);
-      rt->cached_sw_if_index = sw_if_index0;
-      if (first_int_addr)
-        rt->cached_ip4_address = first_int_addr->as_u32;
-      else
-        rt->cached_ip4_address = 0;
-    }
-
-  if (PREDICT_FALSE(ip4_addr == rt->cached_ip4_address))
-    return 1;
-  else
-    return 0;
-}
-
-#endif /* __included_snat_h__ */
diff --git a/src/plugins/snat/snat_all_api_h.h b/src/plugins/snat/snat_all_api_h.h
deleted file mode 100644
index 49017700..00000000
--- a/src/plugins/snat/snat_all_api_h.h
+++ /dev/null
@@ -1,19 +0,0 @@
-
-/*
- * snat_all_api_h.h - skeleton vpp engine plug-in api #include file
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/* Include the generated file, see BUILT_SOURCES in Makefile.am */
-#include 
diff --git a/src/plugins/snat/snat_api.c b/src/plugins/snat/snat_api.c
deleted file mode 100644
index 227074f9..00000000
--- a/src/plugins/snat/snat_api.c
+++ /dev/null
@@ -1,1970 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief SNAT plugin API implementation
- */
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-/* define message structures */
-#define vl_typedefs
-#include 
-#undef vl_typedefs
-
-/* define generated endian-swappers */
-#define vl_endianfun
-#include 
-#undef vl_endianfun
-
-#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
-
-#define REPLY_MSG_ID_BASE sm->msg_id_base
-#include 
-
-/* Get the API version number */
-#define vl_api_version(n,v) static u32 api_version=(v);
-#include 
-#undef vl_api_version
-
-/* Macro to finish up custom dump fns */
-#define FINISH                                  \
-    vec_add1 (s, 0);                            \
-    vl_print (handle, (char *)s);               \
-    vec_free (s);                               \
-    return handle;
-
-static void
-  vl_api_snat_add_address_range_t_handler
-  (vl_api_snat_add_address_range_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_add_address_range_reply_t *rmp;
-  ip4_address_t this_addr;
-  u32 start_host_order, end_host_order;
-  u32 vrf_id;
-  int i, count;
-  int rv = 0;
-  u32 *tmp;
-
-  if (mp->is_ip4 != 1)
-    {
-      rv = VNET_API_ERROR_UNIMPLEMENTED;
-      goto send_reply;
-    }
-
-  if (sm->static_mapping_only)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  tmp = (u32 *) mp->first_ip_address;
-  start_host_order = clib_host_to_net_u32 (tmp[0]);
-  tmp = (u32 *) mp->last_ip_address;
-  end_host_order = clib_host_to_net_u32 (tmp[0]);
-
-  count = (end_host_order - start_host_order) + 1;
-
-  vrf_id = clib_host_to_net_u32 (mp->vrf_id);
-
-  if (count > 1024)
-    clib_warning ("%U - %U, %d addresses...",
-		  format_ip4_address, mp->first_ip_address,
-		  format_ip4_address, mp->last_ip_address, count);
-
-  memcpy (&this_addr.as_u8, mp->first_ip_address, 4);
-
-  for (i = 0; i < count; i++)
-    {
-      if (mp->is_add)
-	snat_add_address (sm, &this_addr, vrf_id);
-      else
-	rv = snat_del_address (sm, this_addr, 0);
-
-      if (rv)
-	goto send_reply;
-
-      increment_v4_address (&this_addr);
-    }
-
-send_reply:
-  REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY);
-}
-
-static void *vl_api_snat_add_address_range_t_print
-  (vl_api_snat_add_address_range_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_add_address_range ");
-  s = format (s, "%U ", format_ip4_address, mp->first_ip_address);
-  if (memcmp (mp->first_ip_address, mp->last_ip_address, 4))
-    {
-      s = format (s, " - %U ", format_ip4_address, mp->last_ip_address);
-    }
-  FINISH;
-}
-
-static void
-  send_snat_address_details
-  (snat_address_t * a, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_address_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_SNAT_ADDRESS_DETAILS + sm->msg_id_base);
-  rmp->is_ip4 = 1;
-  clib_memcpy (rmp->ip_address, &(a->addr), 4);
-  if (a->fib_index != ~0)
-    {
-      fib_table_t *fib = fib_table_get (a->fib_index, FIB_PROTOCOL_IP4);
-      rmp->vrf_id = ntohl (fib->ft_table_id);
-    }
-  else
-    rmp->vrf_id = ~0;
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-vl_api_snat_address_dump_t_handler (vl_api_snat_address_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_address_t *a;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  vec_foreach (a, sm->addresses)
-    send_snat_address_details (a, q, mp->context);
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_address_dump_t_print
-  (vl_api_snat_address_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_address_dump ");
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_interface_add_del_feature_t_handler
-  (vl_api_snat_interface_add_del_feature_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_interface_add_del_feature_reply_t *rmp;
-  u8 is_del = mp->is_add == 0;
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  int rv = 0;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  rv = snat_interface_add_del (sw_if_index, mp->is_inside, is_del);
-
-  BAD_SW_IF_INDEX_LABEL;
-
-  REPLY_MACRO (VL_API_SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY);
-}
-
-static void *vl_api_snat_interface_add_del_feature_t_print
-  (vl_api_snat_interface_add_del_feature_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_interface_add_del_feature ");
-  s = format (s, "sw_if_index %d %s %s",
-	      clib_host_to_net_u32 (mp->sw_if_index),
-	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
-
-  FINISH;
-}
-
-static void
-  send_snat_interface_details
-  (snat_interface_t * i, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_interface_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_SNAT_INTERFACE_DETAILS + sm->msg_id_base);
-  rmp->sw_if_index = ntohl (i->sw_if_index);
-  rmp->is_inside = i->is_inside;
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-vl_api_snat_interface_dump_t_handler (vl_api_snat_interface_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_interface_t *i;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  pool_foreach (i, sm->interfaces,
-  ({
-    send_snat_interface_details(i, q, mp->context);
-  }));
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_interface_dump_t_print
-  (vl_api_snat_interface_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_interface_dump ");
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_interface_add_del_output_feature_t_handler
-  (vl_api_snat_interface_add_del_output_feature_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_interface_add_del_output_feature_reply_t *rmp;
-  u8 is_del = mp->is_add == 0;
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  int rv = 0;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  rv = snat_interface_add_del_output_feature (sw_if_index, mp->is_inside,
-					      is_del);
-
-  BAD_SW_IF_INDEX_LABEL;
-
-  REPLY_MACRO (VL_API_SNAT_INTERFACE_ADD_DEL_OUTPUT_FEATURE_REPLY);
-}
-
-static void *vl_api_snat_interface_add_del_output_feature_t_print
-  (vl_api_snat_interface_add_del_output_feature_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_interface_add_del_output_feature ");
-  s = format (s, "sw_if_index %d %s %s",
-	      clib_host_to_net_u32 (mp->sw_if_index),
-	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
-
-  FINISH;
-}
-
-static void
-send_snat_interface_output_feature_details (snat_interface_t * i,
-					    unix_shared_memory_queue_t * q,
-					    u32 context)
-{
-  vl_api_snat_interface_output_feature_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id =
-    ntohs (VL_API_SNAT_INTERFACE_OUTPUT_FEATURE_DETAILS + sm->msg_id_base);
-  rmp->sw_if_index = ntohl (i->sw_if_index);
-  rmp->context = context;
-  rmp->is_inside = i->is_inside;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-  vl_api_snat_interface_output_feature_dump_t_handler
-  (vl_api_snat_interface_output_feature_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_interface_t *i;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  pool_foreach (i, sm->output_feature_interfaces,
-  ({
-    send_snat_interface_output_feature_details(i, q, mp->context);
-  }));
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_interface_output_feature_dump_t_print
-  (vl_api_snat_interface_output_feature_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_interface_output_feature_dump ");
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_add_static_mapping_t_handler
-  (vl_api_snat_add_static_mapping_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_add_static_mapping_reply_t *rmp;
-  ip4_address_t local_addr, external_addr;
-  u16 local_port = 0, external_port = 0;
-  u32 vrf_id, external_sw_if_index;
-  int rv = 0;
-  snat_protocol_t proto;
-
-  if (mp->is_ip4 != 1)
-    {
-      rv = VNET_API_ERROR_UNIMPLEMENTED;
-      goto send_reply;
-    }
-
-  memcpy (&local_addr.as_u8, mp->local_ip_address, 4);
-  memcpy (&external_addr.as_u8, mp->external_ip_address, 4);
-  if (mp->addr_only == 0)
-    {
-      local_port = clib_net_to_host_u16 (mp->local_port);
-      external_port = clib_net_to_host_u16 (mp->external_port);
-    }
-  vrf_id = clib_net_to_host_u32 (mp->vrf_id);
-  external_sw_if_index = clib_net_to_host_u32 (mp->external_sw_if_index);
-  proto = ip_proto_to_snat_proto (mp->protocol);
-
-  rv = snat_add_static_mapping (local_addr, external_addr, local_port,
-				external_port, vrf_id, mp->addr_only,
-				external_sw_if_index, proto, mp->is_add);
-
-send_reply:
-  REPLY_MACRO (VL_API_SNAT_ADD_ADDRESS_RANGE_REPLY);
-}
-
-static void *vl_api_snat_add_static_mapping_t_print
-  (vl_api_snat_add_static_mapping_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_add_static_mapping ");
-  s = format (s, "protocol %d local_addr %U external_addr %U ",
-	      mp->protocol,
-	      format_ip4_address, mp->local_ip_address,
-	      format_ip4_address, mp->external_ip_address);
-
-  if (mp->addr_only == 0)
-    s = format (s, "local_port %d external_port %d ",
-		clib_net_to_host_u16 (mp->local_port),
-		clib_net_to_host_u16 (mp->external_port));
-
-  if (mp->vrf_id != ~0)
-    s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id));
-
-  if (mp->external_sw_if_index != ~0)
-    s = format (s, "external_sw_if_index %d",
-		clib_net_to_host_u32 (mp->external_sw_if_index));
-  FINISH;
-}
-
-static void
-  send_snat_static_mapping_details
-  (snat_static_mapping_t * m, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_static_mapping_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id =
-    ntohs (VL_API_SNAT_STATIC_MAPPING_DETAILS + sm->msg_id_base);
-  rmp->is_ip4 = 1;
-  rmp->addr_only = m->addr_only;
-  clib_memcpy (rmp->local_ip_address, &(m->local_addr), 4);
-  clib_memcpy (rmp->external_ip_address, &(m->external_addr), 4);
-  rmp->local_port = htons (m->local_port);
-  rmp->external_port = htons (m->external_port);
-  rmp->external_sw_if_index = ~0;
-  rmp->vrf_id = htonl (m->vrf_id);
-  rmp->protocol = snat_proto_to_ip_proto (m->proto);
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-  send_snat_static_map_resolve_details
-  (snat_static_map_resolve_t * m, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_static_mapping_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id =
-    ntohs (VL_API_SNAT_STATIC_MAPPING_DETAILS + sm->msg_id_base);
-  rmp->is_ip4 = 1;
-  rmp->addr_only = m->addr_only;
-  clib_memcpy (rmp->local_ip_address, &(m->l_addr), 4);
-  rmp->local_port = htons (m->l_port);
-  rmp->external_port = htons (m->e_port);
-  rmp->external_sw_if_index = htonl (m->sw_if_index);
-  rmp->vrf_id = htonl (m->vrf_id);
-  rmp->protocol = snat_proto_to_ip_proto (m->proto);
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-  vl_api_snat_static_mapping_dump_t_handler
-  (vl_api_snat_static_mapping_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_static_mapping_t *m;
-  snat_static_map_resolve_t *rp;
-  int j;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  pool_foreach (m, sm->static_mappings,
-  ({
-      send_snat_static_mapping_details (m, q, mp->context);
-  }));
-  /* *INDENT-ON* */
-
-  for (j = 0; j < vec_len (sm->to_resolve); j++)
-    {
-      rp = sm->to_resolve + j;
-      send_snat_static_map_resolve_details (rp, q, mp->context);
-    }
-}
-
-static void *vl_api_snat_static_mapping_dump_t_print
-  (vl_api_snat_static_mapping_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_static_mapping_dump ");
-
-  FINISH;
-}
-
-static void
-vl_api_snat_control_ping_t_handler (vl_api_snat_control_ping_t * mp)
-{
-  vl_api_snat_control_ping_reply_t *rmp;
-  snat_main_t *sm = &snat_main;
-  int rv = 0;
-
-  /* *INDENT-OFF* */
-  REPLY_MACRO2 (VL_API_SNAT_CONTROL_PING_REPLY,
-  ({
-    rmp->vpe_pid = ntohl (getpid ());
-  }));
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_control_ping_t_print
-  (vl_api_snat_control_ping_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_control_ping ");
-
-  FINISH;
-}
-
-static void
-vl_api_snat_show_config_t_handler (vl_api_snat_show_config_t * mp)
-{
-  vl_api_snat_show_config_reply_t *rmp;
-  snat_main_t *sm = &snat_main;
-  int rv = 0;
-
-  /* *INDENT-OFF* */
-  REPLY_MACRO2 (VL_API_SNAT_SHOW_CONFIG_REPLY,
-  ({
-    rmp->translation_buckets = htonl (sm->translation_buckets);
-    rmp->translation_memory_size = htonl (sm->translation_memory_size);
-    rmp->user_buckets = htonl (sm->user_buckets);
-    rmp->user_memory_size = htonl (sm->user_memory_size);
-    rmp->max_translations_per_user = htonl (sm->max_translations_per_user);
-    rmp->outside_vrf_id = htonl (sm->outside_vrf_id);
-    rmp->inside_vrf_id = htonl (sm->inside_vrf_id);
-    rmp->static_mapping_only = sm->static_mapping_only;
-    rmp->static_mapping_connection_tracking =
-      sm->static_mapping_connection_tracking;
-    rmp->deterministic = sm->deterministic;
-  }));
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_show_config_t_print
-  (vl_api_snat_show_config_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_show_config ");
-
-  FINISH;
-}
-
-static void
-vl_api_snat_set_workers_t_handler (vl_api_snat_set_workers_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_set_workers_reply_t *rmp;
-  int rv = 0;
-  uword *bitmap = 0;
-  u64 mask = clib_net_to_host_u64 (mp->worker_mask);
-
-  if (sm->num_workers < 2)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask));
-  rv = snat_set_workers (bitmap);
-  clib_bitmap_free (bitmap);
-
-send_reply:
-  REPLY_MACRO (VL_API_SNAT_SET_WORKERS_REPLY);
-}
-
-static void *vl_api_snat_set_workers_t_print
-  (vl_api_snat_set_workers_t * mp, void *handle)
-{
-  u8 *s;
-  uword *bitmap = 0;
-  u8 first = 1;
-  int i;
-  u64 mask = clib_net_to_host_u64 (mp->worker_mask);
-
-  s = format (0, "SCRIPT: snat_set_workers ");
-  bitmap = clib_bitmap_set_multiple (bitmap, 0, mask, BITS (mask));
-  /* *INDENT-OFF* */
-  clib_bitmap_foreach (i, bitmap,
-    ({
-      if (first)
-        s = format (s, "%d", i);
-      else
-        s = format (s, ",%d", i);
-      first = 0;
-    }));
-  /* *INDENT-ON* */
-  clib_bitmap_free (bitmap);
-  FINISH;
-}
-
-static void
-  send_snat_worker_details
-  (u32 worker_index, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_worker_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  vlib_worker_thread_t *w =
-    vlib_worker_threads + worker_index + sm->first_worker_index;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_SNAT_WORKER_DETAILS + sm->msg_id_base);
-  rmp->context = context;
-  rmp->worker_index = htonl (worker_index);
-  rmp->lcore_id = htonl (w->lcore_id);
-  strncpy ((char *) rmp->name, (char *) w->name, ARRAY_LEN (rmp->name) - 1);
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-vl_api_snat_worker_dump_t_handler (vl_api_snat_worker_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  u32 *worker_index;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  vec_foreach (worker_index, sm->workers)
-    send_snat_worker_details(*worker_index, q, mp->context);
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_worker_dump_t_print
-  (vl_api_snat_worker_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_worker_dump ");
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_add_del_interface_addr_t_handler
-  (vl_api_snat_add_del_interface_addr_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_add_del_interface_addr_reply_t *rmp;
-  u8 is_del = mp->is_add == 0;
-  u32 sw_if_index = ntohl (mp->sw_if_index);
-  int rv = 0;
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  rv = snat_add_interface_address (sm, sw_if_index, is_del);
-
-  BAD_SW_IF_INDEX_LABEL;
-
-  REPLY_MACRO (VL_API_SNAT_ADD_DEL_INTERFACE_ADDR_REPLY);
-}
-
-static void *vl_api_snat_add_del_interface_addr_t_print
-  (vl_api_snat_add_del_interface_addr_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_add_del_interface_addr ");
-  s = format (s, "sw_if_index %d %s",
-	      clib_host_to_net_u32 (mp->sw_if_index),
-	      mp->is_add ? "" : "del");
-
-  FINISH;
-}
-
-static void
-  send_snat_interface_addr_details
-  (u32 sw_if_index, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_interface_addr_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id =
-    ntohs (VL_API_SNAT_INTERFACE_ADDR_DETAILS + sm->msg_id_base);
-  rmp->sw_if_index = ntohl (sw_if_index);
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-  vl_api_snat_interface_addr_dump_t_handler
-  (vl_api_snat_interface_addr_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  u32 *i;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  vec_foreach (i, sm->auto_add_sw_if_indices)
-    send_snat_interface_addr_details(*i, q, mp->context);
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_interface_addr_dump_t_print
-  (vl_api_snat_interface_addr_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_interface_addr_dump ");
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_ipfix_enable_disable_t_handler
-  (vl_api_snat_ipfix_enable_disable_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_ipfix_enable_disable_reply_t *rmp;
-  int rv = 0;
-
-  rv = snat_ipfix_logging_enable_disable (mp->enable,
-					  clib_host_to_net_u32
-					  (mp->domain_id),
-					  clib_host_to_net_u16
-					  (mp->src_port));
-
-  REPLY_MACRO (VL_API_SNAT_IPFIX_ENABLE_DISABLE_REPLY);
-}
-
-static void *vl_api_snat_ipfix_enable_disable_t_print
-  (vl_api_snat_ipfix_enable_disable_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_ipfix_enable_disable ");
-  if (mp->domain_id)
-    s = format (s, "domain %d ", clib_net_to_host_u32 (mp->domain_id));
-  if (mp->src_port)
-    s = format (s, "src_port %d ", clib_net_to_host_u16 (mp->src_port));
-  if (!mp->enable)
-    s = format (s, "disable ");
-
-  FINISH;
-}
-
-static void
-  send_snat_user_details
-  (snat_user_t * u, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_user_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  fib_table_t *fib = fib_table_get (u->fib_index, FIB_PROTOCOL_IP4);
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_SNAT_USER_DETAILS + sm->msg_id_base);
-
-  rmp->vrf_id = ntohl (fib->ft_table_id);
-
-  rmp->is_ip4 = 1;
-  clib_memcpy (rmp->ip_address, &(u->addr), 4);
-  rmp->nsessions = ntohl (u->nsessions);
-  rmp->nstaticsessions = ntohl (u->nstaticsessions);
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-vl_api_snat_user_dump_t_handler (vl_api_snat_user_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_main_per_thread_data_t *tsm;
-  snat_user_t *u;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  vec_foreach (tsm, sm->per_thread_data)
-    vec_foreach (u, tsm->users)
-      send_snat_user_details (u, q, mp->context);
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_user_dump_t_print
-  (vl_api_snat_user_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_user_dump ");
-
-  FINISH;
-}
-
-static void
-  send_snat_user_session_details
-  (snat_session_t * s, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_user_session_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id =
-    ntohs (VL_API_SNAT_USER_SESSION_DETAILS + sm->msg_id_base);
-  rmp->is_ip4 = 1;
-  clib_memcpy (rmp->outside_ip_address, (&s->out2in.addr), 4);
-  clib_memcpy (rmp->inside_ip_address, (&s->in2out.addr), 4);
-  rmp->is_static = s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING ? 1 : 0;
-  rmp->last_heard = clib_host_to_net_u64 ((u64) s->last_heard);
-  rmp->total_bytes = clib_host_to_net_u64 (s->total_bytes);
-  rmp->total_pkts = ntohl (s->total_pkts);
-  rmp->context = context;
-  if (snat_is_unk_proto_session (s))
-    {
-      rmp->outside_port = 0;
-      rmp->inside_port = 0;
-      rmp->protocol = ntohs (s->in2out.port);
-    }
-  else
-    {
-      rmp->outside_port = s->out2in.port;
-      rmp->inside_port = s->in2out.port;
-      rmp->protocol = ntohs (snat_proto_to_ip_proto (s->in2out.protocol));
-    }
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-  vl_api_snat_user_session_dump_t_handler
-  (vl_api_snat_user_session_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_main_per_thread_data_t *tsm;
-  snat_session_t *s;
-  clib_bihash_kv_8_8_t key, value;
-  snat_user_key_t ukey;
-  snat_user_t *u;
-  u32 session_index, head_index, elt_index;
-  dlist_elt_t *head, *elt;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-  if (!mp->is_ip4)
-    return;
-
-  clib_memcpy (&ukey.addr, mp->ip_address, 4);
-  ukey.fib_index = fib_table_find (FIB_PROTOCOL_IP4, ntohl (mp->vrf_id));
-  key.key = ukey.as_u64;
-  if (!clib_bihash_search_8_8 (&sm->worker_by_in, &key, &value))
-    tsm = vec_elt_at_index (sm->per_thread_data, value.value);
-  else
-    tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
-  if (clib_bihash_search_8_8 (&sm->user_hash, &key, &value))
-    return;
-  u = pool_elt_at_index (tsm->users, value.value);
-  if (!u->nsessions && !u->nstaticsessions)
-    return;
-
-  head_index = u->sessions_per_user_list_head_index;
-  head = pool_elt_at_index (tsm->list_pool, head_index);
-  elt_index = head->next;
-  elt = pool_elt_at_index (tsm->list_pool, elt_index);
-  session_index = elt->value;
-  while (session_index != ~0)
-    {
-      s = pool_elt_at_index (tsm->sessions, session_index);
-
-      send_snat_user_session_details (s, q, mp->context);
-
-      elt_index = elt->next;
-      elt = pool_elt_at_index (tsm->list_pool, elt_index);
-      session_index = elt->value;
-    }
-}
-
-static void *vl_api_snat_user_session_dump_t_print
-  (vl_api_snat_user_session_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_user_session_dump ");
-  s = format (s, "ip_address %U vrf_id %d\n",
-	      format_ip4_address, mp->ip_address,
-	      clib_net_to_host_u32 (mp->vrf_id));
-
-  FINISH;
-}
-
-/****************************/
-/*** detrministic NAT/CGN ***/
-/****************************/
-
-static void
-vl_api_snat_add_det_map_t_handler (vl_api_snat_add_det_map_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_add_det_map_reply_t *rmp;
-  int rv = 0;
-  ip4_address_t in_addr, out_addr;
-
-  clib_memcpy (&in_addr, mp->in_addr, 4);
-  clib_memcpy (&out_addr, mp->out_addr, 4);
-  rv = snat_det_add_map (sm, &in_addr, mp->in_plen, &out_addr,
-			 mp->out_plen, mp->is_add);
-
-  REPLY_MACRO (VL_API_SNAT_ADD_DET_MAP_REPLY);
-}
-
-static void *vl_api_snat_add_det_map_t_print
-  (vl_api_snat_add_det_map_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_add_det_map ");
-  s = format (s, "inside address %U/%d outside address %U/%d\n",
-	      format_ip4_address, mp->in_addr, mp->in_plen,
-	      format_ip4_address, mp->out_addr, mp->out_plen);
-
-  FINISH;
-}
-
-static void
-vl_api_snat_det_forward_t_handler (vl_api_snat_det_forward_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_det_forward_reply_t *rmp;
-  int rv = 0;
-  u16 lo_port = 0, hi_port = 0;
-  snat_det_map_t *dm;
-  ip4_address_t in_addr, out_addr;
-
-  out_addr.as_u32 = 0;
-  clib_memcpy (&in_addr, mp->in_addr, 4);
-  dm = snat_det_map_by_user (sm, &in_addr);
-  if (!dm)
-    {
-      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
-      goto send_reply;
-    }
-
-  snat_det_forward (dm, &in_addr, &out_addr, &lo_port);
-  hi_port = lo_port + dm->ports_per_host - 1;
-
-send_reply:
-  /* *INDENT-OFF* */
-  REPLY_MACRO2 (VL_API_SNAT_DET_FORWARD_REPLY,
-  ({
-    rmp->out_port_lo = ntohs (lo_port);
-    rmp->out_port_hi = ntohs (hi_port);
-    rmp->is_ip4 = 1;
-    memset (rmp->out_addr, 0, 16);
-    clib_memcpy (rmp->out_addr, &out_addr, 4);
-  }))
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_det_forward_t_print
-  (vl_api_snat_det_forward_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: smat_det_forward_t");
-  s = format (s, "inside ip address %U\n", format_ip4_address, mp->in_addr);
-
-  FINISH;
-}
-
-static void
-vl_api_snat_det_reverse_t_handler (vl_api_snat_det_reverse_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_det_reverse_reply_t *rmp;
-  int rv = 0;
-  ip4_address_t out_addr, in_addr;
-  snat_det_map_t *dm;
-
-  in_addr.as_u32 = 0;
-  clib_memcpy (&out_addr, mp->out_addr, 4);
-  dm = snat_det_map_by_out (sm, &out_addr);
-  if (!dm)
-    {
-      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
-      goto send_reply;
-    }
-
-  snat_det_reverse (dm, &out_addr, htons (mp->out_port), &in_addr);
-
-send_reply:
-  /* *INDENT-OFF* */
-  REPLY_MACRO2 (VL_API_SNAT_DET_REVERSE_REPLY,
-  ({
-    rmp->is_ip4 = 1;
-    memset (rmp->in_addr, 0, 16);
-    clib_memcpy (rmp->in_addr, &in_addr, 4);
-  }))
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_det_reverse_t_print
-  (vl_api_snat_det_reverse_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: smat_det_reverse_t");
-  s = format (s, "outside ip address %U outside port %d",
-	      format_ip4_address, mp->out_addr, ntohs (mp->out_port));
-
-  FINISH;
-}
-
-static void
-  sent_snat_det_map_details
-  (snat_det_map_t * m, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_det_map_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_SNAT_DET_MAP_DETAILS + sm->msg_id_base);
-  rmp->is_ip4 = 1;
-  clib_memcpy (rmp->in_addr, &m->in_addr, 4);
-  rmp->in_plen = m->in_plen;
-  clib_memcpy (rmp->out_addr, &m->out_addr, 4);
-  rmp->out_plen = m->out_plen;
-  rmp->sharing_ratio = htonl (m->sharing_ratio);
-  rmp->ports_per_host = htons (m->ports_per_host);
-  rmp->ses_num = htonl (m->ses_num);
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-vl_api_snat_det_map_dump_t_handler (vl_api_snat_det_map_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  snat_det_map_t *m;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  /* *INDENT-OFF* */
-  vec_foreach(m, sm->det_maps)
-    sent_snat_det_map_details(m, q, mp->context);
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_det_map_dump_t_print
-  (vl_api_snat_det_map_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_det_map_dump ");
-
-  FINISH;
-}
-
-static void
-vl_api_snat_det_set_timeouts_t_handler (vl_api_snat_det_set_timeouts_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_det_set_timeouts_reply_t *rmp;
-  int rv = 0;
-
-  sm->udp_timeout = ntohl (mp->udp);
-  sm->tcp_established_timeout = ntohl (mp->tcp_established);
-  sm->tcp_transitory_timeout = ntohl (mp->tcp_transitory);
-  sm->icmp_timeout = ntohl (mp->icmp);
-
-  REPLY_MACRO (VL_API_SNAT_DET_SET_TIMEOUTS_REPLY);
-}
-
-static void *vl_api_snat_det_set_timeouts_t_print
-  (vl_api_snat_det_set_timeouts_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_det_set_timeouts ");
-  s = format (s, "udp %d tcp_established %d tcp_transitory %d icmp %d\n",
-	      ntohl (mp->udp),
-	      ntohl (mp->tcp_established),
-	      ntohl (mp->tcp_transitory), ntohl (mp->icmp));
-
-  FINISH;
-}
-
-static void
-vl_api_snat_det_get_timeouts_t_handler (vl_api_snat_det_get_timeouts_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_det_get_timeouts_reply_t *rmp;
-  int rv = 0;
-
-  /* *INDENT-OFF* */
-  REPLY_MACRO2 (VL_API_SNAT_DET_GET_TIMEOUTS_REPLY,
-  ({
-    rmp->udp = htonl (sm->udp_timeout);
-    rmp->tcp_established = htonl (sm->tcp_established_timeout);
-    rmp->tcp_transitory = htonl (sm->tcp_transitory_timeout);
-    rmp->icmp = htonl (sm->icmp_timeout);
-  }))
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_snat_det_get_timeouts_t_print
-  (vl_api_snat_det_get_timeouts_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_det_get_timeouts");
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_det_close_session_out_t_handler
-  (vl_api_snat_det_close_session_out_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_det_close_session_out_reply_t *rmp;
-  ip4_address_t out_addr, ext_addr, in_addr;
-  snat_det_out_key_t key;
-  snat_det_map_t *dm;
-  snat_det_session_t *ses;
-  int rv = 0;
-
-  clib_memcpy (&out_addr, mp->out_addr, 4);
-  clib_memcpy (&ext_addr, mp->ext_addr, 4);
-
-  dm = snat_det_map_by_out (sm, &out_addr);
-  if (!dm)
-    {
-      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
-      goto send_reply;
-    }
-  snat_det_reverse (dm, &ext_addr, ntohs (mp->out_port), &in_addr);
-  key.ext_host_addr = ext_addr;
-  key.ext_host_port = mp->ext_port;
-  key.out_port = mp->out_port;
-  ses = snat_det_get_ses_by_out (dm, &in_addr, key.as_u64);
-  if (!ses)
-    {
-      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
-      goto send_reply;
-    }
-  snat_det_ses_close (dm, ses);
-
-send_reply:
-  REPLY_MACRO (VL_API_SNAT_DET_CLOSE_SESSION_OUT_REPLY);
-}
-
-static void *vl_api_snat_det_close_session_out_t_print
-  (vl_api_snat_det_close_session_out_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_det_close_session_out ");
-  s = format (s, "out_addr %U out_port %d "
-	      "ext_addr %U ext_port %d\n",
-	      format_ip4_address, mp->out_addr, ntohs (mp->out_port),
-	      format_ip4_address, mp->ext_addr, ntohs (mp->ext_port));
-
-  FINISH;
-}
-
-static void
-  vl_api_snat_det_close_session_in_t_handler
-  (vl_api_snat_det_close_session_in_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  vl_api_snat_det_close_session_in_reply_t *rmp;
-  ip4_address_t in_addr, ext_addr;
-  snat_det_out_key_t key;
-  snat_det_map_t *dm;
-  snat_det_session_t *ses;
-  int rv = 0;
-
-  clib_memcpy (&in_addr, mp->in_addr, 4);
-  clib_memcpy (&ext_addr, mp->ext_addr, 4);
-
-  dm = snat_det_map_by_user (sm, &in_addr);
-  if (!dm)
-    {
-      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
-      goto send_reply;
-    }
-  key.ext_host_addr = ext_addr;
-  key.ext_host_port = mp->ext_port;
-  ses = snat_det_find_ses_by_in (dm, &in_addr, mp->in_port, key);
-  if (!ses)
-    {
-      rv = VNET_API_ERROR_NO_SUCH_ENTRY;
-      goto send_reply;
-    }
-  snat_det_ses_close (dm, ses);
-
-send_reply:
-  REPLY_MACRO (VL_API_SNAT_DET_CLOSE_SESSION_OUT_REPLY);
-}
-
-static void *vl_api_snat_det_close_session_in_t_print
-  (vl_api_snat_det_close_session_in_t * mp, void *handle)
-{
-  u8 *s;
-  s = format (0, "SCRIPT: snat_det_close_session_in ");
-  s = format (s, "in_addr %U in_port %d "
-	      "ext_addr %U ext_port %d\n",
-	      format_ip4_address, mp->in_addr, ntohs (mp->in_port),
-	      format_ip4_address, mp->ext_addr, ntohs (mp->ext_port));
-
-  FINISH;
-}
-
-static void
-  send_snat_det_session_details
-  (snat_det_session_t * s, unix_shared_memory_queue_t * q, u32 context)
-{
-  vl_api_snat_det_session_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_SNAT_DET_SESSION_DETAILS + sm->msg_id_base);
-  rmp->is_ip4 = 1;
-  rmp->in_port = s->in_port;
-  clib_memcpy (rmp->ext_addr, &s->out.ext_host_addr, 4);
-  rmp->ext_port = s->out.ext_host_port;
-  rmp->out_port = s->out.out_port;
-  rmp->state = s->state;
-  rmp->expire = ntohl (s->expire);
-  rmp->context = context;
-
-  vl_msg_api_send_shmem (q, (u8 *) & rmp);
-}
-
-static void
-vl_api_snat_det_session_dump_t_handler (vl_api_snat_det_session_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  snat_main_t *sm = &snat_main;
-  ip4_address_t user_addr;
-  snat_det_map_t *dm;
-  snat_det_session_t *s, empty_ses;
-  u16 i;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-  if (!mp->is_ip4)
-    return;
-
-  memset (&empty_ses, 0, sizeof (empty_ses));
-  clib_memcpy (&user_addr, mp->user_addr, 4);
-  dm = snat_det_map_by_user (sm, &user_addr);
-  if (!dm)
-    return;
-
-  s = dm->sessions + snat_det_user_ses_offset (&user_addr, dm->in_plen);
-  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
-    {
-      if (s->out.as_u64)
-	send_snat_det_session_details (s, q, mp->context);
-      s++;
-    }
-}
-
-static void *vl_api_snat_det_session_dump_t_print
-  (vl_api_snat_det_session_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_det_session_dump ");
-  s = format (s, "user_addr %U\n", format_ip4_address, mp->user_addr);
-
-  FINISH;
-}
-
-/*************/
-/*** NAT64 ***/
-/*************/
-
-static void
-  vl_api_nat64_add_del_pool_addr_range_t_handler
-  (vl_api_nat64_add_del_pool_addr_range_t * mp)
-{
-  vl_api_nat64_add_del_pool_addr_range_reply_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_main_t *nm = &nat64_main;
-  int rv = 0;
-  ip4_address_t this_addr;
-  u32 start_host_order, end_host_order;
-  u32 vrf_id;
-  int i, count;
-  u32 *tmp;
-
-  if (nm->is_disabled)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  tmp = (u32 *) mp->start_addr;
-  start_host_order = clib_host_to_net_u32 (tmp[0]);
-  tmp = (u32 *) mp->end_addr;
-  end_host_order = clib_host_to_net_u32 (tmp[0]);
-
-  count = (end_host_order - start_host_order) + 1;
-
-  vrf_id = clib_host_to_net_u32 (mp->vrf_id);
-
-  memcpy (&this_addr.as_u8, mp->start_addr, 4);
-
-  for (i = 0; i < count; i++)
-    {
-      if ((rv = nat64_add_del_pool_addr (&this_addr, vrf_id, mp->is_add)))
-	goto send_reply;
-
-      increment_v4_address (&this_addr);
-    }
-
-send_reply:
-  REPLY_MACRO (VL_API_NAT64_ADD_DEL_POOL_ADDR_RANGE_REPLY);
-}
-
-static void *vl_api_nat64_add_del_pool_addr_range_t_print
-  (vl_api_nat64_add_del_pool_addr_range_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_add_del_pool_addr_range ");
-  s = format (s, "%U - %U vrf_id %u %s\n",
-	      format_ip4_address, mp->start_addr,
-	      format_ip4_address, mp->end_addr,
-	      ntohl (mp->vrf_id), mp->is_add ? "" : "del");
-
-  FINISH;
-}
-
-typedef struct nat64_api_walk_ctx_t_
-{
-  unix_shared_memory_queue_t *q;
-  u32 context;
-} nat64_api_walk_ctx_t;
-
-static int
-nat64_api_pool_walk (snat_address_t * a, void *arg)
-{
-  vl_api_nat64_pool_addr_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_api_walk_ctx_t *ctx = arg;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_NAT64_POOL_ADDR_DETAILS + sm->msg_id_base);
-  clib_memcpy (rmp->address, &(a->addr), 4);
-  if (a->fib_index != ~0)
-    {
-      fib_table_t *fib = fib_table_get (a->fib_index, FIB_PROTOCOL_IP6);
-      if (!fib)
-	return -1;
-      rmp->vrf_id = ntohl (fib->ft_table_id);
-    }
-  else
-    rmp->vrf_id = ~0;
-  rmp->context = ctx->context;
-
-  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
-
-  return 0;
-}
-
-static void
-vl_api_nat64_pool_addr_dump_t_handler (vl_api_nat64_pool_addr_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  nat64_api_walk_ctx_t ctx = {
-    .q = q,
-    .context = mp->context,
-  };
-
-  nat64_pool_addr_walk (nat64_api_pool_walk, &ctx);
-}
-
-static void *
-vl_api_nat64_pool_addr_dump_t_print (vl_api_nat64_pool_addr_dump_t * mp,
-				     void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_pool_addr_dump\n");
-
-  FINISH;
-}
-
-static void
-vl_api_nat64_add_del_interface_t_handler (vl_api_nat64_add_del_interface_t *
-					  mp)
-{
-  snat_main_t *sm = &snat_main;
-  nat64_main_t *nm = &nat64_main;
-  vl_api_nat64_add_del_interface_reply_t *rmp;
-  int rv = 0;
-
-  if (nm->is_disabled)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  VALIDATE_SW_IF_INDEX (mp);
-
-  rv =
-    nat64_add_del_interface (ntohl (mp->sw_if_index), mp->is_inside,
-			     mp->is_add);
-
-  BAD_SW_IF_INDEX_LABEL;
-
-send_reply:
-  REPLY_MACRO (VL_API_NAT64_ADD_DEL_INTERFACE_REPLY);
-}
-
-static void *
-vl_api_nat64_add_del_interface_t_print (vl_api_nat64_add_del_interface_t * mp,
-					void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_add_del_interface ");
-  s = format (s, "sw_if_index %d %s %s",
-	      clib_host_to_net_u32 (mp->sw_if_index),
-	      mp->is_inside ? "in" : "out", mp->is_add ? "" : "del");
-
-  FINISH;
-}
-
-static int
-nat64_api_interface_walk (snat_interface_t * i, void *arg)
-{
-  vl_api_nat64_interface_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_api_walk_ctx_t *ctx = arg;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_NAT64_INTERFACE_DETAILS + sm->msg_id_base);
-  rmp->sw_if_index = ntohl (i->sw_if_index);
-  rmp->is_inside = i->is_inside;
-  rmp->context = ctx->context;
-
-  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
-
-  return 0;
-}
-
-static void
-vl_api_nat64_interface_dump_t_handler (vl_api_nat64_interface_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  nat64_api_walk_ctx_t ctx = {
-    .q = q,
-    .context = mp->context,
-  };
-
-  nat64_interfaces_walk (nat64_api_interface_walk, &ctx);
-}
-
-static void *
-vl_api_nat64_interface_dump_t_print (vl_api_nat64_interface_dump_t * mp,
-				     void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_interface_dump ");
-
-  FINISH;
-}
-
-static void
-  vl_api_nat64_add_del_static_bib_t_handler
-  (vl_api_nat64_add_del_static_bib_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  nat64_main_t *nm = &nat64_main;
-  vl_api_nat64_add_del_static_bib_reply_t *rmp;
-  ip6_address_t in_addr;
-  ip4_address_t out_addr;
-  int rv = 0;
-
-  if (nm->is_disabled)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  memcpy (&in_addr.as_u8, mp->i_addr, 16);
-  memcpy (&out_addr.as_u8, mp->o_addr, 4);
-
-  rv =
-    nat64_add_del_static_bib_entry (&in_addr, &out_addr,
-				    clib_net_to_host_u16 (mp->i_port),
-				    clib_net_to_host_u16 (mp->o_port),
-				    mp->proto,
-				    clib_net_to_host_u32 (mp->vrf_id),
-				    mp->is_add);
-
-send_reply:
-  REPLY_MACRO (VL_API_NAT64_ADD_DEL_STATIC_BIB_REPLY);
-}
-
-static void *vl_api_nat64_add_del_static_bib_t_print
-  (vl_api_nat64_add_del_static_bib_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_add_del_static_bib ");
-  s = format (s, "protocol %d i_addr %U o_addr %U ",
-	      mp->proto,
-	      format_ip6_address, mp->i_addr, format_ip4_address, mp->o_addr);
-
-  if (mp->vrf_id != ~0)
-    s = format (s, "vrf %d", clib_net_to_host_u32 (mp->vrf_id));
-
-  FINISH;
-}
-
-static int
-nat64_api_bib_walk (nat64_db_bib_entry_t * bibe, void *arg)
-{
-  vl_api_nat64_bib_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_api_walk_ctx_t *ctx = arg;
-  fib_table_t *fib;
-
-  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  if (!fib)
-    return -1;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_NAT64_BIB_DETAILS + sm->msg_id_base);
-  rmp->context = ctx->context;
-  clib_memcpy (rmp->i_addr, &(bibe->in_addr), 16);
-  clib_memcpy (rmp->o_addr, &(bibe->out_addr), 4);
-  rmp->i_port = bibe->in_port;
-  rmp->o_port = bibe->out_port;
-  rmp->vrf_id = ntohl (fib->ft_table_id);
-  rmp->proto = bibe->proto;
-  rmp->is_static = bibe->is_static;
-  rmp->ses_num = ntohl (bibe->ses_num);
-
-  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
-
-  return 0;
-}
-
-static void
-vl_api_nat64_bib_dump_t_handler (vl_api_nat64_bib_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  nat64_api_walk_ctx_t ctx = {
-    .q = q,
-    .context = mp->context,
-  };
-
-  nat64_db_bib_walk (&nm->db, mp->proto, nat64_api_bib_walk, &ctx);
-}
-
-static void *
-vl_api_nat64_bib_dump_t_print (vl_api_nat64_bib_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_bib_dump protocol %d", mp->proto);
-
-  FINISH;
-}
-
-static void
-vl_api_nat64_set_timeouts_t_handler (vl_api_nat64_set_timeouts_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  nat64_main_t *nm = &nat64_main;
-  vl_api_nat64_set_timeouts_reply_t *rmp;
-  int rv = 0;
-
-  if (nm->is_disabled)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  rv = nat64_set_icmp_timeout (ntohl (mp->icmp));
-  if (rv)
-    goto send_reply;
-  rv = nat64_set_udp_timeout (ntohl (mp->udp));
-  if (rv)
-    goto send_reply;
-  rv =
-    nat64_set_tcp_timeouts (ntohl (mp->tcp_trans), ntohl (mp->tcp_est),
-			    ntohl (mp->tcp_incoming_syn));
-
-send_reply:
-  REPLY_MACRO (VL_API_NAT64_SET_TIMEOUTS_REPLY);
-}
-
-static void *vl_api_nat64_set_timeouts_t_print
-  (vl_api_nat64_set_timeouts_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_set_timeouts ");
-  s =
-    format (s,
-	    "udp %d icmp %d, tcp_trans %d, tcp_est %d, tcp_incoming_syn %d\n",
-	    ntohl (mp->udp), ntohl (mp->icmp), ntohl (mp->tcp_trans),
-	    ntohl (mp->tcp_est), ntohl (mp->tcp_incoming_syn));
-
-  FINISH;
-}
-
-static void
-vl_api_nat64_get_timeouts_t_handler (vl_api_nat64_get_timeouts_t * mp)
-{
-  snat_main_t *sm = &snat_main;
-  nat64_main_t *nm = &nat64_main;
-  vl_api_nat64_get_timeouts_reply_t *rmp;
-  int rv = 0;
-
-  if (nm->is_disabled)
-    return;
-
-  /* *INDENT-OFF* */
-  REPLY_MACRO2 (VL_API_NAT64_GET_TIMEOUTS_REPLY,
-  ({
-    rmp->udp = htonl (nat64_get_udp_timeout());
-    rmp->icmp = htonl (nat64_get_icmp_timeout());
-    rmp->tcp_trans = htonl (nat64_get_tcp_trans_timeout());
-    rmp->tcp_est = htonl (nat64_get_tcp_est_timeout());
-    rmp->tcp_incoming_syn = htonl (nat64_get_tcp_incoming_syn_timeout());
-  }))
-  /* *INDENT-ON* */
-}
-
-static void *vl_api_nat64_get_timeouts_t_print
-  (vl_api_nat64_get_timeouts_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_get_timeouts");
-
-  FINISH;
-}
-
-static int
-nat64_api_st_walk (nat64_db_st_entry_t * ste, void *arg)
-{
-  vl_api_nat64_st_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_api_walk_ctx_t *ctx = arg;
-  nat64_main_t *nm = &nat64_main;
-  nat64_db_bib_entry_t *bibe;
-  fib_table_t *fib;
-
-  bibe = nat64_db_bib_entry_by_index (&nm->db, ste->proto, ste->bibe_index);
-  if (!bibe)
-    return -1;
-
-  fib = fib_table_get (bibe->fib_index, FIB_PROTOCOL_IP6);
-  if (!fib)
-    return -1;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_NAT64_ST_DETAILS + sm->msg_id_base);
-  rmp->context = ctx->context;
-  clib_memcpy (rmp->il_addr, &(bibe->in_addr), 16);
-  clib_memcpy (rmp->ol_addr, &(bibe->out_addr), 4);
-  rmp->il_port = bibe->in_port;
-  rmp->ol_port = bibe->out_port;
-  clib_memcpy (rmp->ir_addr, &(ste->in_r_addr), 16);
-  clib_memcpy (rmp->or_addr, &(ste->out_r_addr), 4);
-  rmp->il_port = ste->r_port;
-  rmp->vrf_id = ntohl (fib->ft_table_id);
-  rmp->proto = ste->proto;
-
-  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
-
-  return 0;
-}
-
-static void
-vl_api_nat64_st_dump_t_handler (vl_api_nat64_st_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  nat64_api_walk_ctx_t ctx = {
-    .q = q,
-    .context = mp->context,
-  };
-
-  nat64_db_st_walk (&nm->db, mp->proto, nat64_api_st_walk, &ctx);
-}
-
-static void *
-vl_api_nat64_st_dump_t_print (vl_api_nat64_st_dump_t * mp, void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: snat_st_dump protocol %d", mp->proto);
-
-  FINISH;
-}
-
-static void
-vl_api_nat64_add_del_prefix_t_handler (vl_api_nat64_add_del_prefix_t * mp)
-{
-  vl_api_nat64_add_del_prefix_reply_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_main_t *nm = &nat64_main;
-  ip6_address_t prefix;
-  int rv = 0;
-
-  if (nm->is_disabled)
-    {
-      rv = VNET_API_ERROR_FEATURE_DISABLED;
-      goto send_reply;
-    }
-
-  memcpy (&prefix.as_u8, mp->prefix, 16);
-
-  rv =
-    nat64_add_del_prefix (&prefix, mp->prefix_len,
-			  clib_net_to_host_u32 (mp->vrf_id), mp->is_add);
-send_reply:
-  REPLY_MACRO (VL_API_NAT64_ADD_DEL_PREFIX_REPLY);
-}
-
-static void *
-vl_api_nat64_add_del_prefix_t_print (vl_api_nat64_add_del_prefix_t * mp,
-				     void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_add_del_prefix %U/%u vrf_id %u %s\n",
-	      format_ip6_address, mp->prefix, mp->prefix_len,
-	      ntohl (mp->vrf_id), mp->is_add ? "" : "del");
-
-  FINISH;
-}
-
-static int
-nat64_api_prefix_walk (nat64_prefix_t * p, void *arg)
-{
-  vl_api_nat64_prefix_details_t *rmp;
-  snat_main_t *sm = &snat_main;
-  nat64_api_walk_ctx_t *ctx = arg;
-
-  rmp = vl_msg_api_alloc (sizeof (*rmp));
-  memset (rmp, 0, sizeof (*rmp));
-  rmp->_vl_msg_id = ntohs (VL_API_NAT64_PREFIX_DETAILS + sm->msg_id_base);
-  clib_memcpy (rmp->prefix, &(p->prefix), 16);
-  rmp->prefix_len = p->plen;
-  rmp->vrf_id = ntohl (p->vrf_id);
-  rmp->context = ctx->context;
-
-  vl_msg_api_send_shmem (ctx->q, (u8 *) & rmp);
-
-  return 0;
-}
-
-static void
-vl_api_nat64_prefix_dump_t_handler (vl_api_nat64_prefix_dump_t * mp)
-{
-  unix_shared_memory_queue_t *q;
-  nat64_main_t *nm = &nat64_main;
-
-  if (nm->is_disabled)
-    return;
-
-  q = vl_api_client_index_to_input_queue (mp->client_index);
-  if (q == 0)
-    return;
-
-  nat64_api_walk_ctx_t ctx = {
-    .q = q,
-    .context = mp->context,
-  };
-
-  nat64_prefix_walk (nat64_api_prefix_walk, &ctx);
-}
-
-static void *
-vl_api_nat64_prefix_dump_t_print (vl_api_nat64_prefix_dump_t * mp,
-				  void *handle)
-{
-  u8 *s;
-
-  s = format (0, "SCRIPT: nat64_prefix_dump\n");
-
-  FINISH;
-}
-
-/* List of message types that this plugin understands */
-#define foreach_snat_plugin_api_msg                                     \
-_(SNAT_ADD_ADDRESS_RANGE, snat_add_address_range)                       \
-_(SNAT_INTERFACE_ADD_DEL_FEATURE, snat_interface_add_del_feature)       \
-_(SNAT_ADD_STATIC_MAPPING, snat_add_static_mapping)                     \
-_(SNAT_CONTROL_PING, snat_control_ping)                                 \
-_(SNAT_STATIC_MAPPING_DUMP, snat_static_mapping_dump)                   \
-_(SNAT_SHOW_CONFIG, snat_show_config)                                   \
-_(SNAT_ADDRESS_DUMP, snat_address_dump)                                 \
-_(SNAT_INTERFACE_DUMP, snat_interface_dump)                             \
-_(SNAT_SET_WORKERS, snat_set_workers)                                   \
-_(SNAT_WORKER_DUMP, snat_worker_dump)                                   \
-_(SNAT_ADD_DEL_INTERFACE_ADDR, snat_add_del_interface_addr)             \
-_(SNAT_INTERFACE_ADDR_DUMP, snat_interface_addr_dump)                   \
-_(SNAT_IPFIX_ENABLE_DISABLE, snat_ipfix_enable_disable)                 \
-_(SNAT_USER_DUMP, snat_user_dump)                                       \
-_(SNAT_USER_SESSION_DUMP, snat_user_session_dump)                       \
-_(SNAT_INTERFACE_ADD_DEL_OUTPUT_FEATURE,                                \
-  snat_interface_add_del_output_feature)                                \
-_(SNAT_INTERFACE_OUTPUT_FEATURE_DUMP,                                   \
-  snat_interface_output_feature_dump)                                   \
-_(SNAT_ADD_DET_MAP, snat_add_det_map)                                   \
-_(SNAT_DET_FORWARD, snat_det_forward)                                   \
-_(SNAT_DET_REVERSE, snat_det_reverse)                                   \
-_(SNAT_DET_MAP_DUMP, snat_det_map_dump)                                 \
-_(SNAT_DET_SET_TIMEOUTS, snat_det_set_timeouts)                         \
-_(SNAT_DET_GET_TIMEOUTS, snat_det_get_timeouts)                         \
-_(SNAT_DET_CLOSE_SESSION_OUT, snat_det_close_session_out)               \
-_(SNAT_DET_CLOSE_SESSION_IN, snat_det_close_session_in)                 \
-_(SNAT_DET_SESSION_DUMP, snat_det_session_dump)                         \
-_(NAT64_ADD_DEL_POOL_ADDR_RANGE, nat64_add_del_pool_addr_range)         \
-_(NAT64_POOL_ADDR_DUMP, nat64_pool_addr_dump)                           \
-_(NAT64_ADD_DEL_INTERFACE, nat64_add_del_interface)                     \
-_(NAT64_INTERFACE_DUMP, nat64_interface_dump)                           \
-_(NAT64_ADD_DEL_STATIC_BIB, nat64_add_del_static_bib)                   \
-_(NAT64_BIB_DUMP, nat64_bib_dump)                                       \
-_(NAT64_SET_TIMEOUTS, nat64_set_timeouts)                               \
-_(NAT64_GET_TIMEOUTS, nat64_get_timeouts)                               \
-_(NAT64_ST_DUMP, nat64_st_dump)                                         \
-_(NAT64_ADD_DEL_PREFIX, nat64_add_del_prefix)                           \
-_(NAT64_PREFIX_DUMP, nat64_prefix_dump)
-
-/* Set up the API message handling tables */
-static clib_error_t *
-snat_plugin_api_hookup (vlib_main_t * vm)
-{
-  snat_main_t *sm __attribute__ ((unused)) = &snat_main;
-#define _(N,n)                                                  \
-    vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),     \
-                           #n,					\
-                           vl_api_##n##_t_handler,              \
-                           vl_noop_handler,                     \
-                           vl_api_##n##_t_endian,               \
-                           vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
-  foreach_snat_plugin_api_msg;
-#undef _
-
-  return 0;
-}
-
-#define vl_msg_name_crc_list
-#include 
-#undef vl_msg_name_crc_list
-
-static void
-setup_message_id_table (snat_main_t * sm, api_main_t * am)
-{
-#define _(id,n,crc) \
-  vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + sm->msg_id_base);
-  foreach_vl_msg_name_crc_snat;
-#undef _
-}
-
-static void
-plugin_custom_dump_configure (snat_main_t * sm)
-{
-#define _(n,f) sm->api_main->msg_print_handlers \
-  [VL_API_##n + sm->msg_id_base]                \
-    = (void *) vl_api_##f##_t_print;
-  foreach_snat_plugin_api_msg;
-#undef _
-}
-
-clib_error_t *
-snat_api_init (vlib_main_t * vm, snat_main_t * sm)
-{
-  u8 *name;
-  clib_error_t *error = 0;
-
-  name = format (0, "snat_%08x%c", api_version, 0);
-
-  /* Ask for a correctly-sized block of API message decode slots */
-  sm->msg_id_base =
-    vl_msg_api_get_msg_ids ((char *) name, VL_MSG_FIRST_AVAILABLE);
-
-  error = snat_plugin_api_hookup (vm);
-
-  /* Add our API messages to the global name_crc hash table */
-  setup_message_id_table (sm, sm->api_main);
-
-  plugin_custom_dump_configure (sm);
-
-  vec_free (name);
-
-  return error;
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/snat_det.c b/src/plugins/snat/snat_det.c
deleted file mode 100644
index 2d6fce85..00000000
--- a/src/plugins/snat/snat_det.c
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * snat_det.c - deterministic NAT
- *
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief deterministic NAT
- */
-
-#include 
-
-
-/**
- * @brief Add/delete deterministic NAT mapping.
- *
- * Create bijective mapping of inside address to outside address and port range
- * pairs, with the purpose of enabling deterministic NAT to reduce logging in
- * CGN deployments.
- *
- * @param sm       SNAT main.
- * @param in_addr  Inside network address.
- * @param in_plen  Inside network prefix length.
- * @param out_addr Outside network address.
- * @param out_plen Outside network prefix length.
- * @param is_add   If 0 delete, otherwise add.
- */
-int
-snat_det_add_map (snat_main_t * sm, ip4_address_t * in_addr, u8 in_plen,
-		  ip4_address_t * out_addr, u8 out_plen, int is_add)
-{
-  snat_det_map_t *det_map;
-  static snat_det_session_t empty_snat_det_session = { 0 };
-  snat_interface_t *i;
-  ip4_address_t in_cmp, out_cmp;
-  u8 found = 0;
-
-  in_cmp.as_u32 = in_addr->as_u32 & ip4_main.fib_masks[in_plen];
-  out_cmp.as_u32 = out_addr->as_u32 & ip4_main.fib_masks[out_plen];
-  vec_foreach (det_map, sm->det_maps)
-  {
-    /* Checking for overlapping addresses to be added here */
-    if (det_map->in_addr.as_u32 == in_cmp.as_u32 &&
-	det_map->in_plen == in_plen &&
-	det_map->out_addr.as_u32 == out_cmp.as_u32 &&
-	det_map->out_plen == out_plen)
-      {
-	found = 1;
-	break;
-      }
-  }
-
-  /* If found, don't add again */
-  if (found && is_add)
-    return VNET_API_ERROR_VALUE_EXIST;
-
-  /* If not found, don't delete */
-  if (!found && !is_add)
-    return VNET_API_ERROR_NO_SUCH_ENTRY;
-
-  if (is_add)
-    {
-      pool_get (sm->det_maps, det_map);
-      memset (det_map, 0, sizeof (*det_map));
-      det_map->in_addr.as_u32 = in_cmp.as_u32;
-      det_map->in_plen = in_plen;
-      det_map->out_addr.as_u32 = out_cmp.as_u32;
-      det_map->out_plen = out_plen;
-      det_map->sharing_ratio = (1 << (32 - in_plen)) / (1 << (32 - out_plen));
-      det_map->ports_per_host = (65535 - 1023) / det_map->sharing_ratio;
-
-      vec_validate_init_empty (det_map->sessions,
-			       SNAT_DET_SES_PER_USER * (1 << (32 - in_plen)) -
-			       1, empty_snat_det_session);
-    }
-  else
-    {
-      vec_free (det_map->sessions);
-      vec_del1 (sm->det_maps, det_map - sm->det_maps);
-    }
-
-  /* Add/del external address range to FIB */
-  /* *INDENT-OFF* */
-  pool_foreach (i, sm->interfaces,
-  ({
-    if (i->is_inside)
-      continue;
-
-    snat_add_del_addr_to_fib(out_addr, out_plen, i->sw_if_index, is_add);
-    break;
-  }));
-  /* *INDENT-ON* */
-  return 0;
-}
-
-/**
- * @brief The 'snat-det-expire-walk' process's main loop.
- *
- * Check expire time for active sessions.
- */
-static uword
-snat_det_expire_walk_fn (vlib_main_t * vm, vlib_node_runtime_t * rt,
-			 vlib_frame_t * f)
-{
-  snat_main_t *sm = &snat_main;
-  snat_det_map_t *dm;
-  snat_det_session_t *ses;
-
-  while (sm->deterministic)
-    {
-      vlib_process_wait_for_event_or_clock (vm, 10.0);
-      vlib_process_get_events (vm, NULL);
-      u32 now = (u32) vlib_time_now (vm);
-      /* *INDENT-OFF* */
-      pool_foreach (dm, sm->det_maps,
-      ({
-        vec_foreach(ses, dm->sessions)
-          {
-            /* Delete if session expired */
-            if (ses->in_port && (ses->expire < now))
-              snat_det_ses_close (dm, ses);
-          }
-      }));
-      /* *INDENT-ON* */
-    }
-
-  return 0;
-}
-
-static vlib_node_registration_t snat_det_expire_walk_node;
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (snat_det_expire_walk_node, static) = {
-    .function = snat_det_expire_walk_fn,
-    .type = VLIB_NODE_TYPE_PROCESS,
-    .name =
-    "snat-det-expire-walk",
-};
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/snat_det.h b/src/plugins/snat/snat_det.h
deleted file mode 100644
index f4fdb256..00000000
--- a/src/plugins/snat/snat_det.h
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * snat_det.h - deterministic nat definitions
- *
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file
- * @brief deterministic NAT definitions
- */
-
-#ifndef __included_snat_det_h__
-#define __included_snat_det_h__
-
-#include 
-#include 
-#include 
-
-
-#define SNAT_DET_SES_PER_USER 1000
-
-
-int snat_det_add_map (snat_main_t * sm, ip4_address_t * in_addr, u8 in_plen,
-		      ip4_address_t * out_addr, u8 out_plen, int is_add);
-
-always_inline int
-is_addr_in_net (ip4_address_t * addr, ip4_address_t * net, u8 plen)
-{
-  if (net->as_u32 == (addr->as_u32 & ip4_main.fib_masks[plen]))
-    return 1;
-  return 0;
-}
-
-always_inline snat_det_map_t *
-snat_det_map_by_user (snat_main_t * sm, ip4_address_t * user_addr)
-{
-  snat_det_map_t *dm;
-
-  /* *INDENT-OFF* */
-  pool_foreach (dm, sm->det_maps,
-  ({
-    if (is_addr_in_net(user_addr, &dm->in_addr, dm->in_plen))
-      return dm;
-  }));
-  /* *INDENT-ON* */
-  return 0;
-}
-
-always_inline snat_det_map_t *
-snat_det_map_by_out (snat_main_t * sm, ip4_address_t * out_addr)
-{
-  snat_det_map_t *dm;
-
-  /* *INDENT-OFF* */
-  pool_foreach (dm, sm->det_maps,
-  ({
-    if (is_addr_in_net(out_addr, &dm->out_addr, dm->out_plen))
-      return dm;
-  }));
-  /* *INDENT-ON* */
-  return 0;
-}
-
-always_inline void
-snat_det_forward (snat_det_map_t * dm, ip4_address_t * in_addr,
-		  ip4_address_t * out_addr, u16 * lo_port)
-{
-  u32 in_offset, out_offset;
-
-  in_offset = clib_net_to_host_u32 (in_addr->as_u32) -
-    clib_net_to_host_u32 (dm->in_addr.as_u32);
-  out_offset = in_offset / dm->sharing_ratio;
-  out_addr->as_u32 =
-    clib_host_to_net_u32 (clib_net_to_host_u32 (dm->out_addr.as_u32) +
-			  out_offset);
-  *lo_port = 1024 + dm->ports_per_host * (in_offset % dm->sharing_ratio);
-}
-
-always_inline void
-snat_det_reverse (snat_det_map_t * dm, ip4_address_t * out_addr, u16 out_port,
-		  ip4_address_t * in_addr)
-{
-  u32 in_offset1, in_offset2, out_offset;
-
-  out_offset = clib_net_to_host_u32 (out_addr->as_u32) -
-    clib_net_to_host_u32 (dm->out_addr.as_u32);
-  in_offset1 = out_offset * dm->sharing_ratio;
-  in_offset2 = (out_port - 1024) / dm->ports_per_host;
-  in_addr->as_u32 =
-    clib_host_to_net_u32 (clib_net_to_host_u32 (dm->in_addr.as_u32) +
-			  in_offset1 + in_offset2);
-}
-
-always_inline u32
-snat_det_user_ses_offset (ip4_address_t * addr, u8 plen)
-{
-  return (clib_net_to_host_u32 (addr->as_u32) & pow2_mask (32 - plen)) *
-    SNAT_DET_SES_PER_USER;
-}
-
-always_inline snat_det_session_t *
-snat_det_get_ses_by_out (snat_det_map_t * dm, ip4_address_t * in_addr,
-			 u64 out_key)
-{
-  u32 user_offset;
-  u16 i;
-
-  user_offset = snat_det_user_ses_offset (in_addr, dm->in_plen);
-  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
-    {
-      if (dm->sessions[i + user_offset].out.as_u64 == out_key)
-	return &dm->sessions[i + user_offset];
-    }
-
-  return 0;
-}
-
-always_inline snat_det_session_t *
-snat_det_find_ses_by_in (snat_det_map_t * dm, ip4_address_t * in_addr,
-			 u16 in_port, snat_det_out_key_t out_key)
-{
-  snat_det_session_t *ses;
-  u32 user_offset;
-  u16 i;
-
-  user_offset = snat_det_user_ses_offset (in_addr, dm->in_plen);
-  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
-    {
-      ses = &dm->sessions[i + user_offset];
-      if (ses->in_port == in_port &&
-	  ses->out.ext_host_addr.as_u32 == out_key.ext_host_addr.as_u32 &&
-	  ses->out.ext_host_port == out_key.ext_host_port)
-	return &dm->sessions[i + user_offset];
-    }
-
-  return 0;
-}
-
-always_inline snat_det_session_t *
-snat_det_ses_create (snat_det_map_t * dm, ip4_address_t * in_addr,
-		     u16 in_port, snat_det_out_key_t * out)
-{
-  u32 user_offset;
-  u16 i;
-
-  user_offset = snat_det_user_ses_offset (in_addr, dm->in_plen);
-
-  for (i = 0; i < SNAT_DET_SES_PER_USER; i++)
-    {
-      if (!dm->sessions[i + user_offset].in_port)
-	{
-	  if (__sync_bool_compare_and_swap
-	      (&dm->sessions[i + user_offset].in_port, 0, in_port))
-	    {
-	      dm->sessions[i + user_offset].out.as_u64 = out->as_u64;
-	      dm->sessions[i + user_offset].state = SNAT_SESSION_UNKNOWN;
-	      dm->sessions[i + user_offset].expire = 0;
-	      __sync_add_and_fetch (&dm->ses_num, 1);
-	      return &dm->sessions[i + user_offset];
-	    }
-	}
-    }
-
-  snat_ipfix_logging_max_entries_per_user (in_addr->as_u32);
-  return 0;
-}
-
-always_inline void
-snat_det_ses_close (snat_det_map_t * dm, snat_det_session_t * ses)
-{
-  if (__sync_bool_compare_and_swap (&ses->in_port, ses->in_port, 0))
-    {
-      ses->out.as_u64 = 0;
-      __sync_add_and_fetch (&dm->ses_num, -1);
-    }
-}
-
-#endif /* __included_snat_det_h__ */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/plugins/snat/snat_ipfix_logging.c b/src/plugins/snat/snat_ipfix_logging.c
deleted file mode 100644
index c68dc540..00000000
--- a/src/plugins/snat/snat_ipfix_logging.c
+++ /dev/null
@@ -1,835 +0,0 @@
-/*
- * snat_ipfix_logging.c - NAT Events IPFIX logging
- *
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-#include 
-#include 
-#include 
-
-snat_ipfix_logging_main_t snat_ipfix_logging_main;
-
-#define NAT44_SESSION_CREATE_LEN 26
-#define NAT_ADDRESSES_EXHAUTED_LEN 13
-#define MAX_ENTRIES_PER_USER_LEN 17
-
-#define NAT44_SESSION_CREATE_FIELD_COUNT 8
-#define NAT_ADDRESSES_EXHAUTED_FIELD_COUNT 3
-#define MAX_ENTRIES_PER_USER_FIELD_COUNT 4
-
-typedef struct {
-  u8 nat_event;
-  u32 src_ip;
-  u32 nat_src_ip;
-  snat_protocol_t snat_proto;
-  u16 src_port;
-  u16 nat_src_port;
-  u32 vrf_id;
-} snat_ipfix_logging_nat44_ses_args_t;
-
-typedef struct {
-  u32 pool_id;
-} snat_ipfix_logging_addr_exhausted_args_t;
-
-typedef struct {
-  u32 src_ip;
-} snat_ipfix_logging_max_entries_per_user_args_t;
-
-/**
- * @brief Create an IPFIX template packet rewrite string
- *
- * @param frm               flow report main
- * @param fr                flow report
- * @param collector_address collector address
- * @param src_address       source address
- * @param collector_port    collector
- * @param event             NAT event ID
- * @param quota_event       NAT quota exceeded event ID
- *
- * @returns template packet
- */
-static inline u8 *
-snat_template_rewrite (flow_report_main_t * frm,
-                       flow_report_t * fr,
-                       ip4_address_t * collector_address,
-                       ip4_address_t * src_address,
-                       u16 collector_port,
-                       nat_event_t event,
-                       quota_exceed_event_t quota_event)
-{
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-  ip4_header_t *ip;
-  udp_header_t *udp;
-  ipfix_message_header_t *h;
-  ipfix_set_header_t *s;
-  ipfix_template_header_t *t;
-  ipfix_field_specifier_t *f;
-  ipfix_field_specifier_t *first_field;
-  u8 *rewrite = 0;
-  ip4_ipfix_template_packet_t *tp;
-  u32 field_count = 0;
-  flow_report_stream_t *stream;
-
-  stream = &frm->streams[fr->stream_index];
-  silm->stream_index = fr->stream_index;
-
-  if (event == NAT_ADDRESSES_EXHAUTED)
-    {
-      field_count = NAT_ADDRESSES_EXHAUTED_FIELD_COUNT;
-      silm->addr_exhausted_template_id = fr->template_id;
-    }
-  else if (event == NAT44_SESSION_CREATE)
-    {
-      field_count = NAT44_SESSION_CREATE_FIELD_COUNT;
-      silm->nat44_session_template_id = fr->template_id;
-    }
-  else if (event == QUOTA_EXCEEDED)
-    {
-      if (quota_event == MAX_ENTRIES_PER_USER)
-        {
-          field_count = MAX_ENTRIES_PER_USER_FIELD_COUNT;
-          silm->max_entries_per_user_template_id = fr->template_id;
-        }
-    }
-
-  /* allocate rewrite space */
-  vec_validate_aligned (rewrite,
-			sizeof (ip4_ipfix_template_packet_t)
-			+ field_count * sizeof (ipfix_field_specifier_t) - 1,
-			CLIB_CACHE_LINE_BYTES);
-
-  tp = (ip4_ipfix_template_packet_t *) rewrite;
-  ip = (ip4_header_t *) & tp->ip4;
-  udp = (udp_header_t *) (ip + 1);
-  h = (ipfix_message_header_t *) (udp + 1);
-  s = (ipfix_set_header_t *) (h + 1);
-  t = (ipfix_template_header_t *) (s + 1);
-  first_field = f = (ipfix_field_specifier_t *) (t + 1);
-
-  ip->ip_version_and_header_length = 0x45;
-  ip->ttl = 254;
-  ip->protocol = IP_PROTOCOL_UDP;
-  ip->src_address.as_u32 = src_address->as_u32;
-  ip->dst_address.as_u32 = collector_address->as_u32;
-  udp->src_port = clib_host_to_net_u16 (stream->src_port);
-  udp->dst_port = clib_host_to_net_u16 (collector_port);
-  udp->length = clib_host_to_net_u16 (vec_len (rewrite) - sizeof (*ip));
-
-  /* FIXUP: message header export_time */
-  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
-
-  /* Add TLVs to the template */
-  if (event == NAT_ADDRESSES_EXHAUTED)
-    {
-      f->e_id_length = ipfix_e_id_length (0, observationTimeMilliseconds, 8);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, natEvent, 1);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, natPoolId, 4);
-      f++;
-    }
-  else if (event == NAT44_SESSION_CREATE)
-    {
-      f->e_id_length = ipfix_e_id_length (0, observationTimeMilliseconds, 8);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, natEvent, 1);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, sourceIPv4Address, 4);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, postNATSourceIPv4Address, 4);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, protocolIdentifier, 1);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, sourceTransportPort, 2);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, postNAPTSourceTransportPort, 2);
-      f++;
-      f->e_id_length = ipfix_e_id_length (0, ingressVRFID, 4);
-      f++;
-    }
-  else if (event == QUOTA_EXCEEDED)
-    {
-      if (quota_event == MAX_ENTRIES_PER_USER)
-        {
-          f->e_id_length = ipfix_e_id_length (0, observationTimeMilliseconds,
-                                              8);
-          f++;
-          f->e_id_length = ipfix_e_id_length (0, natEvent, 1);
-          f++;
-          f->e_id_length = ipfix_e_id_length (0, natQuotaExceededEvent, 4);
-          f++;
-          f->e_id_length = ipfix_e_id_length (0, sourceIPv4Address, 4);
-          f++;
-        }
-    }
-
-  /* Back to the template packet... */
-  ip = (ip4_header_t *) & tp->ip4;
-  udp = (udp_header_t *) (ip + 1);
-
-  ASSERT (f - first_field);
-  /* Field count in this template */
-  t->id_count = ipfix_id_count (fr->template_id, f - first_field);
-
-  /* set length in octets */
-  s->set_id_length =
-    ipfix_set_id_length (2 /* set_id */ , (u8 *) f - (u8 *) s);
-
-  /* message length in octets */
-  h->version_length = version_length ((u8 *) f - (u8 *) h);
-
-  ip->length = clib_host_to_net_u16 ((u8 *) f - (u8 *) ip);
-  ip->checksum = ip4_header_checksum (ip);
-
-  return rewrite;
-}
-
-u8 *
-snat_template_rewrite_addr_exhausted (flow_report_main_t * frm,
-                                      flow_report_t * fr,
-                                      ip4_address_t * collector_address,
-                                      ip4_address_t * src_address,
-                                      u16 collector_port)
-{
-  return snat_template_rewrite (frm, fr, collector_address, src_address,
-                                collector_port, NAT_ADDRESSES_EXHAUTED, 0);
-}
-
-u8 *
-snat_template_rewrite_nat44_session (flow_report_main_t * frm,
-                                     flow_report_t * fr,
-                                     ip4_address_t * collector_address,
-                                     ip4_address_t * src_address,
-                                     u16 collector_port)
-{
-  return snat_template_rewrite (frm, fr, collector_address, src_address,
-                                collector_port, NAT44_SESSION_CREATE, 0);
-}
-
-u8 *
-snat_template_rewrite_max_entries_per_usr (flow_report_main_t * frm,
-                                           flow_report_t * fr,
-                                           ip4_address_t * collector_address,
-                                           ip4_address_t * src_address,
-                                           u16 collector_port)
-{
-  return snat_template_rewrite (frm, fr, collector_address, src_address,
-                                collector_port, QUOTA_EXCEEDED,
-                                MAX_ENTRIES_PER_USER);
-}
-
-static inline void
-snat_ipfix_header_create (flow_report_main_t * frm,
-                          vlib_buffer_t * b0,
-                          u32 * offset)
-{
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-  flow_report_stream_t *stream;
-  ip4_ipfix_template_packet_t * tp;
-  ipfix_message_header_t * h = 0;
-  ipfix_set_header_t * s = 0;
-  ip4_header_t * ip;
-  udp_header_t * udp;
-
-  stream = &frm->streams[silm->stream_index];
-
-  b0->current_data = 0;
-  b0->current_length = sizeof (*ip) + sizeof (*udp) + sizeof (*h) +
-                       sizeof (*s);
-  b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_FLOW_REPORT);
-  vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
-  vnet_buffer (b0)->sw_if_index[VLIB_TX] = frm->fib_index;
-  tp = vlib_buffer_get_current (b0);
-  ip = (ip4_header_t *) &tp->ip4;
-  udp = (udp_header_t *) (ip+1);
-  h = (ipfix_message_header_t *)(udp+1);
-  s = (ipfix_set_header_t *)(h+1);
-
-  ip->ip_version_and_header_length = 0x45;
-  ip->ttl = 254;
-  ip->protocol = IP_PROTOCOL_UDP;
-  ip->flags_and_fragment_offset = 0;
-  ip->src_address.as_u32 = frm->src_address.as_u32;
-  ip->dst_address.as_u32 = frm->ipfix_collector.as_u32;
-  udp->src_port = clib_host_to_net_u16 (stream->src_port);
-  udp->dst_port = clib_host_to_net_u16 (frm->collector_port);
-  udp->checksum = 0;
-
-  h->export_time = clib_host_to_net_u32 (
-    (u32) (((f64)frm->unix_time_0) + (vlib_time_now(frm->vlib_main) -
-    frm->vlib_time_0)));
-  h->sequence_number = clib_host_to_net_u32 (stream->sequence_number++);
-  h->domain_id = clib_host_to_net_u32 (stream->domain_id);
-
-  *offset = (u32) (((u8 *)(s+1)) - (u8 *)tp);
-}
-
-static inline void
-snat_ipfix_send (flow_report_main_t * frm,
-                 vlib_frame_t * f,
-                 vlib_buffer_t * b0,
-                 u16 template_id)
-{
-  ip4_ipfix_template_packet_t * tp;
-  ipfix_message_header_t * h = 0;
-  ipfix_set_header_t * s = 0;
-  ip4_header_t * ip;
-  udp_header_t * udp;
-  vlib_main_t * vm = frm->vlib_main;
-
-  tp = vlib_buffer_get_current (b0);
-  ip = (ip4_header_t *) & tp->ip4;
-  udp = (udp_header_t *) (ip + 1);
-  h = (ipfix_message_header_t *) (udp + 1);
-  s = (ipfix_set_header_t *) (h + 1);
-
-  s->set_id_length = ipfix_set_id_length (template_id,
-                                          b0->current_length -
-                                          (sizeof (*ip) + sizeof (*udp) +
-                                           sizeof (*h)));
-  h->version_length = version_length (b0->current_length -
-                                      (sizeof (*ip) + sizeof (*udp)));
-
-  ip->length = clib_host_to_net_u16 (b0->current_length);
-  ip->checksum = ip4_header_checksum (ip);
-  udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
-
-  if (frm->udp_checksum)
-    {
-      udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
-      if (udp->checksum == 0)
-        udp->checksum = 0xffff;
-    }
-
-  ASSERT (ip->checksum == ip4_header_checksum (ip));
-
-  vlib_put_frame_to_node (vm, ip4_lookup_node.index, f);
-}
-
-static void
-snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip,
-                              snat_protocol_t snat_proto, u16 src_port,
-                              u16 nat_src_port, u32 vrf_id, int do_flush)
-{
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-  flow_report_main_t *frm = &flow_report_main;
-  vlib_frame_t *f;
-  vlib_buffer_t *b0 = 0;
-  u32 bi0 = ~0;
-  u32 offset;
-  vlib_main_t * vm = frm->vlib_main;
-  u64 now;
-  vlib_buffer_free_list_t *fl;
-  u8 proto = ~0;
-
-  if (!silm->enabled)
-    return;
-
-  proto = snat_proto_to_ip_proto (snat_proto);
-
-  now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
-  now += silm->milisecond_time_0;
-
-  b0 = silm->nat44_session_buffer;
-
-  if (PREDICT_FALSE (b0 == 0))
-    {
-      if (do_flush)
-        return;
-
-      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
-        {
-          clib_warning ("can't allocate buffer for NAT IPFIX event");
-          return;
-        }
-
-      b0 = silm->nat44_session_buffer =
-        vlib_get_buffer (vm, bi0);
-      fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-      vlib_buffer_init_for_free_list (b0, fl);
-      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-      offset = 0;
-    }
-  else
-    {
-      bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->nat44_session_next_record_offset;
-    }
-
-  f = silm->nat44_session_frame;
-  if (PREDICT_FALSE (f == 0))
-    {
-      u32 * to_next;
-      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->nat44_session_frame = f;
-      to_next = vlib_frame_vector_args (f);
-      to_next[0] = bi0;
-      f->n_vectors = 1;
-    }
-
-  if (PREDICT_FALSE (offset == 0))
-    snat_ipfix_header_create (frm, b0, &offset);
-
-  if (PREDICT_TRUE (do_flush == 0))
-    {
-      u64 time_stamp = clib_host_to_net_u64 (now);
-      clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
-      offset += sizeof (time_stamp);
-
-      clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event));
-      offset += sizeof (nat_event);
-
-      clib_memcpy (b0->data + offset, &src_ip, sizeof (src_ip));
-      offset += sizeof (src_ip);
-
-      clib_memcpy (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip));
-      offset += sizeof (nat_src_ip);
-
-      clib_memcpy (b0->data + offset, &proto, sizeof (proto));
-      offset += sizeof (proto);
-
-      clib_memcpy (b0->data + offset, &src_port, sizeof (src_port));
-      offset += sizeof (src_port);
-
-      clib_memcpy (b0->data + offset, &nat_src_port, sizeof (nat_src_port));
-      offset += sizeof (nat_src_port);
-
-      clib_memcpy (b0->data + offset, &vrf_id, sizeof(vrf_id));
-      offset += sizeof (vrf_id);
-
-      b0->current_length += NAT44_SESSION_CREATE_LEN;
-    }
-
-  if (PREDICT_FALSE (do_flush || (offset + NAT44_SESSION_CREATE_LEN) > frm->path_mtu))
-    {
-      snat_ipfix_send (frm, f, b0, silm->nat44_session_template_id);
-      silm->nat44_session_frame = 0;
-      silm->nat44_session_buffer = 0;
-      offset = 0;
-    }
-  silm->nat44_session_next_record_offset = offset;
- }
-
-static void
-snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush)
-{
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-  flow_report_main_t *frm = &flow_report_main;
-  vlib_frame_t *f;
-  vlib_buffer_t *b0 = 0;
-  u32 bi0 = ~0;
-  u32 offset;
-  vlib_main_t * vm = frm->vlib_main;
-  u64 now;
-  vlib_buffer_free_list_t *fl;
-  u8 nat_event = NAT_ADDRESSES_EXHAUTED;
-
-  if (!silm->enabled)
-    return;
-
-  now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
-  now += silm->milisecond_time_0;
-
-  b0 = silm->addr_exhausted_buffer;
-
-  if (PREDICT_FALSE (b0 == 0))
-    {
-      if (do_flush)
-        return;
-
-      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
-        {
-          clib_warning ("can't allocate buffer for NAT IPFIX event");
-          return;
-        }
-
-      b0 = silm->addr_exhausted_buffer =
-        vlib_get_buffer (vm, bi0);
-      fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-      vlib_buffer_init_for_free_list (b0, fl);
-      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-      offset = 0;
-    }
-  else
-    {
-      bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->addr_exhausted_next_record_offset;
-    }
-
-  f = silm->addr_exhausted_frame;
-  if (PREDICT_FALSE (f == 0))
-    {
-      u32 * to_next;
-      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->addr_exhausted_frame = f;
-      to_next = vlib_frame_vector_args (f);
-      to_next[0] = bi0;
-      f->n_vectors = 1;
-    }
-
-  if (PREDICT_FALSE (offset == 0))
-    snat_ipfix_header_create (frm, b0, &offset);
-
-  if (PREDICT_TRUE (do_flush == 0))
-    {
-      u64 time_stamp = clib_host_to_net_u64 (now);
-      clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
-      offset += sizeof (time_stamp);
-
-      clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event));
-      offset += sizeof (nat_event);
-
-      clib_memcpy (b0->data + offset, &pool_id, sizeof(pool_id));
-      offset += sizeof (pool_id);
-
-      b0->current_length += NAT_ADDRESSES_EXHAUTED_LEN;
-    }
-
-  if (PREDICT_FALSE (do_flush || (offset + NAT_ADDRESSES_EXHAUTED_LEN) > frm->path_mtu))
-    {
-      snat_ipfix_send (frm, f, b0, silm->addr_exhausted_template_id);
-      silm->addr_exhausted_frame = 0;
-      silm->addr_exhausted_buffer = 0;
-      offset = 0;
-    }
-  silm->addr_exhausted_next_record_offset = offset;
-}
-
-static void
-snat_ipfix_logging_max_entries_per_usr (u32 src_ip, int do_flush)
-{
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-  flow_report_main_t *frm = &flow_report_main;
-  vlib_frame_t *f;
-  vlib_buffer_t *b0 = 0;
-  u32 bi0 = ~0;
-  u32 offset;
-  vlib_main_t * vm = frm->vlib_main;
-  u64 now;
-  vlib_buffer_free_list_t *fl;
-  u8 nat_event = QUOTA_EXCEEDED;
-  u32 quota_event = MAX_ENTRIES_PER_USER;
-
-  if (!silm->enabled)
-    return;
-
-  now = (u64) ((vlib_time_now (vm) - silm->vlib_time_0) * 1e3);
-  now += silm->milisecond_time_0;
-
-  b0 = silm->max_entries_per_user_buffer;
-
-  if (PREDICT_FALSE (b0 == 0))
-    {
-      if (do_flush)
-        return;
-
-      if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
-        {
-          clib_warning ("can't allocate buffer for NAT IPFIX event");
-          return;
-        }
-
-      b0 = silm->max_entries_per_user_buffer =
-        vlib_get_buffer (vm, bi0);
-      fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-      vlib_buffer_init_for_free_list (b0, fl);
-      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-      offset = 0;
-    }
-  else
-    {
-      bi0 = vlib_get_buffer_index (vm, b0);
-      offset = silm->max_entries_per_user_next_record_offset;
-    }
-
-  f = silm->max_entries_per_user_frame;
-  if (PREDICT_FALSE (f == 0))
-    {
-      u32 * to_next;
-      f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
-      silm->max_entries_per_user_frame = f;
-      to_next = vlib_frame_vector_args (f);
-      to_next[0] = bi0;
-      f->n_vectors = 1;
-    }
-
-  if (PREDICT_FALSE (offset == 0))
-    snat_ipfix_header_create (frm, b0, &offset);
-
-  if (PREDICT_TRUE (do_flush == 0))
-    {
-      u64 time_stamp = clib_host_to_net_u64 (now);
-      clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp));
-      offset += sizeof (time_stamp);
-
-      clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event));
-      offset += sizeof (nat_event);
-
-      clib_memcpy (b0->data + offset, "a_event, sizeof(quota_event));
-      offset += sizeof (quota_event);
-
-      clib_memcpy (b0->data + offset, &src_ip, sizeof (src_ip));
-      offset += sizeof (src_ip);
-
-      b0->current_length += MAX_ENTRIES_PER_USER_LEN;
-    }
-
-  if (PREDICT_FALSE (do_flush || (offset + MAX_ENTRIES_PER_USER_LEN) > frm->path_mtu))
-    {
-      snat_ipfix_send (frm, f, b0, silm->max_entries_per_user_template_id);
-      silm->max_entries_per_user_frame = 0;
-      silm->max_entries_per_user_buffer = 0;
-      offset = 0;
-    }
-  silm->max_entries_per_user_next_record_offset = offset;
-}
-
-static void
-snat_ipfix_logging_nat44_ses_rpc_cb (snat_ipfix_logging_nat44_ses_args_t *a)
-{
-  snat_ipfix_logging_nat44_ses(a->nat_event, a->src_ip, a->nat_src_ip,
-                               a->snat_proto, a->src_port, a->nat_src_port,
-                               a->vrf_id, 0);
-}
-
-/**
- * @brief Generate NAT44 session create event
- *
- * @param src_ip       source IPv4 address
- * @param nat_src_ip   transaltes source IPv4 address
- * @param snat_proto   SNAT transport protocol
- * @param src_port     source port
- * @param nat_src_port translated source port
- * @param vrf_id       VRF ID
- */
-void
-snat_ipfix_logging_nat44_ses_create (u32 src_ip,
-                                     u32 nat_src_ip,
-                                     snat_protocol_t snat_proto,
-                                     u16 src_port,
-                                     u16 nat_src_port,
-                                     u32 vrf_id)
-{
-  snat_ipfix_logging_nat44_ses_args_t a;
-
-  a.nat_event = NAT44_SESSION_CREATE;
-  a.src_ip = src_ip;
-  a.nat_src_ip = nat_src_ip;
-  a.snat_proto = snat_proto;
-  a.src_port = src_port;
-  a.nat_src_port = nat_src_port;
-  a.vrf_id = vrf_id;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_nat44_ses_rpc_cb, (u8 *) &a,
-                               sizeof (a));
-}
-
-/**
- * @brief Generate NAT44 session delete event
- *
- * @param src_ip       source IPv4 address
- * @param nat_src_ip   transaltes source IPv4 address
- * @param snat_proto   SNAT transport protocol
- * @param src_port     source port
- * @param nat_src_port translated source port
- * @param vrf_id       VRF ID
- */
-void
-snat_ipfix_logging_nat44_ses_delete (u32 src_ip,
-                                     u32 nat_src_ip,
-                                     snat_protocol_t snat_proto,
-                                     u16 src_port,
-                                     u16 nat_src_port,
-                                     u32 vrf_id)
-{
-  snat_ipfix_logging_nat44_ses_args_t a;
-
-  a.nat_event = NAT44_SESSION_DELETE;
-  a.src_ip = src_ip;
-  a.nat_src_ip = nat_src_ip;
-  a.snat_proto = snat_proto;
-  a.src_port = src_port;
-  a.nat_src_port = nat_src_port;
-  a.vrf_id = vrf_id;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_nat44_ses_rpc_cb, (u8 *) &a,
-                               sizeof (a));
-}
-
-vlib_frame_t *
-snat_data_callback_nat44_session (flow_report_main_t * frm,
-                                  flow_report_t * fr,
-                                  vlib_frame_t * f,
-                                  u32 * to_next,
-                                  u32 node_index)
-{
-  snat_ipfix_logging_nat44_ses(0, 0, 0, 0, 0, 0, 0, 1);
-  return f;
-}
-
-static void
-snat_ipfix_logging_addr_exhausted_rpc_cb
- (snat_ipfix_logging_addr_exhausted_args_t * a)
-{
-  snat_ipfix_logging_addr_exhausted(a->pool_id, 0);
-}
-
-/**
- * @brief Generate NAT addresses exhausted event
- *
- * @param pool_id NAT pool ID
- */
-void
-snat_ipfix_logging_addresses_exhausted(u32 pool_id)
-{
-  //TODO: This event SHOULD be rate limited
-  snat_ipfix_logging_addr_exhausted_args_t a;
-
-  a.pool_id = pool_id;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_addr_exhausted_rpc_cb,
-                               (u8 *) &a, sizeof (a));
-}
-
-vlib_frame_t *
-snat_data_callback_addr_exhausted (flow_report_main_t * frm,
-                                   flow_report_t * fr,
-                                   vlib_frame_t * f,
-                                   u32 * to_next,
-                                   u32 node_index)
-{
-  snat_ipfix_logging_addr_exhausted(0, 1);
-  return f;
-}
-
-static void
-snat_ipfix_logging_max_entries_per_usr_rpc_cb
- (snat_ipfix_logging_max_entries_per_user_args_t * a)
-{
-  snat_ipfix_logging_max_entries_per_usr(a->src_ip, 0);
-}
-
-/**
- * @brief Generate maximum entries per user exceeded event
- *
- * @param src_ip source IPv4 address
- */
-void
-snat_ipfix_logging_max_entries_per_user(u32 src_ip)
-{
-  //TODO: This event SHOULD be rate limited
-  snat_ipfix_logging_max_entries_per_user_args_t a;
-
-  a.src_ip = src_ip;
-
-  vl_api_rpc_call_main_thread (snat_ipfix_logging_max_entries_per_usr_rpc_cb,
-                               (u8 *) &a, sizeof (a));
-}
-
-vlib_frame_t *
-snat_data_callback_max_entries_per_usr (flow_report_main_t * frm,
-                                        flow_report_t * fr,
-                                        vlib_frame_t * f,
-                                        u32 * to_next,
-                                        u32 node_index)
-{
-  snat_ipfix_logging_max_entries_per_usr(0, 1);
-  return f;
-}
-
-/**
- * @brief Enable/disable SNAT IPFIX logging
- *
- * @param enable    1 if enable, 0 if disable
- * @param domain_id observation domain ID
- * @param src_port  source port number
- *
- * @returns 0 if success
- */
-int
-snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port)
-{
-  snat_main_t * sm = &snat_main;
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-  flow_report_main_t *frm = &flow_report_main;
-  vnet_flow_report_add_del_args_t a;
-  int rv;
-  u8 e = enable ? 1 : 0;
-
-  if (silm->enabled == e)
-    return 0;
-
-  silm->enabled = e;
-
-  memset (&a, 0, sizeof (a));
-  a.is_add = enable;
-  a.domain_id = domain_id ? domain_id : 1;
-  a.src_port = src_port ? src_port : UDP_DST_PORT_ipfix;
-
-  if (sm->deterministic)
-    {
-      a.rewrite_callback = snat_template_rewrite_max_entries_per_usr;
-      a.flow_data_callback = snat_data_callback_max_entries_per_usr;
-
-      rv = vnet_flow_report_add_del (frm, &a, NULL);
-      if (rv)
-        {
-          clib_warning ("vnet_flow_report_add_del returned %d", rv);
-          return -1;
-        }
-    }
-  else
-    {
-      a.rewrite_callback = snat_template_rewrite_nat44_session;
-      a.flow_data_callback = snat_data_callback_nat44_session;
-
-      rv = vnet_flow_report_add_del (frm, &a, NULL);
-      if (rv)
-        {
-          clib_warning ("vnet_flow_report_add_del returned %d", rv);
-          return -1;
-        }
-
-      a.rewrite_callback = snat_template_rewrite_addr_exhausted;
-      a.flow_data_callback = snat_data_callback_addr_exhausted;
-
-      rv = vnet_flow_report_add_del (frm, &a, NULL);
-      if (rv)
-        {
-          clib_warning ("vnet_flow_report_add_del returned %d", rv);
-          return -1;
-        }
-    }
-
-  return 0;
-}
-
-/**
- * @brief Initialize SNAT IPFIX logging
- *
- * @param vm vlib main
- */
-void
-snat_ipfix_logging_init (vlib_main_t * vm)
-{
-  snat_ipfix_logging_main_t *silm = &snat_ipfix_logging_main;
-
-  silm->enabled = 0;
-
-  /* Set up time reference pair */
-  silm->vlib_time_0 = vlib_time_now (vm);
-  silm->milisecond_time_0 = unix_time_now_nsec () * 1e-6;
-}
diff --git a/src/plugins/snat/snat_ipfix_logging.h b/src/plugins/snat/snat_ipfix_logging.h
deleted file mode 100644
index 45c1a7bf..00000000
--- a/src/plugins/snat/snat_ipfix_logging.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * snat_ipfix_logging.h - NAT Events IPFIX logging
- *
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_snat_ipfix_logging_h__
-#define __included_snat_ipfix_logging_h__
-
-typedef enum {
-  NAT_ADDRESSES_EXHAUTED = 3,
-  NAT44_SESSION_CREATE = 4,
-  NAT44_SESSION_DELETE = 5,
-  NAT_PORTS_EXHAUSTED = 12,
-  QUOTA_EXCEEDED = 13,
-} nat_event_t;
-
-typedef enum {
-  MAX_ENTRIES_PER_USER = 3,
-} quota_exceed_event_t;
-
-typedef struct {
-  /** S-NAT IPFIX logging enabled */
-  u8 enabled;
-
-  /** ipfix buffers under construction */
-  vlib_buffer_t *nat44_session_buffer;
-  vlib_buffer_t *addr_exhausted_buffer;
-  vlib_buffer_t *max_entries_per_user_buffer;
-
-  /** frames containing ipfix buffers */
-  vlib_frame_t *nat44_session_frame;
-  vlib_frame_t *addr_exhausted_frame;
-  vlib_frame_t *max_entries_per_user_frame;
-
-  /** next record offset */
-  u32 nat44_session_next_record_offset;
-  u32 addr_exhausted_next_record_offset;
-  u32 max_entries_per_user_next_record_offset;
-
-  /** Time reference pair */
-  u64 milisecond_time_0;
-  f64 vlib_time_0;
-
-  /** template IDs */
-  u16 nat44_session_template_id;
-  u16 addr_exhausted_template_id;
-  u16 max_entries_per_user_template_id;
-
-  /** stream index */
-  u32 stream_index;
-} snat_ipfix_logging_main_t;
-
-extern snat_ipfix_logging_main_t snat_ipfix_logging_main;
-
-void snat_ipfix_logging_init (vlib_main_t * vm);
-int snat_ipfix_logging_enable_disable (int enable, u32 domain_id, u16 src_port);
-void snat_ipfix_logging_nat44_ses_create (u32 src_ip, u32 nat_src_ip,
-                                          snat_protocol_t snat_proto,
-                                          u16 src_port, u16 nat_src_port,
-                                          u32 vrf_id);
-void snat_ipfix_logging_nat44_ses_delete (u32 src_ip, u32 nat_src_ip,
-                                          snat_protocol_t snat_proto,
-                                          u16 src_port, u16 nat_src_port,
-                                          u32 vrf_id);
-void snat_ipfix_logging_addresses_exhausted(u32 pool_id);
-void snat_ipfix_logging_max_entries_per_user(u32 src_ip);
-
-#endif /* __included_snat_ipfix_logging_h__ */
diff --git a/src/plugins/snat/snat_msg_enum.h b/src/plugins/snat/snat_msg_enum.h
deleted file mode 100644
index 2c76fd51..00000000
--- a/src/plugins/snat/snat_msg_enum.h
+++ /dev/null
@@ -1,31 +0,0 @@
-
-/*
- * snat_msg_enum.h - skeleton vpp engine plug-in message enumeration
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef included_snat_msg_enum_h
-#define included_snat_msg_enum_h
-
-#include 
-
-#define vl_msg_id(n,h) n,
-typedef enum {
-#include 
-    /* We'll want to know how many messages IDs we need... */
-    VL_MSG_FIRST_AVAILABLE,
-} vl_msg_id_t;
-#undef vl_msg_id
-
-#endif /* included_snat_msg_enum_h */
diff --git a/src/plugins/snat/snat_test.c b/src/plugins/snat/snat_test.c
deleted file mode 100644
index 905b8fac..00000000
--- a/src/plugins/snat/snat_test.c
+++ /dev/null
@@ -1,1167 +0,0 @@
-
-/*
- * snat.c - skeleton vpp-api-test plug-in 
- *
- * Copyright (c)  
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define __plugin_msg_base snat_test_main.msg_id_base
-#include 
-
-uword unformat_sw_if_index (unformat_input_t * input, va_list * args);
-
-/* Declare message IDs */
-#include 
-
-/* define message structures */
-#define vl_typedefs
-#include  
-#undef vl_typedefs
-
-/* declare message handlers for each api */
-
-#define vl_endianfun             /* define message structures */
-#include  
-#undef vl_endianfun
-
-/* instantiate all the print functions we know about */
-#define vl_print(handle, ...)
-#define vl_printfun
-#include  
-#undef vl_printfun
-
-/* Get the API version number. */
-#define vl_api_version(n,v) static u32 api_version=(v);
-#include 
-#undef vl_api_version
-
-typedef struct {
-    /* API message ID base */
-    u16 msg_id_base;
-    vat_main_t *vat_main;
-} snat_test_main_t;
-
-snat_test_main_t snat_test_main;
-
-#define foreach_standard_reply_retval_handler   \
-_(snat_add_address_range_reply)                 \
-_(snat_interface_add_del_feature_reply)         \
-_(snat_add_static_mapping_reply)                \
-_(snat_set_workers_reply)                       \
-_(snat_add_del_interface_addr_reply)            \
-_(snat_ipfix_enable_disable_reply)              \
-_(snat_add_det_map_reply)                       \
-_(snat_det_set_timeouts_reply)                  \
-_(snat_det_close_session_out_reply)             \
-_(snat_det_close_session_in_reply)
-
-#define _(n)                                            \
-    static void vl_api_##n##_t_handler                  \
-    (vl_api_##n##_t * mp)                               \
-    {                                                   \
-        vat_main_t * vam = snat_test_main.vat_main;   \
-        i32 retval = ntohl(mp->retval);                 \
-        if (vam->async_mode) {                          \
-            vam->async_errors += (retval < 0);          \
-        } else {                                        \
-            vam->retval = retval;                       \
-            vam->result_ready = 1;                      \
-        }                                               \
-    }
-foreach_standard_reply_retval_handler;
-#undef _
-
-/* 
- * Table of message reply handlers, must include boilerplate handlers
- * we just generated
- */
-#define foreach_vpe_api_reply_msg                               \
-_(SNAT_ADD_ADDRESS_RANGE_REPLY, snat_add_address_range_reply)   \
-_(SNAT_INTERFACE_ADD_DEL_FEATURE_REPLY,                         \
-  snat_interface_add_del_feature_reply)                         \
-_(SNAT_ADD_STATIC_MAPPING_REPLY, snat_add_static_mapping_reply) \
-_(SNAT_CONTROL_PING_REPLY, snat_control_ping_reply)             \
-_(SNAT_STATIC_MAPPING_DETAILS, snat_static_mapping_details)     \
-_(SNAT_SHOW_CONFIG_REPLY, snat_show_config_reply)               \
-_(SNAT_ADDRESS_DETAILS, snat_address_details)                   \
-_(SNAT_INTERFACE_DETAILS, snat_interface_details)               \
-_(SNAT_SET_WORKERS_REPLY, snat_set_workers_reply)               \
-_(SNAT_WORKER_DETAILS, snat_worker_details)                     \
-_(SNAT_ADD_DEL_INTERFACE_ADDR_REPLY,                            \
-  snat_add_del_interface_addr_reply)                            \
-_(SNAT_INTERFACE_ADDR_DETAILS, snat_interface_addr_details)     \
-_(SNAT_IPFIX_ENABLE_DISABLE_REPLY,                              \
-  snat_ipfix_enable_disable_reply)                              \
-_(SNAT_USER_DETAILS, snat_user_details)                         \
-_(SNAT_USER_SESSION_DETAILS, snat_user_session_details)         \
-_(SNAT_ADD_DET_MAP_REPLY, snat_add_det_map_reply)               \
-_(SNAT_DET_FORWARD_REPLY, snat_det_forward_reply)               \
-_(SNAT_DET_REVERSE_REPLY, snat_det_reverse_reply)               \
-_(SNAT_DET_MAP_DETAILS, snat_det_map_details)                   \
-_(SNAT_DET_SET_TIMEOUTS_REPLY, snat_det_set_timeouts_reply)     \
-_(SNAT_DET_GET_TIMEOUTS_REPLY, snat_det_get_timeouts_reply)     \
-_(SNAT_DET_CLOSE_SESSION_OUT_REPLY,                             \
-  snat_det_close_session_out_reply)                             \
-_(SNAT_DET_CLOSE_SESSION_IN_REPLY,                              \
-  snat_det_close_session_in_reply)                              \
-_(SNAT_DET_SESSION_DETAILS, snat_det_session_details)
-
-static int api_snat_add_address_range (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  ip4_address_t start_addr, end_addr;
-  u32 start_host_order, end_host_order;
-  vl_api_snat_add_address_range_t * mp;
-  u8 is_add = 1;
-  int count;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "%U - %U",
-                    unformat_ip4_address, &start_addr,
-                    unformat_ip4_address, &end_addr))
-        ;
-      else if (unformat (i, "%U", unformat_ip4_address, &start_addr))
-        end_addr = start_addr;
-      else if (unformat (i, "del"))
-        is_add = 0;
-      else
-        {
-          clib_warning("unknown input '%U'", format_unformat_error, i);
-          return -99;
-        }
-    }
-
-  start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
-  end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
-  
-  if (end_host_order < start_host_order)
-    {
-      errmsg ("end address less than start address\n");
-      return -99;
-    }
-
-  count = (end_host_order - start_host_order) + 1;
-
-  if (count > 1024)
-    {
-    errmsg ("%U - %U, %d addresses...\n",
-           format_ip4_address, &start_addr,
-           format_ip4_address, &end_addr,
-           count);
-    }
-  
-  M(SNAT_ADD_ADDRESS_RANGE, mp);
-
-  memcpy (mp->first_ip_address, &start_addr, 4);
-  memcpy (mp->last_ip_address, &end_addr, 4);
-  mp->is_ip4 = 1;
-  mp->is_add = is_add;
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static int api_snat_interface_add_del_feature (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_interface_add_del_feature_t * mp;
-  u32 sw_if_index;
-  u8 sw_if_index_set = 0;
-  u8 is_inside = 1; 
-  u8 is_add = 1;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
-        sw_if_index_set = 1;
-      else if (unformat (i, "sw_if_index %d", &sw_if_index))
-        sw_if_index_set = 1;
-      else if (unformat (i, "out"))
-        is_inside = 0;
-      else if (unformat (i, "in"))
-        is_inside = 1;
-      else if (unformat (i, "del"))
-        is_add = 0;
-      else
-        {
-          clib_warning("unknown input '%U'", format_unformat_error, i);
-          return -99;
-        }
-    }
-
-  if (sw_if_index_set == 0)
-    {
-      errmsg ("interface / sw_if_index required\n");
-      return -99;
-    }
-
-  M(SNAT_INTERFACE_ADD_DEL_FEATURE, mp);
-  mp->sw_if_index = ntohl(sw_if_index);
-  mp->is_add = is_add;
-  mp->is_inside = is_inside;
-  
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static int api_snat_add_static_mapping(vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_add_static_mapping_t * mp;
-  u8 external_addr_set = 0;
-  u8 local_addr_set = 0;
-  u8 is_add = 1;
-  u8 addr_only = 1;
-  ip4_address_t local_addr, external_addr;
-  u32 local_port = 0, external_port = 0, vrf_id = ~0;
-  u32 sw_if_index = ~0;
-  u8 sw_if_index_set = 0;
-  u32 proto = ~0;
-  u8 proto_set = 0;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "local_addr %U", unformat_ip4_address, &local_addr))
-        local_addr_set = 1;
-      else if (unformat (i, "external_addr %U", unformat_ip4_address,
-                         &external_addr))
-        external_addr_set = 1;
-      else if (unformat (i, "local_port %u", &local_port))
-        addr_only = 0;
-      else if (unformat (i, "external_port %u", &external_port))
-        addr_only = 0;
-      else if (unformat (i, "external_if %U", unformat_sw_if_index, vam,
-                         &sw_if_index))
-        sw_if_index_set = 1;
-      else if (unformat (i, "external_sw_if_index %d", &sw_if_index))
-        sw_if_index_set = 1;
-      else if (unformat (i, "vrf %u", &vrf_id))
-        ;
-      else if (unformat (i, "protocol %u", &proto))
-        proto_set = 1;
-      else if (unformat (i, "del"))
-        is_add = 0;
-      else
-        {
-          clib_warning("unknown input '%U'", format_unformat_error, i);
-          return -99;
-        }
-    }
-
-  if (!addr_only && !proto_set)
-    {
-      errmsg ("protocol required\n");
-      return -99;
-    }
-
-  if (!local_addr_set)
-    {
-      errmsg ("local addr required\n");
-      return -99;
-    }
-  if (!external_addr_set && !sw_if_index_set)
-    {
-      errmsg ("external addr or interface required\n");
-      return -99;
-    }
-
-  M(SNAT_ADD_STATIC_MAPPING, mp);
-  mp->is_add = is_add;
-  mp->is_ip4 = 1;
-  mp->addr_only = addr_only;
-  mp->local_port = ntohs ((u16) local_port);
-  mp->external_port = ntohs ((u16) external_port);
-  mp->external_sw_if_index = ntohl (sw_if_index);
-  mp->vrf_id = ntohl (vrf_id);
-  mp->protocol = (u8) proto;
-  memcpy (mp->local_ip_address, &local_addr, 4);
-  memcpy (mp->external_ip_address, &external_addr, 4);
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_control_ping_reply_t_handler
-  (vl_api_snat_control_ping_reply_t * mp)
-{
-  vat_main_t *vam = &vat_main;
-  i32 retval = ntohl (mp->retval);
-  if (vam->async_mode)
-    {
-      vam->async_errors += (retval < 0);
-    }
-  else
-    {
-      vam->retval = retval;
-      vam->result_ready = 1;
-    }
-}
-
-static void vl_api_snat_static_mapping_details_t_handler
-  (vl_api_snat_static_mapping_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  if (mp->addr_only && mp->external_sw_if_index != ~0)
-      fformat (vam->ofp, "%15U%6s%15d%6s%11d%6d\n",
-               format_ip4_address, &mp->local_ip_address, "",
-               ntohl (mp->external_sw_if_index), "",
-               ntohl (mp->vrf_id),
-               mp->protocol);
-  else if (mp->addr_only && mp->external_sw_if_index == ~0)
-      fformat (vam->ofp, "%15U%6s%15U%6s%11d%6d\n",
-               format_ip4_address, &mp->local_ip_address, "",
-               format_ip4_address, &mp->external_ip_address, "",
-               ntohl (mp->vrf_id),
-               mp->protocol);
-  else if (!mp->addr_only && mp->external_sw_if_index != ~0)
-      fformat (vam->ofp, "%15U%6d%15d%6d%11d%6d\n",
-               format_ip4_address, &mp->local_ip_address,
-               ntohs (mp->local_port),
-               ntohl (mp->external_sw_if_index),
-               ntohs (mp->external_port),
-               ntohl (mp->vrf_id),
-               mp->protocol);
-  else
-      fformat (vam->ofp, "%15U%6d%15U%6d%11d%6d\n",
-               format_ip4_address, &mp->local_ip_address,
-               ntohs (mp->local_port),
-               format_ip4_address, &mp->external_ip_address,
-               ntohs (mp->external_port),
-               ntohl (mp->vrf_id),
-               mp->protocol);
-
-}
-
-static int api_snat_static_mapping_dump(vat_main_t * vam)
-{
-  vl_api_snat_static_mapping_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_static_mapping_dump");
-      return -99;
-    }
-
-  fformat (vam->ofp, "%21s%21s\n", "local", "external");
-  fformat (vam->ofp, "%15s%6s%15s%6s%11s%6s\n", "address", "port",
-           "address/if_idx", "port", "vrf", "proto");
-
-  M(SNAT_STATIC_MAPPING_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_show_config_reply_t_handler
-  (vl_api_snat_show_config_reply_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-  i32 retval = ntohl (mp->retval);
-
-  if (retval >= 0)
-    {
-      fformat (vam->ofp, "translation hash buckets %d\n",
-               ntohl (mp->translation_buckets));
-      fformat (vam->ofp, "translation hash memory %d\n",
-               ntohl (mp->translation_memory_size));
-      fformat (vam->ofp, "user hash buckets %d\n", ntohl (mp->user_buckets));
-      fformat (vam->ofp, "user hash memory %d\n", ntohl (mp->user_memory_size));
-      fformat (vam->ofp, "max translations per user %d\n",
-               ntohl (mp->max_translations_per_user));
-      fformat (vam->ofp, "outside VRF id %d\n", ntohl (mp->outside_vrf_id));
-      fformat (vam->ofp, "inside VRF id %d\n", ntohl (mp->inside_vrf_id));
-      if (mp->static_mapping_only)
-        {
-          fformat (vam->ofp, "static mapping only");
-          if (mp->static_mapping_connection_tracking)
-            fformat (vam->ofp, " connection tracking");
-          fformat (vam->ofp, "\n");
-        }
-    }
-  vam->retval = retval;
-  vam->result_ready = 1;
-}
-
-static int api_snat_show_config(vat_main_t * vam)
-{
-  vl_api_snat_show_config_t * mp;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_show_config");
-      return -99;
-    }
-
-  M(SNAT_SHOW_CONFIG, mp);
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_address_details_t_handler
-  (vl_api_snat_address_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat (vam->ofp, "%U\n", format_ip4_address, &mp->ip_address);
-}
-
-static int api_snat_address_dump(vat_main_t * vam)
-{
-  vl_api_snat_address_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_address_dump");
-      return -99;
-    }
-
-  M(SNAT_ADDRESS_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_interface_details_t_handler
-  (vl_api_snat_interface_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat (vam->ofp, "sw_if_index %d %s\n", ntohl (mp->sw_if_index),
-           mp->is_inside ? "in" : "out");
-}
-
-static int api_snat_interface_dump(vat_main_t * vam)
-{
-  vl_api_snat_interface_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_address_dump");
-      return -99;
-    }
-
-  M(SNAT_INTERFACE_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static int api_snat_set_workers (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_set_workers_t * mp;
-  uword *bitmap;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "%U", unformat_bitmap_list, &bitmap))
-        ;
-      else
-        {
-          clib_warning("unknown input '%U'", format_unformat_error, i);
-          return -99;
-        }
-    }
-
-  M(SNAT_SET_WORKERS, mp);
-  mp->worker_mask = clib_host_to_net_u64 (bitmap[0]);
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_worker_details_t_handler
-  (vl_api_snat_worker_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat (vam->ofp, "worker_index %d (%s at lcore %u)\n",
-           ntohl (mp->worker_index), mp->name, ntohl (mp->lcore_id));
-}
-
-static int api_snat_worker_dump(vat_main_t * vam)
-{
-  vl_api_snat_worker_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_address_dump");
-      return -99;
-    }
-
-  M(SNAT_WORKER_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static int api_snat_add_del_interface_addr (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_add_del_interface_addr_t * mp;
-  u32 sw_if_index;
-  u8 sw_if_index_set = 0;
-  u8 is_add = 1;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
-        sw_if_index_set = 1;
-      else if (unformat (i, "sw_if_index %d", &sw_if_index))
-        sw_if_index_set = 1;
-      else if (unformat (i, "del"))
-        is_add = 0;
-      else
-        {
-          clib_warning("unknown input '%U'", format_unformat_error, i);
-          return -99;
-        }
-    }
-
-  if (sw_if_index_set == 0)
-    {
-      errmsg ("interface / sw_if_index required\n");
-      return -99;
-    }
-
-  M(SNAT_ADD_DEL_INTERFACE_ADDR, mp);
-  mp->sw_if_index = ntohl(sw_if_index);
-  mp->is_add = is_add;
-  
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_interface_addr_details_t_handler
-  (vl_api_snat_interface_addr_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat (vam->ofp, "sw_if_index %d\n", ntohl (mp->sw_if_index));
-}
-
-static int api_snat_interface_addr_dump(vat_main_t * vam)
-{
-  vl_api_snat_interface_addr_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_address_dump");
-      return -99;
-    }
-
-  M(SNAT_INTERFACE_ADDR_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static int api_snat_ipfix_enable_disable (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_ipfix_enable_disable_t * mp;
-  u32 domain_id = 0;
-  u32 src_port = 0;
-  u8 enable = 1;
-  int ret;
-
-  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
-    {
-      if (unformat (i, "domain %d", &domain_id))
-        ;
-      else if (unformat (i, "src_port %d", &src_port))
-        ;
-      else if (unformat (i, "disable"))
-        enable = 0;
-      else
-        {
-          clib_warning("unknown input '%U'", format_unformat_error, i);
-          return -99;
-        }
-    }
-
-  M(SNAT_IPFIX_ENABLE_DISABLE, mp);
-  mp->domain_id = htonl(domain_id);
-  mp->src_port = htons((u16) src_port);
-  mp->enable = enable;
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_user_session_details_t_handler
-  (vl_api_snat_user_session_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat(vam->ofp, "%s session %U:%d to %U:%d protocol id %d "
-                    "total packets %d total bytes %d\n",
-          mp->is_static ? "static" : "dynamic",
-          format_ip4_address, mp->inside_ip_address, ntohl(mp->inside_port),
-          format_ip4_address, mp->outside_ip_address, ntohl(mp->outside_port),
-          ntohl(mp->protocol), ntohl(mp->total_pkts), ntohl(mp->total_bytes));
-}
-
-static int api_snat_user_session_dump(vat_main_t * vam)
-{
-  unformat_input_t* i = vam->input;
-  vl_api_snat_user_session_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  ip4_address_t addr;
-  u32 vrf_id = ~0;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_address_dump");
-      return -99;
-    }
-
-  if (unformat (i, "ip_address %U vrf_id %d",
-                unformat_ip4_address, &addr, &vrf_id))
-    ;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_USER_SESSION_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  memset(mp->ip_address, 0, 16);
-  clib_memcpy(mp->ip_address, &addr, 4);
-  mp->vrf_id = htonl(vrf_id);
-  mp->is_ip4 = 1;
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_user_details_t_handler
-  (vl_api_snat_user_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat(vam->ofp, "user with ip %U with vrf_id %d "
-                    "with %d sessions and %d static sessions\n",
-          format_ip4_address, mp->ip_address, ntohl(mp->vrf_id),
-          ntohl(mp->nsessions), ntohl(mp->nstaticsessions));
-}
-
-static int api_snat_user_dump(vat_main_t * vam)
-{
-  vl_api_snat_user_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_address_dump");
-      return -99;
-    }
-
-  M(SNAT_USER_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static int api_snat_add_det_map (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_add_det_map_t * mp;
-  ip4_address_t in_addr, out_addr;
-  u32 in_plen, out_plen;
-  u8 is_add = 1;
-  int ret;
-
-  if (unformat (i, "in %U/%d out %U/%d",
-                unformat_ip4_address, &in_addr, &in_plen,
-                unformat_ip4_address, &out_addr, &out_plen))
-    ;
-  else if (unformat (i, "del"))
-    is_add = 0;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_ADD_DET_MAP, mp);
-  clib_memcpy(mp->in_addr, &in_addr, 4);
-  mp->in_plen = in_plen;
-  clib_memcpy(mp->out_addr, &out_addr, 4);
-  mp->out_plen = out_plen;
-  mp->is_add = is_add;
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_det_forward_reply_t_handler
-  (vl_api_snat_det_forward_reply_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-  i32 retval = ntohl(mp->retval);
-
-  if (retval >= 0)
-  {
-    fformat (vam->ofp, "outside address %U", format_ip4_address, &mp->out_addr);
-    fformat (vam->ofp, " outside port range start %d", ntohs(mp->out_port_lo));
-    fformat (vam->ofp, " outside port range end %d\n", ntohs(mp->out_port_hi));
-  }
-
-  vam->retval = retval;
-  vam->result_ready = 1;
-}
-
-static int api_snat_det_forward (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_det_forward_t * mp;
-  ip4_address_t in_addr;
-  int ret;
-
-  if (unformat (i, "%U", unformat_ip4_address, &in_addr))
-    ;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_DET_FORWARD, mp);
-  clib_memcpy(mp->in_addr, &in_addr, 4);
-
-  S(mp);
-  W(ret);
-  return ret;
-}
-
-static void vl_api_snat_det_reverse_reply_t_handler
-  (vl_api_snat_det_reverse_reply_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-  i32 retval = ntohl(mp->retval);
-
-  if (retval >= 0)
-  {
-    fformat (vam->ofp, "inside address %U\n", format_ip4_address, &mp->in_addr);
-  }
-
-  vam->retval = retval;
-  vam->result_ready = 1;
-}
-
-static int api_snat_det_reverse (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_det_reverse_t * mp;
-  ip4_address_t out_addr;
-  u16 out_port;
-  int ret;
-
-  if (unformat (i, "%U %d", unformat_ip4_address, &out_addr, &out_port))
-    ;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_DET_REVERSE, mp);
-  clib_memcpy(mp->out_addr, &out_addr, 4);
-  mp->out_port = htons(out_port);
-
-  S(mp);
-  W(ret);
-  return ret;
-}
-
-static void vl_api_snat_det_map_details_t_handler
-  (vl_api_snat_det_map_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat (vam->ofp, "Deterministic S-NAT mapping in %U/%d out %U/%d "
-                     "ports per host %d sharing ratio %d "
-                     "number of sessions %d",
-           format_ip4_address, mp->in_addr, mp->in_plen,
-           format_ip4_address, mp->out_addr, mp->out_plen,
-           ntohs(mp->ports_per_host), ntohl(mp->sharing_ratio),
-           ntohl(mp->ses_num));
-}
-
-static int api_snat_det_map_dump(vat_main_t * vam)
-{
-  vl_api_snat_det_map_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_det_map_dump");
-      return -99;
-    }
-
-  M(SNAT_DET_MAP_DUMP, mp);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-static int api_snat_det_set_timeouts (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_det_set_timeouts_t * mp;
-  u32 udp = SNAT_UDP_TIMEOUT;
-  u32 tcp_established = SNAT_TCP_ESTABLISHED_TIMEOUT;
-  u32 tcp_transitory = SNAT_TCP_TRANSITORY_TIMEOUT;
-  u32 icmp = SNAT_ICMP_TIMEOUT;
-  int ret;
-
-  if (unformat (i, "udp %d", &udp))
-    ;
-  else if (unformat (i, "tcp_established %d", &tcp_established))
-    ;
-  else if (unformat (i, "tcp_transitory %d", &tcp_transitory))
-    ;
-  else if (unformat (i, "icmp %d", &icmp))
-    ;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_DET_SET_TIMEOUTS, mp);
-  mp->udp = htonl(udp);
-  mp->tcp_established = htonl(tcp_established);
-  mp->tcp_transitory = htonl(tcp_transitory);
-  mp->icmp = htonl(icmp);
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_det_get_timeouts_reply_t_handler
-  (vl_api_snat_det_get_timeouts_reply_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-  i32 retval = ntohl (mp->retval);
-
-  if (retval >= 0)
-    {
-      fformat (vam->ofp, "udp timeout: %dsec\n", ntohl (mp->udp));
-      fformat (vam->ofp, "tcp-established timeout: %dsec",
-               ntohl (mp->tcp_established));
-      fformat (vam->ofp, "tcp-transitory timeout: %dsec",
-               ntohl (mp->tcp_transitory));
-      fformat (vam->ofp, "icmp timeout: %dsec", ntohl (mp->icmp));
-    }
-  vam->retval = retval;
-  vam->result_ready = 1;
-}
-
-static int api_snat_det_get_timeouts(vat_main_t * vam)
-{
-  vl_api_snat_det_get_timeouts_t * mp;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_show_config");
-      return -99;
-    }
-
-  M(SNAT_DET_GET_TIMEOUTS, mp);
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static int api_snat_det_close_session_out (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_det_close_session_out_t * mp;
-  ip4_address_t out_addr, ext_addr;
-  u16 out_port, ext_port;
-  int ret;
-
-  if (unformat (i, "%U:%d %U:%d",
-                unformat_ip4_address, &out_addr, &out_port,
-                unformat_ip4_address, &ext_addr, &ext_port))
-    ;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_DET_CLOSE_SESSION_OUT, mp);
-  clib_memcpy(mp->out_addr, &out_addr, 4);
-  mp->out_port = ntohs(out_port);
-  clib_memcpy(mp->ext_addr, &ext_addr, 4);
-  mp->ext_port = ntohs(ext_port);
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static int api_snat_det_close_session_in (vat_main_t * vam)
-{
-  unformat_input_t * i = vam->input;
-  vl_api_snat_det_close_session_in_t * mp;
-  ip4_address_t in_addr, ext_addr;
-  u16 in_port, ext_port;
-  int ret;
-
-  if (unformat (i, "%U:%d %U:%d",
-                unformat_ip4_address, &in_addr, &in_port,
-                unformat_ip4_address, &ext_addr, &ext_port))
-    ;
-  else
-    {
-      clib_warning("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_DET_CLOSE_SESSION_IN, mp);
-  clib_memcpy(mp->in_addr, &in_addr, 4);
-  mp->in_port = ntohs(in_port);
-  clib_memcpy(mp->ext_addr, &ext_addr, 4);
-  mp->ext_port = ntohs(ext_port);
-
-  S(mp);
-  W (ret);
-  return ret;
-}
-
-static void vl_api_snat_det_session_details_t_handler
-  (vl_api_snat_det_session_details_t *mp)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  vat_main_t *vam = sm->vat_main;
-
-  fformat(vam->ofp, "deterministic session, external host address %U, "
-                    "external host port %d, outer port %d, inside port %d",
-          format_ip4_address, mp->ext_addr, mp->ext_port,
-          mp->out_port, mp->in_port);
-}
-
-static int api_snat_det_session_dump(vat_main_t * vam)
-{
-  unformat_input_t* i = vam->input;
-  vl_api_snat_det_session_dump_t * mp;
-  vl_api_snat_control_ping_t *mp_ping;
-  ip4_address_t user_addr;
-  int ret;
-
-  if (vam->json_output)
-    {
-      clib_warning ("JSON output not supported for snat_det_session_dump");
-      return -99;
-    }
-
-  if (unformat (i, "user_addr %U", unformat_ip4_address, &user_addr))
-    ;
-  else
-    {
-      clib_warning ("unknown input '%U'", format_unformat_error, i);
-      return -99;
-    }
-
-  M(SNAT_DET_SESSION_DUMP, mp);
-  clib_memcpy (&mp->user_addr, &user_addr, 4);
-  S(mp);
-
-  /* Use a control ping for synchronization */
-  M(SNAT_CONTROL_PING, mp_ping);
-  S(mp_ping);
-
-  W (ret);
-  return ret;
-}
-
-/* 
- * List of messages that the api test plugin sends,
- * and that the data plane plugin processes
- */
-#define foreach_vpe_api_msg                                      \
-_(snat_add_address_range, " [-  | sw_if_index  [in] [out] [del]")                 \
-_(snat_add_static_mapping, "local_addr  (external_addr " \
-  " | external_if  | external_sw_if_ndex ) "          \
-  "[local_port ] [external_port ] [vrf ] [del] " \
-  "protocol ")                                                \
-_(snat_set_workers, "")                          \
-_(snat_static_mapping_dump, "")                                  \
-_(snat_show_config, "")                                          \
-_(snat_address_dump, "")                                         \
-_(snat_interface_dump, "")                                       \
-_(snat_worker_dump, "")                                          \
-_(snat_add_del_interface_addr,                                   \
-  " | sw_if_index  [del]")                            \
-_(snat_interface_addr_dump, "")                                  \
-_(snat_ipfix_enable_disable, "[domain ] [src_port ] "     \
-  "[disable]")                                                   \
-_(snat_user_dump, "")                                            \
-_(snat_user_session_dump, "ip_address  vrf_id ")   \
-_(snat_add_det_map, "in / out "                \
-  "/ [del]")                                 \
-_(snat_det_forward, "")                                 \
-_(snat_det_reverse, " ")                     \
-_(snat_det_map_dump, "")                                         \
-_(snat_det_set_timeouts, "[udp  | tcp_established  | " \
-  "tcp_transitory  | icmp ]")                          \
-_(snat_det_get_timeouts, "")                                     \
-_(snat_det_close_session_out, ": "           \
-  ":")                                       \
-_(snat_det_close_session_in, ": "              \
-  ":")                                       \
-_(snat_det_session_dump, "ip_address ")
-
-static void 
-snat_vat_api_hookup (vat_main_t *vam)
-{
-  snat_test_main_t * sm __attribute__((unused)) = &snat_test_main;
-  /* Hook up handlers for replies from the data plane plug-in */
-#define _(N,n)                                                  \
-  vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base),       \
-                          #n,                                   \
-                          vl_api_##n##_t_handler,               \
-                          vl_noop_handler,                      \
-                          vl_api_##n##_t_endian,                \
-                          vl_api_##n##_t_print,                 \
-                          sizeof(vl_api_##n##_t), 1); 
-  foreach_vpe_api_reply_msg;
-#undef _
-
-  /* API messages we can send */
-#define _(n,h)                                          \
-  hash_set_mem (vam->function_by_name, #n, api_##n);
-  foreach_vpe_api_msg;
-#undef _    
-    
-  /* Help strings */
-#define _(n,h) hash_set_mem (vam->help_by_name, #n, h);
-  foreach_vpe_api_msg;
-#undef _
-}
-
-clib_error_t * vat_plugin_register (vat_main_t *vam)
-{
-  snat_test_main_t * sm = &snat_test_main;
-  u8 * name;
-
-  sm->vat_main = vam;
-
-  /* Ask the vpp engine for the first assigned message-id */
-  name = format (0, "snat_%08x%c", api_version, 0);
-  sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
-
-  if (sm->msg_id_base != (u16) ~0)
-    snat_vat_api_hookup (vam);
-  
-  vec_free(name);
-  
-  return 0;
-}
diff --git a/src/scripts/vnet/nat44 b/src/scripts/vnet/nat44
new file mode 100644
index 00000000..3292b565
--- /dev/null
+++ b/src/scripts/vnet/nat44
@@ -0,0 +1,41 @@
+create packet-generator interface pg0
+create packet-generator interface pg1
+
+packet-generator new {
+  name f1
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  interface pg0
+  data {
+    UDP: 10.0.0.3 -> 172.16.1.2
+    UDP: 3000 -> 3001
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f2
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  interface pg0
+  data {
+    UDP: 10.0.0.3 -> 172.16.1.2
+    UDP: 3005 -> 3006
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+nat44 add address 172.16.1.3
+set int ip address pg0 10.0.0.1/24
+set int ip address pg1 172.16.1.1/24
+set int state pg0 up
+set int state pg1 up
+set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
+set ip arp static pg0 10.0.0.4 abcd.abcd.abcd
+set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
+set int nat44 in pg0 out pg1
diff --git a/src/scripts/vnet/nat44_det b/src/scripts/vnet/nat44_det
new file mode 100644
index 00000000..629772e6
--- /dev/null
+++ b/src/scripts/vnet/nat44_det
@@ -0,0 +1,108 @@
+create packet-generator interface pg0
+create packet-generator interface pg1
+
+packet-generator new {
+  name f1
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 0
+  interface pg0
+  data {
+    UDP: 10.0.0.3 -> 172.16.1.2
+    UDP: 3000 -> 3001
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f2
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 1
+  interface pg0
+  data {
+    UDP: 10.0.0.3 -> 172.16.1.2
+    UDP: 3005 -> 3006
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f3
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 0
+  interface pg1
+  data {
+    UDP: 172.16.1.2 -> 1.1.1.2
+    UDP: 3001 -> 1141
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f4
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 1
+  interface pg1
+  data {
+    UDP: 172.16.1.2 -> 1.1.1.2
+    UDP: 3006 -> 1146
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f5
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 2
+  interface pg0
+  data {
+    UDP: 10.0.0.4 -> 172.16.1.2
+    UDP: 3005 -> 3006
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f6
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 1
+  interface pg1
+  data {
+    UDP: 172.16.1.2 -> 1.1.1.2
+    UDP: 3006 -> 1177
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+nat44 deterministic add in 10.0.0.0/21 out 1.1.1.2/32
+set int ip address pg0 10.0.0.1/24
+set int ip address pg1 172.16.1.1/24
+set int state pg0 up
+set int state pg1 up
+set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
+set ip arp static pg0 10.0.0.4 abcd.abcd.abcd
+set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
+set int nat44 in pg0 out pg1
+trace add pg-input 10
diff --git a/src/scripts/vnet/nat44_static b/src/scripts/vnet/nat44_static
new file mode 100644
index 00000000..2b8f25ec
--- /dev/null
+++ b/src/scripts/vnet/nat44_static
@@ -0,0 +1,44 @@
+create packet-generator interface pg0
+create packet-generator interface pg1
+
+packet-generator new {
+  name f1
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 0
+  interface pg0
+  data {
+    UDP: 10.0.0.3 -> 172.16.1.2
+    UDP: 3000 -> 3001
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f2
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 1
+  interface pg1
+  data {
+    UDP: 172.16.1.2 -> 172.16.1.3
+    UDP: 3001 -> 3000
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+nat44 add address 172.16.1.3
+nat44 add static mapping local 10.0.0.3 external 172.16.1.3
+set int ip address pg0 10.0.0.1/24
+set int ip address pg1 172.16.1.1/24
+set int state pg0 up
+set int state pg1 up
+set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
+set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
+set int nat44 in pg0 out pg1
+trace add pg-input 10
diff --git a/src/scripts/vnet/nat44_static_with_port b/src/scripts/vnet/nat44_static_with_port
new file mode 100644
index 00000000..15bef1be
--- /dev/null
+++ b/src/scripts/vnet/nat44_static_with_port
@@ -0,0 +1,44 @@
+create packet-generator interface pg0
+create packet-generator interface pg1
+
+packet-generator new {
+  name f1
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 0
+  interface pg0
+  data {
+    UDP: 10.0.0.3 -> 172.16.1.2
+    UDP: 3000 -> 3001
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+
+packet-generator new {
+  name f2
+  limit 1000000
+  node ip4-input
+  size 64-64
+  no-recycle
+  worker 1
+  interface pg1
+  data {
+    UDP: 172.16.1.2 -> 172.16.1.3
+    UDP: 3001 -> 3000
+    length 128 checksum 0 incrementing 1
+  }
+}
+
+nat44 add address 172.16.1.3
+nat44 add static mapping local 10.0.0.3 3000 external 172.16.1.3 3000
+set int ip address pg0 10.0.0.1/24
+set int ip address pg1 172.16.1.1/24
+set int state pg0 up
+set int state pg1 up
+set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
+set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
+set int nat44 in pg0 out pg1
+trace add pg-input 10
diff --git a/src/scripts/vnet/snat b/src/scripts/vnet/snat
deleted file mode 100644
index a711519e..00000000
--- a/src/scripts/vnet/snat
+++ /dev/null
@@ -1,41 +0,0 @@
-create packet-generator interface pg0
-create packet-generator interface pg1
-
-packet-generator new {
-  name f1
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  interface pg0
-  data {
-    UDP: 10.0.0.3 -> 172.16.1.2
-    UDP: 3000 -> 3001
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f2
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  interface pg0
-  data {
-    UDP: 10.0.0.3 -> 172.16.1.2
-    UDP: 3005 -> 3006
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-snat add address 172.16.1.3
-set int ip address pg0 10.0.0.1/24
-set int ip address pg1 172.16.1.1/24
-set int state pg0 up
-set int state pg1 up
-set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
-set ip arp static pg0 10.0.0.4 abcd.abcd.abcd
-set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
-set int snat in pg0 out pg1
diff --git a/src/scripts/vnet/snat_det b/src/scripts/vnet/snat_det
deleted file mode 100644
index d1361bb1..00000000
--- a/src/scripts/vnet/snat_det
+++ /dev/null
@@ -1,108 +0,0 @@
-create packet-generator interface pg0
-create packet-generator interface pg1
-
-packet-generator new {
-  name f1
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 0
-  interface pg0
-  data {
-    UDP: 10.0.0.3 -> 172.16.1.2
-    UDP: 3000 -> 3001
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f2
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 1
-  interface pg0
-  data {
-    UDP: 10.0.0.3 -> 172.16.1.2
-    UDP: 3005 -> 3006
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f3
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 0
-  interface pg1
-  data {
-    UDP: 172.16.1.2 -> 1.1.1.2
-    UDP: 3001 -> 1141
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f4
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 1
-  interface pg1
-  data {
-    UDP: 172.16.1.2 -> 1.1.1.2
-    UDP: 3006 -> 1146
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f5
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 2
-  interface pg0
-  data {
-    UDP: 10.0.0.4 -> 172.16.1.2
-    UDP: 3005 -> 3006
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f6
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 1
-  interface pg1
-  data {
-    UDP: 172.16.1.2 -> 1.1.1.2
-    UDP: 3006 -> 1177
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-snat deterministic add in 10.0.0.0/21 out 1.1.1.2/32
-set int ip address pg0 10.0.0.1/24
-set int ip address pg1 172.16.1.1/24
-set int state pg0 up
-set int state pg1 up
-set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
-set ip arp static pg0 10.0.0.4 abcd.abcd.abcd
-set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
-set int snat in pg0 out pg1
-trace add pg-input 10
diff --git a/src/scripts/vnet/snat_static b/src/scripts/vnet/snat_static
deleted file mode 100644
index 8fe48bff..00000000
--- a/src/scripts/vnet/snat_static
+++ /dev/null
@@ -1,44 +0,0 @@
-create packet-generator interface pg0
-create packet-generator interface pg1
-
-packet-generator new {
-  name f1
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 0
-  interface pg0
-  data {
-    UDP: 10.0.0.3 -> 172.16.1.2
-    UDP: 3000 -> 3001
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f2
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 1
-  interface pg1
-  data {
-    UDP: 172.16.1.2 -> 172.16.1.3
-    UDP: 3001 -> 3000
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-snat add address 172.16.1.3
-snat add static mapping local 10.0.0.3 external 172.16.1.3
-set int ip address pg0 10.0.0.1/24
-set int ip address pg1 172.16.1.1/24
-set int state pg0 up
-set int state pg1 up
-set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
-set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
-set int snat in pg0 out pg1
-trace add pg-input 10
diff --git a/src/scripts/vnet/snat_static_with_port b/src/scripts/vnet/snat_static_with_port
deleted file mode 100644
index f646145a..00000000
--- a/src/scripts/vnet/snat_static_with_port
+++ /dev/null
@@ -1,44 +0,0 @@
-create packet-generator interface pg0
-create packet-generator interface pg1
-
-packet-generator new {
-  name f1
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 0
-  interface pg0
-  data {
-    UDP: 10.0.0.3 -> 172.16.1.2
-    UDP: 3000 -> 3001
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-
-packet-generator new {
-  name f2
-  limit 1000000
-  node ip4-input
-  size 64-64
-  no-recycle
-  worker 1
-  interface pg1
-  data {
-    UDP: 172.16.1.2 -> 172.16.1.3
-    UDP: 3001 -> 3000
-    length 128 checksum 0 incrementing 1
-  }
-}
-
-snat add address 172.16.1.3
-snat add static mapping local 10.0.0.3 3000 external 172.16.1.3 3000
-set int ip address pg0 10.0.0.1/24
-set int ip address pg1 172.16.1.1/24
-set int state pg0 up
-set int state pg1 up
-set ip arp static pg0 10.0.0.3 abcd.abcd.abcd
-set ip arp static pg1 172.16.1.2 cdef.abcd.abcd
-set int snat in pg0 out pg1
-trace add pg-input 10
diff --git a/src/vpp-api/java/Makefile.am b/src/vpp-api/java/Makefile.am
index e0f5203d..637bb774 100644
--- a/src/vpp-api/java/Makefile.am
+++ b/src/vpp-api/java/Makefile.am
@@ -169,23 +169,23 @@ jvpp-pppoe/io_fd_vpp_jvpp_pppoe_JVppPppoeImpl.h: $(jvpp_registry_ok) $(jvpp_pppo
 endif
 
 #
-# SNAT Plugin
+# NAT Plugin
 #
-if ENABLE_SNAT_PLUGIN
-noinst_LTLIBRARIES += libjvpp_snat.la
-libjvpp_snat_la_SOURCES = jvpp-snat/jvpp_snat.c
-libjvpp_snat_la_CPPFLAGS = -Ijvpp-snat
-libjvpp_snat_la_LIBADD = $(JVPP_LIBS)
-libjvpp_snat_la_DEPENDENCIES = libjvpp_common.la
+if ENABLE_NAT_PLUGIN
+noinst_LTLIBRARIES += libjvpp_nat.la
+libjvpp_nat_la_SOURCES = jvpp-nat/jvpp_nat.c
+libjvpp_nat_la_CPPFLAGS = -Ijvpp-nat
+libjvpp_nat_la_LIBADD = $(JVPP_LIBS)
+libjvpp_nat_la_DEPENDENCIES = libjvpp_common.la
 
-BUILT_SOURCES += jvpp-snat/io_fd_vpp_jvpp_snat_JVppSnatImpl.h
-JAR_FILES += jvpp-snat-$(PACKAGE_VERSION).jar
-CLEANDIRS += jvpp-snat/target
+BUILT_SOURCES += jvpp-nat/io_fd_vpp_jvpp_nat_JVppNatImpl.h
+JAR_FILES += jvpp-nat-$(PACKAGE_VERSION).jar
+CLEANDIRS += jvpp-nat/target
 
-jvpp_snat_json_files = @top_builddir@/plugins/snat/snat.api.json
+jvpp_nat_json_files = @top_builddir@/plugins/nat/nat.api.json
 
-jvpp-snat/io_fd_vpp_jvpp_snat_JVppSnatImpl.h: $(jvpp_registry_ok) $(jvpp_snat_json_files)
-	$(call japigen,snat,JVppSnatImpl)
+jvpp-nat/io_fd_vpp_jvpp_nat_JVppNatImpl.h: $(jvpp_registry_ok) $(jvpp_nat_json_files)
+	$(call japigen,nat,JVppNatImpl)
 endif
 
 #
diff --git a/src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/CallbackApiExample.java b/src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/CallbackApiExample.java
new file mode 100644
index 00000000..e4d5cb33
--- /dev/null
+++ b/src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/CallbackApiExample.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.vpp.jvpp.nat.examples;
+
+import io.fd.vpp.jvpp.JVpp;
+import io.fd.vpp.jvpp.JVppRegistry;
+import io.fd.vpp.jvpp.JVppRegistryImpl;
+import io.fd.vpp.jvpp.VppCallbackException;
+import io.fd.vpp.jvpp.nat.JVppNatImpl;
+import io.fd.vpp.jvpp.nat.callback.Nat44InterfaceAddDelFeatureCallback;
+import io.fd.vpp.jvpp.nat.dto.Nat44InterfaceAddDelFeature;
+import io.fd.vpp.jvpp.nat.dto.Nat44InterfaceAddDelFeatureReply;
+
+public class CallbackApiExample {
+
+    static class TestCallback implements Nat44InterfaceAddDelFeatureCallback {
+
+        @Override
+        public void onNat44InterfaceAddDelFeatureReply(final Nat44InterfaceAddDelFeatureReply msg) {
+            System.out.printf("Received Nat44InterfaceAddDelFeatureReply: context=%d%n",
+                msg.context);
+        }
+
+        @Override
+        public void onError(VppCallbackException ex) {
+            System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(),
+                ex.getCtxId(), ex.getErrorCode());
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        testCallbackApi();
+    }
+
+    private static void testCallbackApi() throws Exception {
+        System.out.println("Testing Java callback API for nat plugin");
+        try (final JVppRegistry registry = new JVppRegistryImpl("NatCallbackApiTest");
+             final JVpp jvpp = new JVppNatImpl()) {
+            registry.register(jvpp, new TestCallback());
+
+            System.out.println("Sending Nat44InterfaceAddDelFeature request...");
+            Nat44InterfaceAddDelFeature request = new Nat44InterfaceAddDelFeature();
+            request.isAdd = 1;
+            request.isInside = 1;
+            request.swIfIndex = 1;
+            final int result = jvpp.send(request);
+            System.out.printf("Nat44InterfaceAddDelFeature send result = %d%n", result);
+
+            Thread.sleep(1000);
+
+            System.out.println("Disconnecting...");
+        }
+    }
+}
diff --git a/src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/Readme.txt b/src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/Readme.txt
new file mode 100644
index 00000000..ac75e04e
--- /dev/null
+++ b/src/vpp-api/java/jvpp-nat/io/fd/vpp/jvpp/nat/examples/Readme.txt
@@ -0,0 +1 @@
+sudo java -cp build-vpp-native/vpp/vpp-api/java/jvpp-registry-17.10.jar:build-vpp-native/vpp/vpp-api/java/jvpp-nat-17.10.jar io.fd.vpp.jvpp.nat.examples.CallbackApiExample
diff --git a/src/vpp-api/java/jvpp-nat/jvpp_nat.c b/src/vpp-api/java/jvpp-nat/jvpp_nat.c
new file mode 100644
index 00000000..85217f04
--- /dev/null
+++ b/src/vpp-api/java/jvpp-nat/jvpp_nat.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include 
+
+#include 
+#define vl_typedefs             /* define message structures */
+#include 
+#undef vl_typedefs
+
+#include 
+#include 
+#include 
+
+#if VPPJNI_DEBUG == 1
+  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
+#else
+  #define DEBUG_LOG(...)
+#endif
+
+#include 
+
+#include "jvpp-nat/io_fd_vpp_jvpp_nat_JVppNatImpl.h"
+#include "jvpp_nat.h"
+#include "jvpp-nat/jvpp_nat_gen.h"
+
+/*
+ * Class:     io_fd_vpp_jvpp_nat_JVppNatImpl
+ * Method:    init0
+ * Signature: (JI)V
+ */
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_nat_JVppNatImpl_init0
+  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
+  nat_main_t * plugin_main = &nat_main;
+  clib_warning ("Java_io_fd_vpp_jvpp_nat_JVppNatImpl_init0");
+
+  plugin_main->my_client_index = my_client_index;
+  plugin_main->vl_input_queue = uword_to_pointer (queue_address, unix_shared_memory_queue_t *);
+
+  plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
+  plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
+
+  // verify API has not changed since jar generation
+  #define _(N)             \
+      get_message_id(env, #N);
+      foreach_supported_api_message;
+  #undef _
+
+  #define _(N,n)                                  \
+      vl_msg_api_set_handlers(get_message_id(env, #N), #n,     \
+              vl_api_##n##_t_handler,             \
+              vl_noop_handler,                    \
+              vl_noop_handler,                    \
+              vl_noop_handler,                    \
+              sizeof(vl_api_##n##_t), 1);
+      foreach_api_reply_handler;
+  #undef _
+}
+
+JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_nat_JVppNatImpl_close0
+(JNIEnv *env, jclass clazz) {
+  nat_main_t * plugin_main = &nat_main;
+
+    // cleanup:
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
+    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
+
+    plugin_main->callbackClass = NULL;
+    plugin_main->callbackObject = NULL;
+}
+
+/* Attach thread to JVM and cache class references when initiating JVPP SNAT */
+jint JNI_OnLoad(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return JNI_EVERSION;
+    }
+
+    if (cache_class_references(env) != 0) {
+        clib_warning ("Failed to cache class references\n");
+        return JNI_ERR;
+    }
+
+    return JNI_VERSION_1_8;
+}
+
+/* Clean up cached references when disposing JVPP SNAT */
+void JNI_OnUnload(JavaVM *vm, void *reserved) {
+    JNIEnv* env;
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
+        return;
+    }
+    delete_class_references(env);
+}
diff --git a/src/vpp-api/java/jvpp-nat/jvpp_nat.h b/src/vpp-api/java/jvpp-nat/jvpp_nat.h
new file mode 100644
index 00000000..c8f6b683
--- /dev/null
+++ b/src/vpp-api/java/jvpp-nat/jvpp_nat.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __included_jvpp_nat_h__
+#define __included_jvpp_nat_h__
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* Global state for JVPP-NAT */
+typedef struct {
+    /* Pointer to shared memory queue */
+    unix_shared_memory_queue_t * vl_input_queue;
+
+    /* VPP api client index */
+    u32 my_client_index;
+
+    /* Callback object and class references enabling asynchronous Java calls */
+    jobject callbackObject;
+    jclass callbackClass;
+
+} nat_main_t;
+
+nat_main_t nat_main __attribute__((aligned (64)));
+
+
+#endif /* __included_jvpp_nat_h__ */
diff --git a/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/CallbackApiExample.java b/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/CallbackApiExample.java
deleted file mode 100644
index f4a2943f..00000000
--- a/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/CallbackApiExample.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.fd.vpp.jvpp.snat.examples;
-
-import io.fd.vpp.jvpp.JVpp;
-import io.fd.vpp.jvpp.JVppRegistry;
-import io.fd.vpp.jvpp.JVppRegistryImpl;
-import io.fd.vpp.jvpp.VppCallbackException;
-import io.fd.vpp.jvpp.snat.JVppSnatImpl;
-import io.fd.vpp.jvpp.snat.callback.SnatInterfaceAddDelFeatureCallback;
-import io.fd.vpp.jvpp.snat.dto.SnatInterfaceAddDelFeature;
-import io.fd.vpp.jvpp.snat.dto.SnatInterfaceAddDelFeatureReply;
-
-public class CallbackApiExample {
-
-    static class TestCallback implements SnatInterfaceAddDelFeatureCallback {
-
-        @Override
-        public void onSnatInterfaceAddDelFeatureReply(final SnatInterfaceAddDelFeatureReply msg) {
-            System.out.printf("Received SnatInterfaceAddDelFeatureReply: context=%d%n",
-                msg.context);
-        }
-
-        @Override
-        public void onError(VppCallbackException ex) {
-            System.out.printf("Received onError exception: call=%s, context=%d, retval=%d%n", ex.getMethodName(),
-                ex.getCtxId(), ex.getErrorCode());
-        }
-    }
-
-    public static void main(String[] args) throws Exception {
-        testCallbackApi();
-    }
-
-    private static void testCallbackApi() throws Exception {
-        System.out.println("Testing Java callback API for snat plugin");
-        try (final JVppRegistry registry = new JVppRegistryImpl("SnatCallbackApiTest");
-             final JVpp jvpp = new JVppSnatImpl()) {
-            registry.register(jvpp, new TestCallback());
-
-            System.out.println("Sending SnatInterfaceAddDelFeature request...");
-            SnatInterfaceAddDelFeature request = new SnatInterfaceAddDelFeature();
-            request.isAdd = 1;
-            request.isInside = 1;
-            request.swIfIndex = 1;
-            final int result = jvpp.send(request);
-            System.out.printf("SnatInterfaceAddDelFeature send result = %d%n", result);
-
-            Thread.sleep(1000);
-
-            System.out.println("Disconnecting...");
-        }
-    }
-}
diff --git a/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/Readme.txt b/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/Readme.txt
deleted file mode 100644
index 470850ee..00000000
--- a/src/vpp-api/java/jvpp-snat/io/fd/vpp/jvpp/snat/examples/Readme.txt
+++ /dev/null
@@ -1 +0,0 @@
-sudo java -cp build-vpp-native/vpp/vpp-api/java/jvpp-registry-17.10.jar:build-vpp-native/vpp/vpp-api/java/jvpp-snat-17.10.jar io.fd.vpp.jvpp.snat.examples.CallbackApiExample
diff --git a/src/vpp-api/java/jvpp-snat/jvpp_snat.c b/src/vpp-api/java/jvpp-snat/jvpp_snat.c
deleted file mode 100644
index 5fd6a88b..00000000
--- a/src/vpp-api/java/jvpp-snat/jvpp_snat.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include 
-
-#include 
-#define vl_typedefs             /* define message structures */
-#include 
-#undef vl_typedefs
-
-#include 
-#include 
-#include 
-
-#if VPPJNI_DEBUG == 1
-  #define DEBUG_LOG(...) clib_warning(__VA_ARGS__)
-#else
-  #define DEBUG_LOG(...)
-#endif
-
-#include 
-
-#include "jvpp-snat/io_fd_vpp_jvpp_snat_JVppSnatImpl.h"
-#include "jvpp_snat.h"
-#include "jvpp-snat/jvpp_snat_gen.h"
-
-/*
- * Class:     io_fd_vpp_jvpp_snat_JVppsnatImpl
- * Method:    init0
- * Signature: (JI)V
- */
-JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_init0
-  (JNIEnv *env, jclass clazz, jobject callback, jlong queue_address, jint my_client_index) {
-  snat_main_t * plugin_main = &snat_main;
-  clib_warning ("Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_init0");
-
-  plugin_main->my_client_index = my_client_index;
-  plugin_main->vl_input_queue = uword_to_pointer (queue_address, unix_shared_memory_queue_t *);
-
-  plugin_main->callbackObject = (*env)->NewGlobalRef(env, callback);
-  plugin_main->callbackClass = (jclass)(*env)->NewGlobalRef(env, (*env)->GetObjectClass(env, callback));
-
-  // verify API has not changed since jar generation
-  #define _(N)             \
-      get_message_id(env, #N);
-      foreach_supported_api_message;
-  #undef _
-
-  #define _(N,n)                                  \
-      vl_msg_api_set_handlers(get_message_id(env, #N), #n,     \
-              vl_api_##n##_t_handler,             \
-              vl_noop_handler,                    \
-              vl_noop_handler,                    \
-              vl_noop_handler,                    \
-              sizeof(vl_api_##n##_t), 1);
-      foreach_api_reply_handler;
-  #undef _
-}
-
-JNIEXPORT void JNICALL Java_io_fd_vpp_jvpp_snat_JVppSnatImpl_close0
-(JNIEnv *env, jclass clazz) {
-  snat_main_t * plugin_main = &snat_main;
-
-    // cleanup:
-    (*env)->DeleteGlobalRef(env, plugin_main->callbackClass);
-    (*env)->DeleteGlobalRef(env, plugin_main->callbackObject);
-
-    plugin_main->callbackClass = NULL;
-    plugin_main->callbackObject = NULL;
-}
-
-/* Attach thread to JVM and cache class references when initiating JVPP SNAT */
-jint JNI_OnLoad(JavaVM *vm, void *reserved) {
-    JNIEnv* env;
-
-    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
-        return JNI_EVERSION;
-    }
-
-    if (cache_class_references(env) != 0) {
-        clib_warning ("Failed to cache class references\n");
-        return JNI_ERR;
-    }
-
-    return JNI_VERSION_1_8;
-}
-
-/* Clean up cached references when disposing JVPP SNAT */
-void JNI_OnUnload(JavaVM *vm, void *reserved) {
-    JNIEnv* env;
-    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_8) != JNI_OK) {
-        return;
-    }
-    delete_class_references(env);
-}
diff --git a/src/vpp-api/java/jvpp-snat/jvpp_snat.h b/src/vpp-api/java/jvpp-snat/jvpp_snat.h
deleted file mode 100644
index 7739a411..00000000
--- a/src/vpp-api/java/jvpp-snat/jvpp_snat.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_jvpp_snat_h__
-#define __included_jvpp_snat_h__
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-/* Global state for JVPP-SNAT */
-typedef struct {
-    /* Pointer to shared memory queue */
-    unix_shared_memory_queue_t * vl_input_queue;
-
-    /* VPP api client index */
-    u32 my_client_index;
-
-    /* Callback object and class references enabling asynchronous Java calls */
-    jobject callbackObject;
-    jclass callbackClass;
-
-} snat_main_t;
-
-snat_main_t snat_main __attribute__((aligned (64)));
-
-
-#endif /* __included_jvpp_snat_h__ */
diff --git a/test/test_nat.py b/test/test_nat.py
new file mode 100644
index 00000000..0d622b08
--- /dev/null
+++ b/test/test_nat.py
@@ -0,0 +1,3720 @@
+#!/usr/bin/env python
+
+import socket
+import unittest
+import struct
+
+from framework import VppTestCase, VppTestRunner, running_extended_tests
+from scapy.layers.inet import IP, TCP, UDP, ICMP
+from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
+from scapy.layers.inet6 import IPv6, ICMPv6EchoRequest, ICMPv6EchoReply
+from scapy.layers.inet6 import ICMPv6DestUnreach, IPerror6
+from scapy.layers.l2 import Ether, ARP, GRE
+from scapy.data import IP_PROTOS
+from scapy.packet import bind_layers
+from util import ppp
+from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
+from time import sleep
+
+
+class MethodHolder(VppTestCase):
+    """ NAT create capture and verify method holder """
+
+    @classmethod
+    def setUpClass(cls):
+        super(MethodHolder, cls).setUpClass()
+
+    def tearDown(self):
+        super(MethodHolder, self).tearDown()
+
+    def check_ip_checksum(self, pkt):
+        """
+        Check IP checksum of the packet
+
+        :param pkt: Packet to check IP checksum
+        """
+        new = pkt.__class__(str(pkt))
+        del new['IP'].chksum
+        new = new.__class__(str(new))
+        self.assertEqual(new['IP'].chksum, pkt['IP'].chksum)
+
+    def check_tcp_checksum(self, pkt):
+        """
+        Check TCP checksum in IP packet
+
+        :param pkt: Packet to check TCP checksum
+        """
+        new = pkt.__class__(str(pkt))
+        del new['TCP'].chksum
+        new = new.__class__(str(new))
+        self.assertEqual(new['TCP'].chksum, pkt['TCP'].chksum)
+
+    def check_udp_checksum(self, pkt):
+        """
+        Check UDP checksum in IP packet
+
+        :param pkt: Packet to check UDP checksum
+        """
+        new = pkt.__class__(str(pkt))
+        del new['UDP'].chksum
+        new = new.__class__(str(new))
+        self.assertEqual(new['UDP'].chksum, pkt['UDP'].chksum)
+
+    def check_icmp_errror_embedded(self, pkt):
+        """
+        Check ICMP error embeded packet checksum
+
+        :param pkt: Packet to check ICMP error embeded packet checksum
+        """
+        if pkt.haslayer(IPerror):
+            new = pkt.__class__(str(pkt))
+            del new['IPerror'].chksum
+            new = new.__class__(str(new))
+            self.assertEqual(new['IPerror'].chksum, pkt['IPerror'].chksum)
+
+        if pkt.haslayer(TCPerror):
+            new = pkt.__class__(str(pkt))
+            del new['TCPerror'].chksum
+            new = new.__class__(str(new))
+            self.assertEqual(new['TCPerror'].chksum, pkt['TCPerror'].chksum)
+
+        if pkt.haslayer(UDPerror):
+            if pkt['UDPerror'].chksum != 0:
+                new = pkt.__class__(str(pkt))
+                del new['UDPerror'].chksum
+                new = new.__class__(str(new))
+                self.assertEqual(new['UDPerror'].chksum,
+                                 pkt['UDPerror'].chksum)
+
+        if pkt.haslayer(ICMPerror):
+            del new['ICMPerror'].chksum
+            new = new.__class__(str(new))
+            self.assertEqual(new['ICMPerror'].chksum, pkt['ICMPerror'].chksum)
+
+    def check_icmp_checksum(self, pkt):
+        """
+        Check ICMP checksum in IPv4 packet
+
+        :param pkt: Packet to check ICMP checksum
+        """
+        new = pkt.__class__(str(pkt))
+        del new['ICMP'].chksum
+        new = new.__class__(str(new))
+        self.assertEqual(new['ICMP'].chksum, pkt['ICMP'].chksum)
+        if pkt.haslayer(IPerror):
+            self.check_icmp_errror_embedded(pkt)
+
+    def check_icmpv6_checksum(self, pkt):
+        """
+        Check ICMPv6 checksum in IPv4 packet
+
+        :param pkt: Packet to check ICMPv6 checksum
+        """
+        new = pkt.__class__(str(pkt))
+        if pkt.haslayer(ICMPv6DestUnreach):
+            del new['ICMPv6DestUnreach'].cksum
+            new = new.__class__(str(new))
+            self.assertEqual(new['ICMPv6DestUnreach'].cksum,
+                             pkt['ICMPv6DestUnreach'].cksum)
+            self.check_icmp_errror_embedded(pkt)
+        if pkt.haslayer(ICMPv6EchoRequest):
+            del new['ICMPv6EchoRequest'].cksum
+            new = new.__class__(str(new))
+            self.assertEqual(new['ICMPv6EchoRequest'].cksum,
+                             pkt['ICMPv6EchoRequest'].cksum)
+        if pkt.haslayer(ICMPv6EchoReply):
+            del new['ICMPv6EchoReply'].cksum
+            new = new.__class__(str(new))
+            self.assertEqual(new['ICMPv6EchoReply'].cksum,
+                             pkt['ICMPv6EchoReply'].cksum)
+
+    def create_stream_in(self, in_if, out_if, ttl=64):
+        """
+        Create packet stream for inside network
+
+        :param in_if: Inside interface
+        :param out_if: Outside interface
+        :param ttl: TTL of generated packets
+        """
+        pkts = []
+        # TCP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
+             TCP(sport=self.tcp_port_in, dport=20))
+        pkts.append(p)
+
+        # UDP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
+             UDP(sport=self.udp_port_in, dport=20))
+        pkts.append(p)
+
+        # ICMP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
+             ICMP(id=self.icmp_id_in, type='echo-request'))
+        pkts.append(p)
+
+        return pkts
+
+    def compose_ip6(self, ip4, pref, plen):
+        """
+        Compose IPv4-embedded IPv6 addresses
+
+        :param ip4: IPv4 address
+        :param pref: IPv6 prefix
+        :param plen: IPv6 prefix length
+        :returns: IPv4-embedded IPv6 addresses
+        """
+        pref_n = list(socket.inet_pton(socket.AF_INET6, pref))
+        ip4_n = list(socket.inet_pton(socket.AF_INET, ip4))
+        if plen == 32:
+            pref_n[4] = ip4_n[0]
+            pref_n[5] = ip4_n[1]
+            pref_n[6] = ip4_n[2]
+            pref_n[7] = ip4_n[3]
+        elif plen == 40:
+            pref_n[5] = ip4_n[0]
+            pref_n[6] = ip4_n[1]
+            pref_n[7] = ip4_n[2]
+            pref_n[9] = ip4_n[3]
+        elif plen == 48:
+            pref_n[6] = ip4_n[0]
+            pref_n[7] = ip4_n[1]
+            pref_n[9] = ip4_n[2]
+            pref_n[10] = ip4_n[3]
+        elif plen == 56:
+            pref_n[7] = ip4_n[0]
+            pref_n[9] = ip4_n[1]
+            pref_n[10] = ip4_n[2]
+            pref_n[11] = ip4_n[3]
+        elif plen == 64:
+            pref_n[9] = ip4_n[0]
+            pref_n[10] = ip4_n[1]
+            pref_n[11] = ip4_n[2]
+            pref_n[12] = ip4_n[3]
+        elif plen == 96:
+            pref_n[12] = ip4_n[0]
+            pref_n[13] = ip4_n[1]
+            pref_n[14] = ip4_n[2]
+            pref_n[15] = ip4_n[3]
+        return socket.inet_ntop(socket.AF_INET6, ''.join(pref_n))
+
+    def create_stream_in_ip6(self, in_if, out_if, hlim=64, pref=None, plen=0):
+        """
+        Create IPv6 packet stream for inside network
+
+        :param in_if: Inside interface
+        :param out_if: Outside interface
+        :param ttl: Hop Limit of generated packets
+        :param pref: NAT64 prefix
+        :param plen: NAT64 prefix length
+        """
+        pkts = []
+        if pref is None:
+            dst = ''.join(['64:ff9b::', out_if.remote_ip4])
+        else:
+            dst = self.compose_ip6(out_if.remote_ip4, pref, plen)
+
+        # TCP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IPv6(src=in_if.remote_ip6, dst=dst, hlim=hlim) /
+             TCP(sport=self.tcp_port_in, dport=20))
+        pkts.append(p)
+
+        # UDP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IPv6(src=in_if.remote_ip6, dst=dst, hlim=hlim) /
+             UDP(sport=self.udp_port_in, dport=20))
+        pkts.append(p)
+
+        # ICMP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IPv6(src=in_if.remote_ip6, dst=dst, hlim=hlim) /
+             ICMPv6EchoRequest(id=self.icmp_id_in))
+        pkts.append(p)
+
+        return pkts
+
+    def create_stream_out(self, out_if, dst_ip=None, ttl=64):
+        """
+        Create packet stream for outside network
+
+        :param out_if: Outside interface
+        :param dst_ip: Destination IP address (Default use global NAT address)
+        :param ttl: TTL of generated packets
+        """
+        if dst_ip is None:
+            dst_ip = self.nat_addr
+        pkts = []
+        # TCP
+        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
+             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
+             TCP(dport=self.tcp_port_out, sport=20))
+        pkts.append(p)
+
+        # UDP
+        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
+             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
+             UDP(dport=self.udp_port_out, sport=20))
+        pkts.append(p)
+
+        # ICMP
+        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
+             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
+             ICMP(id=self.icmp_id_out, type='echo-reply'))
+        pkts.append(p)
+
+        return pkts
+
+    def verify_capture_out(self, capture, nat_ip=None, same_port=False,
+                           packet_num=3, dst_ip=None):
+        """
+        Verify captured packets on outside network
+
+        :param capture: Captured packets
+        :param nat_ip: Translated IP address (Default use global NAT address)
+        :param same_port: Sorce port number is not translated (Default False)
+        :param packet_num: Expected number of packets (Default 3)
+        :param dst_ip: Destination IP address (Default do not verify)
+        """
+        if nat_ip is None:
+            nat_ip = self.nat_addr
+        self.assertEqual(packet_num, len(capture))
+        for packet in capture:
+            try:
+                self.check_ip_checksum(packet)
+                self.assertEqual(packet[IP].src, nat_ip)
+                if dst_ip is not None:
+                    self.assertEqual(packet[IP].dst, dst_ip)
+                if packet.haslayer(TCP):
+                    if same_port:
+                        self.assertEqual(packet[TCP].sport, self.tcp_port_in)
+                    else:
+                        self.assertNotEqual(
+                            packet[TCP].sport, self.tcp_port_in)
+                    self.tcp_port_out = packet[TCP].sport
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    if same_port:
+                        self.assertEqual(packet[UDP].sport, self.udp_port_in)
+                    else:
+                        self.assertNotEqual(
+                            packet[UDP].sport, self.udp_port_in)
+                    self.udp_port_out = packet[UDP].sport
+                else:
+                    if same_port:
+                        self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+                    else:
+                        self.assertNotEqual(packet[ICMP].id, self.icmp_id_in)
+                    self.icmp_id_out = packet[ICMP].id
+                    self.check_icmp_checksum(packet)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(outside network):", packet))
+                raise
+
+    def verify_capture_in(self, capture, in_if, packet_num=3):
+        """
+        Verify captured packets on inside network
+
+        :param capture: Captured packets
+        :param in_if: Inside interface
+        :param packet_num: Expected number of packets (Default 3)
+        """
+        self.assertEqual(packet_num, len(capture))
+        for packet in capture:
+            try:
+                self.check_ip_checksum(packet)
+                self.assertEqual(packet[IP].dst, in_if.remote_ip4)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
+                else:
+                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+                    self.check_icmp_checksum(packet)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(inside network):", packet))
+                raise
+
+    def verify_capture_in_ip6(self, capture, src_ip, dst_ip, packet_num=3):
+        """
+        Verify captured IPv6 packets on inside network
+
+        :param capture: Captured packets
+        :param src_ip: Source IP
+        :param dst_ip: Destination IP address
+        :param packet_num: Expected number of packets (Default 3)
+        """
+        self.assertEqual(packet_num, len(capture))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IPv6].src, src_ip)
+                self.assertEqual(packet[IPv6].dst, dst_ip)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
+                    self.check_udp_checksum(packet)
+                else:
+                    self.assertEqual(packet[ICMPv6EchoReply].id,
+                                     self.icmp_id_in)
+                    self.check_icmpv6_checksum(packet)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(inside network):", packet))
+                raise
+
+    def verify_capture_no_translation(self, capture, ingress_if, egress_if):
+        """
+        Verify captured packet that don't have to be translated
+
+        :param capture: Captured packets
+        :param ingress_if: Ingress interface
+        :param egress_if: Egress interface
+        """
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, ingress_if.remote_ip4)
+                self.assertEqual(packet[IP].dst, egress_if.remote_ip4)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].sport, self.tcp_port_in)
+                elif packet.haslayer(UDP):
+                    self.assertEqual(packet[UDP].sport, self.udp_port_in)
+                else:
+                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(inside network):", packet))
+                raise
+
+    def verify_capture_out_with_icmp_errors(self, capture, src_ip=None,
+                                            packet_num=3, icmp_type=11):
+        """
+        Verify captured packets with ICMP errors on outside network
+
+        :param capture: Captured packets
+        :param src_ip: Translated IP address or IP address of VPP
+                       (Default use global NAT address)
+        :param packet_num: Expected number of packets (Default 3)
+        :param icmp_type: Type of error ICMP packet
+                          we are expecting (Default 11)
+        """
+        if src_ip is None:
+            src_ip = self.nat_addr
+        self.assertEqual(packet_num, len(capture))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, src_ip)
+                self.assertTrue(packet.haslayer(ICMP))
+                icmp = packet[ICMP]
+                self.assertEqual(icmp.type, icmp_type)
+                self.assertTrue(icmp.haslayer(IPerror))
+                inner_ip = icmp[IPerror]
+                if inner_ip.haslayer(TCPerror):
+                    self.assertEqual(inner_ip[TCPerror].dport,
+                                     self.tcp_port_out)
+                elif inner_ip.haslayer(UDPerror):
+                    self.assertEqual(inner_ip[UDPerror].dport,
+                                     self.udp_port_out)
+                else:
+                    self.assertEqual(inner_ip[ICMPerror].id, self.icmp_id_out)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(outside network):", packet))
+                raise
+
+    def verify_capture_in_with_icmp_errors(self, capture, in_if, packet_num=3,
+                                           icmp_type=11):
+        """
+        Verify captured packets with ICMP errors on inside network
+
+        :param capture: Captured packets
+        :param in_if: Inside interface
+        :param packet_num: Expected number of packets (Default 3)
+        :param icmp_type: Type of error ICMP packet
+                          we are expecting (Default 11)
+        """
+        self.assertEqual(packet_num, len(capture))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].dst, in_if.remote_ip4)
+                self.assertTrue(packet.haslayer(ICMP))
+                icmp = packet[ICMP]
+                self.assertEqual(icmp.type, icmp_type)
+                self.assertTrue(icmp.haslayer(IPerror))
+                inner_ip = icmp[IPerror]
+                if inner_ip.haslayer(TCPerror):
+                    self.assertEqual(inner_ip[TCPerror].sport,
+                                     self.tcp_port_in)
+                elif inner_ip.haslayer(UDPerror):
+                    self.assertEqual(inner_ip[UDPerror].sport,
+                                     self.udp_port_in)
+                else:
+                    self.assertEqual(inner_ip[ICMPerror].id, self.icmp_id_in)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(inside network):", packet))
+                raise
+
+    def verify_ipfix_nat44_ses(self, data):
+        """
+        Verify IPFIX NAT44 session create/delete event
+
+        :param data: Decoded IPFIX data records
+        """
+        nat44_ses_create_num = 0
+        nat44_ses_delete_num = 0
+        self.assertEqual(6, len(data))
+        for record in data:
+            # natEvent
+            self.assertIn(ord(record[230]), [4, 5])
+            if ord(record[230]) == 4:
+                nat44_ses_create_num += 1
+            else:
+                nat44_ses_delete_num += 1
+            # sourceIPv4Address
+            self.assertEqual(self.pg0.remote_ip4n, record[8])
+            # postNATSourceIPv4Address
+            self.assertEqual(socket.inet_pton(socket.AF_INET, self.nat_addr),
+                             record[225])
+            # ingressVRFID
+            self.assertEqual(struct.pack("!I", 0), record[234])
+            # protocolIdentifier/sourceTransportPort/postNAPTSourceTransportPort
+            if IP_PROTOS.icmp == ord(record[4]):
+                self.assertEqual(struct.pack("!H", self.icmp_id_in), record[7])
+                self.assertEqual(struct.pack("!H", self.icmp_id_out),
+                                 record[227])
+            elif IP_PROTOS.tcp == ord(record[4]):
+                self.assertEqual(struct.pack("!H", self.tcp_port_in),
+                                 record[7])
+                self.assertEqual(struct.pack("!H", self.tcp_port_out),
+                                 record[227])
+            elif IP_PROTOS.udp == ord(record[4]):
+                self.assertEqual(struct.pack("!H", self.udp_port_in),
+                                 record[7])
+                self.assertEqual(struct.pack("!H", self.udp_port_out),
+                                 record[227])
+            else:
+                self.fail("Invalid protocol")
+        self.assertEqual(3, nat44_ses_create_num)
+        self.assertEqual(3, nat44_ses_delete_num)
+
+    def verify_ipfix_addr_exhausted(self, data):
+        """
+        Verify IPFIX NAT addresses event
+
+        :param data: Decoded IPFIX data records
+        """
+        self.assertEqual(1, len(data))
+        record = data[0]
+        # natEvent
+        self.assertEqual(ord(record[230]), 3)
+        # natPoolID
+        self.assertEqual(struct.pack("!I", 0), record[283])
+
+
+class TestNAT44(MethodHolder):
+    """ NAT44 Test Cases """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestNAT44, cls).setUpClass()
+
+        try:
+            cls.tcp_port_in = 6303
+            cls.tcp_port_out = 6303
+            cls.udp_port_in = 6304
+            cls.udp_port_out = 6304
+            cls.icmp_id_in = 6305
+            cls.icmp_id_out = 6305
+            cls.nat_addr = '10.0.0.3'
+            cls.ipfix_src_port = 4739
+            cls.ipfix_domain_id = 1
+
+            cls.create_pg_interfaces(range(9))
+            cls.interfaces = list(cls.pg_interfaces[0:4])
+
+            for i in cls.interfaces:
+                i.admin_up()
+                i.config_ip4()
+                i.resolve_arp()
+
+            cls.pg0.generate_remote_hosts(3)
+            cls.pg0.configure_ipv4_neighbors()
+
+            cls.overlapping_interfaces = list(list(cls.pg_interfaces[4:7]))
+
+            cls.pg4._local_ip4 = "172.16.255.1"
+            cls.pg4._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
+            cls.pg4._remote_hosts[0]._ip4 = "172.16.255.2"
+            cls.pg4.set_table_ip4(10)
+            cls.pg5._local_ip4 = "172.17.255.3"
+            cls.pg5._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
+            cls.pg5._remote_hosts[0]._ip4 = "172.17.255.4"
+            cls.pg5.set_table_ip4(10)
+            cls.pg6._local_ip4 = "172.16.255.1"
+            cls.pg6._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
+            cls.pg6._remote_hosts[0]._ip4 = "172.16.255.2"
+            cls.pg6.set_table_ip4(20)
+            for i in cls.overlapping_interfaces:
+                i.config_ip4()
+                i.admin_up()
+                i.resolve_arp()
+
+            cls.pg7.admin_up()
+            cls.pg8.admin_up()
+
+        except Exception:
+            super(TestNAT44, cls).tearDownClass()
+            raise
+
+    def clear_nat44(self):
+        """
+        Clear NAT44 configuration.
+        """
+        # I found no elegant way to do this
+        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg7.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg7.sw_if_index,
+                                   is_add=0)
+        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg8.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg8.sw_if_index,
+                                   is_add=0)
+
+        for intf in [self.pg7, self.pg8]:
+            neighbors = self.vapi.ip_neighbor_dump(intf.sw_if_index)
+            for n in neighbors:
+                self.vapi.ip_neighbor_add_del(intf.sw_if_index,
+                                              n.mac_address,
+                                              n.ip_address,
+                                              is_add=0)
+
+        if self.pg7.has_ip4_config:
+            self.pg7.unconfig_ip4()
+
+        interfaces = self.vapi.nat44_interface_addr_dump()
+        for intf in interfaces:
+            self.vapi.nat44_add_interface_addr(intf.sw_if_index, is_add=0)
+
+        self.vapi.nat_ipfix(enable=0, src_port=self.ipfix_src_port,
+                            domain_id=self.ipfix_domain_id)
+        self.ipfix_src_port = 4739
+        self.ipfix_domain_id = 1
+
+        interfaces = self.vapi.nat44_interface_dump()
+        for intf in interfaces:
+            self.vapi.nat44_interface_add_del_feature(intf.sw_if_index,
+                                                      intf.is_inside,
+                                                      is_add=0)
+
+        interfaces = self.vapi.nat44_interface_output_feature_dump()
+        for intf in interfaces:
+            self.vapi.nat44_interface_add_del_output_feature(intf.sw_if_index,
+                                                             intf.is_inside,
+                                                             is_add=0)
+
+        static_mappings = self.vapi.nat44_static_mapping_dump()
+        for sm in static_mappings:
+            self.vapi.nat44_add_del_static_mapping(
+                sm.local_ip_address,
+                sm.external_ip_address,
+                local_port=sm.local_port,
+                external_port=sm.external_port,
+                addr_only=sm.addr_only,
+                vrf_id=sm.vrf_id,
+                protocol=sm.protocol,
+                is_add=0)
+
+        adresses = self.vapi.nat44_address_dump()
+        for addr in adresses:
+            self.vapi.nat44_add_del_address_range(addr.ip_address,
+                                                  addr.ip_address,
+                                                  is_add=0)
+
+    def nat44_add_static_mapping(self, local_ip, external_ip='0.0.0.0',
+                                 local_port=0, external_port=0, vrf_id=0,
+                                 is_add=1, external_sw_if_index=0xFFFFFFFF,
+                                 proto=0):
+        """
+        Add/delete NAT44 static mapping
+
+        :param local_ip: Local IP address
+        :param external_ip: External IP address
+        :param local_port: Local port number (Optional)
+        :param external_port: External port number (Optional)
+        :param vrf_id: VRF ID (Default 0)
+        :param is_add: 1 if add, 0 if delete (Default add)
+        :param external_sw_if_index: External interface instead of IP address
+        :param proto: IP protocol (Mandatory if port specified)
+        """
+        addr_only = 1
+        if local_port and external_port:
+            addr_only = 0
+        l_ip = socket.inet_pton(socket.AF_INET, local_ip)
+        e_ip = socket.inet_pton(socket.AF_INET, external_ip)
+        self.vapi.nat44_add_del_static_mapping(
+            l_ip,
+            e_ip,
+            external_sw_if_index,
+            local_port,
+            external_port,
+            addr_only,
+            vrf_id,
+            proto,
+            is_add)
+
+    def nat44_add_address(self, ip, is_add=1, vrf_id=0xFFFFFFFF):
+        """
+        Add/delete NAT44 address
+
+        :param ip: IP address
+        :param is_add: 1 if add, 0 if delete (Default add)
+        """
+        nat_addr = socket.inet_pton(socket.AF_INET, ip)
+        self.vapi.nat44_add_del_address_range(nat_addr, nat_addr, is_add,
+                                              vrf_id=vrf_id)
+
+    def test_dynamic(self):
+        """ NAT44 dynamic translation test """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+    def test_dynamic_icmp_errors_in2out_ttl_1(self):
+        """ NAT44 handling of client packets with TTL=1 """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # Client side - generate traffic
+        pkts = self.create_stream_in(self.pg0, self.pg1, ttl=1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Client side - verify ICMP type 11 packets
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in_with_icmp_errors(capture, self.pg0)
+
+    def test_dynamic_icmp_errors_out2in_ttl_1(self):
+        """ NAT44 handling of server packets with TTL=1 """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # Client side - create sessions
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Server side - generate traffic
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+        pkts = self.create_stream_out(self.pg1, ttl=1)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Server side - verify ICMP type 11 packets
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out_with_icmp_errors(capture,
+                                                 src_ip=self.pg1.local_ip4)
+
+    def test_dynamic_icmp_errors_in2out_ttl_2(self):
+        """ NAT44 handling of error responses to client packets with TTL=2 """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # Client side - generate traffic
+        pkts = self.create_stream_in(self.pg0, self.pg1, ttl=2)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Server side - simulate ICMP type 11 response
+        capture = self.pg1.get_capture(len(pkts))
+        pkts = [Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+                IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+                ICMP(type=11) / packet[IP] for packet in capture]
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Client side - verify ICMP type 11 packets
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in_with_icmp_errors(capture, self.pg0)
+
+    def test_dynamic_icmp_errors_out2in_ttl_2(self):
+        """ NAT44 handling of error responses to server packets with TTL=2 """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # Client side - create sessions
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Server side - generate traffic
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+        pkts = self.create_stream_out(self.pg1, ttl=2)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Client side - simulate ICMP type 11 response
+        capture = self.pg0.get_capture(len(pkts))
+        pkts = [Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+                IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                ICMP(type=11) / packet[IP] for packet in capture]
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # Server side - verify ICMP type 11 packets
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out_with_icmp_errors(capture)
+
+    def test_ping_out_interface_from_outside(self):
+        """ Ping NAT44 out interface from outside network """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+             IP(src=self.pg1.remote_ip4, dst=self.pg1.local_ip4) /
+             ICMP(id=self.icmp_id_out, type='echo-request'))
+        pkts = [p]
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.assertEqual(1, len(capture))
+        packet = capture[0]
+        try:
+            self.assertEqual(packet[IP].src, self.pg1.local_ip4)
+            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+            self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+            self.assertEqual(packet[ICMP].type, 0)  # echo reply
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet "
+                                  "(outside network):", packet))
+            raise
+
+    def test_ping_internal_host_from_outside(self):
+        """ Ping internal host from outside network """
+
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # out2in
+        pkt = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+               IP(src=self.pg1.remote_ip4, dst=self.nat_addr, ttl=64) /
+               ICMP(id=self.icmp_id_out, type='echo-request'))
+        self.pg1.add_stream(pkt)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        self.verify_capture_in(capture, self.pg0, packet_num=1)
+        self.assert_equal(capture[0][IP].proto, IP_PROTOS.icmp)
+
+        # in2out
+        pkt = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+               IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4, ttl=64) /
+               ICMP(id=self.icmp_id_in, type='echo-reply'))
+        self.pg0.add_stream(pkt)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(1)
+        self.verify_capture_out(capture, same_port=True, packet_num=1)
+        self.assert_equal(capture[0][IP].proto, IP_PROTOS.icmp)
+
+    def test_static_in(self):
+        """ 1:1 NAT initialized from inside network """
+
+        nat_ip = "10.0.0.10"
+        self.tcp_port_out = 6303
+        self.udp_port_out = 6304
+        self.icmp_id_out = 6305
+
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, nat_ip)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip, True)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1, nat_ip)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+    def test_static_out(self):
+        """ 1:1 NAT initialized from outside network """
+
+        nat_ip = "10.0.0.20"
+        self.tcp_port_out = 6303
+        self.udp_port_out = 6304
+        self.icmp_id_out = 6305
+
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, nat_ip)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1, nat_ip)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip, True)
+
+    def test_static_with_port_in(self):
+        """ 1:1 NAPT initialized from inside network """
+
+        self.tcp_port_out = 3606
+        self.udp_port_out = 3607
+        self.icmp_id_out = 3608
+
+        self.nat44_add_address(self.nat_addr)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr,
+                                      self.tcp_port_in, self.tcp_port_out,
+                                      proto=IP_PROTOS.tcp)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr,
+                                      self.udp_port_in, self.udp_port_out,
+                                      proto=IP_PROTOS.udp)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr,
+                                      self.icmp_id_in, self.icmp_id_out,
+                                      proto=IP_PROTOS.icmp)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+    def test_static_with_port_out(self):
+        """ 1:1 NAPT initialized from outside network """
+
+        self.tcp_port_out = 30606
+        self.udp_port_out = 30607
+        self.icmp_id_out = 30608
+
+        self.nat44_add_address(self.nat_addr)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr,
+                                      self.tcp_port_in, self.tcp_port_out,
+                                      proto=IP_PROTOS.tcp)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr,
+                                      self.udp_port_in, self.udp_port_out,
+                                      proto=IP_PROTOS.udp)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, self.nat_addr,
+                                      self.icmp_id_in, self.icmp_id_out,
+                                      proto=IP_PROTOS.icmp)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+    def test_static_vrf_aware(self):
+        """ 1:1 NAT VRF awareness """
+
+        nat_ip1 = "10.0.0.30"
+        nat_ip2 = "10.0.0.40"
+        self.tcp_port_out = 6303
+        self.udp_port_out = 6304
+        self.icmp_id_out = 6305
+
+        self.nat44_add_static_mapping(self.pg4.remote_ip4, nat_ip1,
+                                      vrf_id=10)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, nat_ip2,
+                                      vrf_id=10)
+        self.vapi.nat44_interface_add_del_feature(self.pg3.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg4.sw_if_index)
+
+        # inside interface VRF match NAT44 static mapping VRF
+        pkts = self.create_stream_in(self.pg4, self.pg3)
+        self.pg4.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip1, True)
+
+        # inside interface VRF don't match NAT44 static mapping VRF (packets
+        # are dropped)
+        pkts = self.create_stream_in(self.pg0, self.pg3)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        self.pg3.assert_nothing_captured()
+
+    def test_multiple_inside_interfaces(self):
+        """ NAT44 multiple non-overlapping address space inside interfaces """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg3.sw_if_index,
+                                                  is_inside=0)
+
+        # between two NAT44 inside interfaces (no translation)
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_no_translation(capture, self.pg0, self.pg1)
+
+        # from NAT44 inside to interface without NAT44 feature (no translation)
+        pkts = self.create_stream_in(self.pg0, self.pg2)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        self.verify_capture_no_translation(capture, self.pg0, self.pg2)
+
+        # in2out 1st interface
+        pkts = self.create_stream_in(self.pg0, self.pg3)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in 1st interface
+        pkts = self.create_stream_out(self.pg3)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+        # in2out 2nd interface
+        pkts = self.create_stream_in(self.pg1, self.pg3)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in 2nd interface
+        pkts = self.create_stream_out(self.pg3)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg1)
+
+    def test_inside_overlapping_interfaces(self):
+        """ NAT44 multiple inside interfaces with overlapping address space """
+
+        static_nat_ip = "10.0.0.10"
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg3.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.nat44_interface_add_del_feature(self.pg4.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg5.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg6.sw_if_index)
+        self.nat44_add_static_mapping(self.pg6.remote_ip4, static_nat_ip,
+                                      vrf_id=20)
+
+        # between NAT44 inside interfaces with same VRF (no translation)
+        pkts = self.create_stream_in(self.pg4, self.pg5)
+        self.pg4.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(len(pkts))
+        self.verify_capture_no_translation(capture, self.pg4, self.pg5)
+
+        # between NAT44 inside interfaces with different VRF (hairpinning)
+        p = (Ether(src=self.pg4.remote_mac, dst=self.pg4.local_mac) /
+             IP(src=self.pg4.remote_ip4, dst=static_nat_ip) /
+             TCP(sport=1234, dport=5678))
+        self.pg4.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.nat_addr)
+            self.assertEqual(ip.dst, self.pg6.remote_ip4)
+            self.assertNotEqual(tcp.sport, 1234)
+            self.assertEqual(tcp.dport, 5678)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # in2out 1st interface
+        pkts = self.create_stream_in(self.pg4, self.pg3)
+        self.pg4.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in 1st interface
+        pkts = self.create_stream_out(self.pg3)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg4.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg4)
+
+        # in2out 2nd interface
+        pkts = self.create_stream_in(self.pg5, self.pg3)
+        self.pg5.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in 2nd interface
+        pkts = self.create_stream_out(self.pg3)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg5)
+
+        # pg5 session dump
+        addresses = self.vapi.nat44_address_dump()
+        self.assertEqual(len(addresses), 1)
+        sessions = self.vapi.nat44_user_session_dump(self.pg5.remote_ip4n, 10)
+        self.assertEqual(len(sessions), 3)
+        for session in sessions:
+            self.assertFalse(session.is_static)
+            self.assertEqual(session.inside_ip_address[0:4],
+                             self.pg5.remote_ip4n)
+            self.assertEqual(session.outside_ip_address,
+                             addresses[0].ip_address)
+        self.assertEqual(sessions[0].protocol, IP_PROTOS.tcp)
+        self.assertEqual(sessions[1].protocol, IP_PROTOS.udp)
+        self.assertEqual(sessions[2].protocol, IP_PROTOS.icmp)
+        self.assertEqual(sessions[0].inside_port, self.tcp_port_in)
+        self.assertEqual(sessions[1].inside_port, self.udp_port_in)
+        self.assertEqual(sessions[2].inside_port, self.icmp_id_in)
+        self.assertEqual(sessions[0].outside_port, self.tcp_port_out)
+        self.assertEqual(sessions[1].outside_port, self.udp_port_out)
+        self.assertEqual(sessions[2].outside_port, self.icmp_id_out)
+
+        # in2out 3rd interface
+        pkts = self.create_stream_in(self.pg6, self.pg3)
+        self.pg6.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture, static_nat_ip, True)
+
+        # out2in 3rd interface
+        pkts = self.create_stream_out(self.pg3, static_nat_ip)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg6)
+
+        # general user and session dump verifications
+        users = self.vapi.nat44_user_dump()
+        self.assertTrue(len(users) >= 3)
+        addresses = self.vapi.nat44_address_dump()
+        self.assertEqual(len(addresses), 1)
+        for user in users:
+            sessions = self.vapi.nat44_user_session_dump(user.ip_address,
+                                                         user.vrf_id)
+            for session in sessions:
+                self.assertEqual(user.ip_address, session.inside_ip_address)
+                self.assertTrue(session.total_bytes > session.total_pkts > 0)
+                self.assertTrue(session.protocol in
+                                [IP_PROTOS.tcp, IP_PROTOS.udp,
+                                 IP_PROTOS.icmp])
+
+        # pg4 session dump
+        sessions = self.vapi.nat44_user_session_dump(self.pg4.remote_ip4n, 10)
+        self.assertTrue(len(sessions) >= 4)
+        for session in sessions:
+            self.assertFalse(session.is_static)
+            self.assertEqual(session.inside_ip_address[0:4],
+                             self.pg4.remote_ip4n)
+            self.assertEqual(session.outside_ip_address,
+                             addresses[0].ip_address)
+
+        # pg6 session dump
+        sessions = self.vapi.nat44_user_session_dump(self.pg6.remote_ip4n, 20)
+        self.assertTrue(len(sessions) >= 3)
+        for session in sessions:
+            self.assertTrue(session.is_static)
+            self.assertEqual(session.inside_ip_address[0:4],
+                             self.pg6.remote_ip4n)
+            self.assertEqual(map(ord, session.outside_ip_address[0:4]),
+                             map(int, static_nat_ip.split('.')))
+            self.assertTrue(session.inside_port in
+                            [self.tcp_port_in, self.udp_port_in,
+                             self.icmp_id_in])
+
+    def test_hairpinning(self):
+        """ NAT44 hairpinning - 1:1 NAPT """
+
+        host = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        host_in_port = 1234
+        host_out_port = 0
+        server_in_port = 5678
+        server_out_port = 8765
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+        # add static mapping for server
+        self.nat44_add_static_mapping(server.ip4, self.nat_addr,
+                                      server_in_port, server_out_port,
+                                      proto=IP_PROTOS.tcp)
+
+        # send packet from host to server
+        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
+             IP(src=host.ip4, dst=self.nat_addr) /
+             TCP(sport=host_in_port, dport=server_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.nat_addr)
+            self.assertEqual(ip.dst, server.ip4)
+            self.assertNotEqual(tcp.sport, host_in_port)
+            self.assertEqual(tcp.dport, server_in_port)
+            self.check_tcp_checksum(p)
+            host_out_port = tcp.sport
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # send reply from server to host
+        p = (Ether(src=server.mac, dst=self.pg0.local_mac) /
+             IP(src=server.ip4, dst=self.nat_addr) /
+             TCP(sport=server_in_port, dport=host_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.nat_addr)
+            self.assertEqual(ip.dst, host.ip4)
+            self.assertEqual(tcp.sport, server_out_port)
+            self.assertEqual(tcp.dport, host_in_port)
+            self.check_tcp_checksum(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:"), p)
+            raise
+
+    def test_hairpinning2(self):
+        """ NAT44 hairpinning - 1:1 NAT"""
+
+        server1_nat_ip = "10.0.0.10"
+        server2_nat_ip = "10.0.0.11"
+        host = self.pg0.remote_hosts[0]
+        server1 = self.pg0.remote_hosts[1]
+        server2 = self.pg0.remote_hosts[2]
+        server_tcp_port = 22
+        server_udp_port = 20
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # add static mapping for servers
+        self.nat44_add_static_mapping(server1.ip4, server1_nat_ip)
+        self.nat44_add_static_mapping(server2.ip4, server2_nat_ip)
+
+        # host to server1
+        pkts = []
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=host.ip4, dst=server1_nat_ip) /
+             TCP(sport=self.tcp_port_in, dport=server_tcp_port))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=host.ip4, dst=server1_nat_ip) /
+             UDP(sport=self.udp_port_in, dport=server_udp_port))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=host.ip4, dst=server1_nat_ip) /
+             ICMP(id=self.icmp_id_in, type='echo-request'))
+        pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, self.nat_addr)
+                self.assertEqual(packet[IP].dst, server1.ip4)
+                if packet.haslayer(TCP):
+                    self.assertNotEqual(packet[TCP].sport, self.tcp_port_in)
+                    self.assertEqual(packet[TCP].dport, server_tcp_port)
+                    self.tcp_port_out = packet[TCP].sport
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    self.assertNotEqual(packet[UDP].sport, self.udp_port_in)
+                    self.assertEqual(packet[UDP].dport, server_udp_port)
+                    self.udp_port_out = packet[UDP].sport
+                else:
+                    self.assertNotEqual(packet[ICMP].id, self.icmp_id_in)
+                    self.icmp_id_out = packet[ICMP].id
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+        # server1 to host
+        pkts = []
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server1.ip4, dst=self.nat_addr) /
+             TCP(sport=server_tcp_port, dport=self.tcp_port_out))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server1.ip4, dst=self.nat_addr) /
+             UDP(sport=server_udp_port, dport=self.udp_port_out))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server1.ip4, dst=self.nat_addr) /
+             ICMP(id=self.icmp_id_out, type='echo-reply'))
+        pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, server1_nat_ip)
+                self.assertEqual(packet[IP].dst, host.ip4)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
+                    self.assertEqual(packet[TCP].sport, server_tcp_port)
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
+                    self.assertEqual(packet[UDP].sport, server_udp_port)
+                else:
+                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+        # server2 to server1
+        pkts = []
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server2.ip4, dst=server1_nat_ip) /
+             TCP(sport=self.tcp_port_in, dport=server_tcp_port))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server2.ip4, dst=server1_nat_ip) /
+             UDP(sport=self.udp_port_in, dport=server_udp_port))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server2.ip4, dst=server1_nat_ip) /
+             ICMP(id=self.icmp_id_in, type='echo-request'))
+        pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, server2_nat_ip)
+                self.assertEqual(packet[IP].dst, server1.ip4)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].sport, self.tcp_port_in)
+                    self.assertEqual(packet[TCP].dport, server_tcp_port)
+                    self.tcp_port_out = packet[TCP].sport
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    self.assertEqual(packet[UDP].sport, self.udp_port_in)
+                    self.assertEqual(packet[UDP].dport, server_udp_port)
+                    self.udp_port_out = packet[UDP].sport
+                else:
+                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+                    self.icmp_id_out = packet[ICMP].id
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+        # server1 to server2
+        pkts = []
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server1.ip4, dst=server2_nat_ip) /
+             TCP(sport=server_tcp_port, dport=self.tcp_port_out))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server1.ip4, dst=server2_nat_ip) /
+             UDP(sport=server_udp_port, dport=self.udp_port_out))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=server1.ip4, dst=server2_nat_ip) /
+             ICMP(id=self.icmp_id_out, type='echo-reply'))
+        pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, server1_nat_ip)
+                self.assertEqual(packet[IP].dst, server2.ip4)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
+                    self.assertEqual(packet[TCP].sport, server_tcp_port)
+                    self.check_tcp_checksum(packet)
+                elif packet.haslayer(UDP):
+                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
+                    self.assertEqual(packet[UDP].sport, server_udp_port)
+                else:
+                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+    def test_max_translations_per_user(self):
+        """ MAX translations per user - recycle the least recently used """
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # get maximum number of translations per user
+        nat44_config = self.vapi.nat_show_config()
+
+        # send more than maximum number of translations per user packets
+        pkts_num = nat44_config.max_translations_per_user + 5
+        pkts = []
+        for port in range(0, pkts_num):
+            p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                 TCP(sport=1025 + port))
+            pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # verify number of translated packet
+        self.pg1.get_capture(pkts_num)
+
+    def test_interface_addr(self):
+        """ Acquire NAT44 addresses from interface """
+        self.vapi.nat44_add_interface_addr(self.pg7.sw_if_index)
+
+        # no address in NAT pool
+        adresses = self.vapi.nat44_address_dump()
+        self.assertEqual(0, len(adresses))
+
+        # configure interface address and check NAT address pool
+        self.pg7.config_ip4()
+        adresses = self.vapi.nat44_address_dump()
+        self.assertEqual(1, len(adresses))
+        self.assertEqual(adresses[0].ip_address[0:4], self.pg7.local_ip4n)
+
+        # remove interface address and check NAT address pool
+        self.pg7.unconfig_ip4()
+        adresses = self.vapi.nat44_address_dump()
+        self.assertEqual(0, len(adresses))
+
+    def test_interface_addr_static_mapping(self):
+        """ Static mapping with addresses from interface """
+        self.vapi.nat44_add_interface_addr(self.pg7.sw_if_index)
+        self.nat44_add_static_mapping(
+            '1.2.3.4',
+            external_sw_if_index=self.pg7.sw_if_index)
+
+        # static mappings with external interface
+        static_mappings = self.vapi.nat44_static_mapping_dump()
+        self.assertEqual(1, len(static_mappings))
+        self.assertEqual(self.pg7.sw_if_index,
+                         static_mappings[0].external_sw_if_index)
+
+        # configure interface address and check static mappings
+        self.pg7.config_ip4()
+        static_mappings = self.vapi.nat44_static_mapping_dump()
+        self.assertEqual(1, len(static_mappings))
+        self.assertEqual(static_mappings[0].external_ip_address[0:4],
+                         self.pg7.local_ip4n)
+        self.assertEqual(0xFFFFFFFF, static_mappings[0].external_sw_if_index)
+
+        # remove interface address and check static mappings
+        self.pg7.unconfig_ip4()
+        static_mappings = self.vapi.nat44_static_mapping_dump()
+        self.assertEqual(0, len(static_mappings))
+
+    def test_ipfix_nat44_sess(self):
+        """ IPFIX logging NAT44 session created/delted """
+        self.ipfix_domain_id = 10
+        self.ipfix_src_port = 20202
+        colector_port = 30303
+        bind_layers(UDP, IPFIX, dport=30303)
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.set_ipfix_exporter(collector_address=self.pg3.remote_ip4n,
+                                     src_address=self.pg3.local_ip4n,
+                                     path_mtu=512,
+                                     template_interval=10,
+                                     collector_port=colector_port)
+        self.vapi.nat_ipfix(domain_id=self.ipfix_domain_id,
+                            src_port=self.ipfix_src_port)
+
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+        self.nat44_add_address(self.nat_addr, is_add=0)
+        self.vapi.cli("ipfix flush")  # FIXME this should be an API call
+        capture = self.pg3.get_capture(3)
+        ipfix = IPFIXDecoder()
+        # first load template
+        for p in capture:
+            self.assertTrue(p.haslayer(IPFIX))
+            self.assertEqual(p[IP].src, self.pg3.local_ip4)
+            self.assertEqual(p[IP].dst, self.pg3.remote_ip4)
+            self.assertEqual(p[UDP].sport, self.ipfix_src_port)
+            self.assertEqual(p[UDP].dport, colector_port)
+            self.assertEqual(p[IPFIX].observationDomainID,
+                             self.ipfix_domain_id)
+            if p.haslayer(Template):
+                ipfix.add_template(p.getlayer(Template))
+        # verify events in data set
+        for p in capture:
+            if p.haslayer(Data):
+                data = ipfix.decode_data_set(p.getlayer(Set))
+                self.verify_ipfix_nat44_ses(data)
+
+    def test_ipfix_addr_exhausted(self):
+        """ IPFIX logging NAT addresses exhausted """
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.set_ipfix_exporter(collector_address=self.pg3.remote_ip4n,
+                                     src_address=self.pg3.local_ip4n,
+                                     path_mtu=512,
+                                     template_interval=10)
+        self.vapi.nat_ipfix(domain_id=self.ipfix_domain_id,
+                            src_port=self.ipfix_src_port)
+
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             TCP(sport=3025))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(0)
+        self.vapi.cli("ipfix flush")  # FIXME this should be an API call
+        capture = self.pg3.get_capture(3)
+        ipfix = IPFIXDecoder()
+        # first load template
+        for p in capture:
+            self.assertTrue(p.haslayer(IPFIX))
+            self.assertEqual(p[IP].src, self.pg3.local_ip4)
+            self.assertEqual(p[IP].dst, self.pg3.remote_ip4)
+            self.assertEqual(p[UDP].sport, self.ipfix_src_port)
+            self.assertEqual(p[UDP].dport, 4739)
+            self.assertEqual(p[IPFIX].observationDomainID,
+                             self.ipfix_domain_id)
+            if p.haslayer(Template):
+                ipfix.add_template(p.getlayer(Template))
+        # verify events in data set
+        for p in capture:
+            if p.haslayer(Data):
+                data = ipfix.decode_data_set(p.getlayer(Set))
+                self.verify_ipfix_addr_exhausted(data)
+
+    def test_pool_addr_fib(self):
+        """ NAT44 add pool addresses to FIB """
+        static_addr = '10.0.0.10'
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, static_addr)
+
+        # NAT44 address
+        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
+             ARP(op=ARP.who_has, pdst=self.nat_addr,
+                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(1)
+        self.assertTrue(capture[0].haslayer(ARP))
+        self.assertTrue(capture[0][ARP].op, ARP.is_at)
+
+        # 1:1 NAT address
+        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
+             ARP(op=ARP.who_has, pdst=static_addr,
+                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(1)
+        self.assertTrue(capture[0].haslayer(ARP))
+        self.assertTrue(capture[0][ARP].op, ARP.is_at)
+
+        # send ARP to non-NAT44 interface
+        p = (Ether(src=self.pg2.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
+             ARP(op=ARP.who_has, pdst=self.nat_addr,
+                 psrc=self.pg2.remote_ip4, hwsrc=self.pg2.remote_mac))
+        self.pg2.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(0)
+
+        # remove addresses and verify
+        self.nat44_add_address(self.nat_addr, is_add=0)
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, static_addr,
+                                      is_add=0)
+
+        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
+             ARP(op=ARP.who_has, pdst=self.nat_addr,
+                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(0)
+
+        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
+             ARP(op=ARP.who_has, pdst=static_addr,
+                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(0)
+
+    def test_vrf_mode(self):
+        """ NAT44 tenant VRF aware address pool mode """
+
+        vrf_id1 = 1
+        vrf_id2 = 2
+        nat_ip1 = "10.0.0.10"
+        nat_ip2 = "10.0.0.11"
+
+        self.pg0.unconfig_ip4()
+        self.pg1.unconfig_ip4()
+        self.pg0.set_table_ip4(vrf_id1)
+        self.pg1.set_table_ip4(vrf_id2)
+        self.pg0.config_ip4()
+        self.pg1.config_ip4()
+
+        self.nat44_add_address(nat_ip1, vrf_id=vrf_id1)
+        self.nat44_add_address(nat_ip2, vrf_id=vrf_id2)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg2.sw_if_index,
+                                                  is_inside=0)
+
+        # first VRF
+        pkts = self.create_stream_in(self.pg0, self.pg2)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip1)
+
+        # second VRF
+        pkts = self.create_stream_in(self.pg1, self.pg2)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip2)
+
+    def test_vrf_feature_independent(self):
+        """ NAT44 tenant VRF independent address pool mode """
+
+        nat_ip1 = "10.0.0.10"
+        nat_ip2 = "10.0.0.11"
+
+        self.nat44_add_address(nat_ip1)
+        self.nat44_add_address(nat_ip2)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg2.sw_if_index,
+                                                  is_inside=0)
+
+        # first VRF
+        pkts = self.create_stream_in(self.pg0, self.pg2)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip1)
+
+        # second VRF
+        pkts = self.create_stream_in(self.pg1, self.pg2)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip1)
+
+    def test_dynamic_ipless_interfaces(self):
+        """ NAT44 interfaces without configured IP address """
+
+        self.vapi.ip_neighbor_add_del(self.pg7.sw_if_index,
+                                      self.pg7.remote_mac,
+                                      self.pg7.remote_ip4n,
+                                      is_static=1)
+        self.vapi.ip_neighbor_add_del(self.pg8.sw_if_index,
+                                      self.pg8.remote_mac,
+                                      self.pg8.remote_ip4n,
+                                      is_static=1)
+
+        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg7.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg7.sw_if_index)
+        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg8.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg8.sw_if_index)
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg7.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg8.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg7, self.pg8)
+        self.pg7.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg8.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg8, self.nat_addr)
+        self.pg8.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg7.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg7)
+
+    def test_static_ipless_interfaces(self):
+        """ NAT44 interfaces without configured IP address - 1:1 NAT """
+
+        self.vapi.ip_neighbor_add_del(self.pg7.sw_if_index,
+                                      self.pg7.remote_mac,
+                                      self.pg7.remote_ip4n,
+                                      is_static=1)
+        self.vapi.ip_neighbor_add_del(self.pg8.sw_if_index,
+                                      self.pg8.remote_mac,
+                                      self.pg8.remote_ip4n,
+                                      is_static=1)
+
+        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg7.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg7.sw_if_index)
+        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg8.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg8.sw_if_index)
+
+        self.nat44_add_static_mapping(self.pg7.remote_ip4, self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg7.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg8.sw_if_index,
+                                                  is_inside=0)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg8)
+        self.pg8.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg7.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg7)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg7, self.pg8)
+        self.pg7.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg8.get_capture(len(pkts))
+        self.verify_capture_out(capture, self.nat_addr, True)
+
+    def test_static_with_port_ipless_interfaces(self):
+        """ NAT44 interfaces without configured IP address - 1:1 NAPT """
+
+        self.tcp_port_out = 30606
+        self.udp_port_out = 30607
+        self.icmp_id_out = 30608
+
+        self.vapi.ip_neighbor_add_del(self.pg7.sw_if_index,
+                                      self.pg7.remote_mac,
+                                      self.pg7.remote_ip4n,
+                                      is_static=1)
+        self.vapi.ip_neighbor_add_del(self.pg8.sw_if_index,
+                                      self.pg8.remote_mac,
+                                      self.pg8.remote_ip4n,
+                                      is_static=1)
+
+        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg7.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg7.sw_if_index)
+        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg8.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg8.sw_if_index)
+
+        self.nat44_add_address(self.nat_addr)
+        self.nat44_add_static_mapping(self.pg7.remote_ip4, self.nat_addr,
+                                      self.tcp_port_in, self.tcp_port_out,
+                                      proto=IP_PROTOS.tcp)
+        self.nat44_add_static_mapping(self.pg7.remote_ip4, self.nat_addr,
+                                      self.udp_port_in, self.udp_port_out,
+                                      proto=IP_PROTOS.udp)
+        self.nat44_add_static_mapping(self.pg7.remote_ip4, self.nat_addr,
+                                      self.icmp_id_in, self.icmp_id_out,
+                                      proto=IP_PROTOS.icmp)
+        self.vapi.nat44_interface_add_del_feature(self.pg7.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg8.sw_if_index,
+                                                  is_inside=0)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg8)
+        self.pg8.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg7.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg7)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg7, self.pg8)
+        self.pg7.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg8.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+    def test_static_unknown_proto(self):
+        """ 1:1 NAT translate packet with unknown protocol """
+        nat_ip = "10.0.0.10"
+        self.nat44_add_static_mapping(self.pg0.remote_ip4, nat_ip)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, nat_ip)
+            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # out2in
+        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+             IP(src=self.pg1.remote_ip4, dst=nat_ip) /
+             GRE() /
+             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.pg1.remote_ip4)
+            self.assertEqual(packet[IP].dst, self.pg0.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def test_hairpinning_static_unknown_proto(self):
+        """ 1:1 NAT translate packet with unknown protocol - hairpinning """
+
+        host = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+
+        host_nat_ip = "10.0.0.10"
+        server_nat_ip = "10.0.0.11"
+
+        self.nat44_add_static_mapping(host.ip4, host_nat_ip)
+        self.nat44_add_static_mapping(server.ip4, server_nat_ip)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # host to server
+        p = (Ether(dst=self.pg0.local_mac, src=host.mac) /
+             IP(src=host.ip4, dst=server_nat_ip) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, host_nat_ip)
+            self.assertEqual(packet[IP].dst, server.ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # server to host
+        p = (Ether(dst=self.pg0.local_mac, src=server.mac) /
+             IP(src=server.ip4, dst=host_nat_ip) /
+             GRE() /
+             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, server_nat_ip)
+            self.assertEqual(packet[IP].dst, host.ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def test_unknown_proto(self):
+        """ NAT44 translate packet with unknown protocol """
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             TCP(sport=self.tcp_port_in, dport=20))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.nat_addr)
+            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # out2in
+        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+             IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+             GRE() /
+             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.pg1.remote_ip4)
+            self.assertEqual(packet[IP].dst, self.pg0.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def test_hairpinning_unknown_proto(self):
+        """ NAT44 translate packet with unknown protocol - hairpinning """
+        host = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        host_in_port = 1234
+        host_out_port = 0
+        server_in_port = 5678
+        server_out_port = 8765
+        server_nat_ip = "10.0.0.11"
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # add static mapping for server
+        self.nat44_add_static_mapping(server.ip4, server_nat_ip)
+
+        # host to server
+        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
+             IP(src=host.ip4, dst=server_nat_ip) /
+             TCP(sport=host_in_port, dport=server_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=host.mac) /
+             IP(src=host.ip4, dst=server_nat_ip) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.nat_addr)
+            self.assertEqual(packet[IP].dst, server.ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # server to host
+        p = (Ether(dst=self.pg0.local_mac, src=server.mac) /
+             IP(src=server.ip4, dst=self.nat_addr) /
+             GRE() /
+             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, server_nat_ip)
+            self.assertEqual(packet[IP].dst, host.ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def test_output_feature(self):
+        """ NAT44 interface output feature (in2out postrouting) """
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg1.sw_if_index,
+                                                         is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+    def test_output_feature_vrf_aware(self):
+        """ NAT44 interface output feature VRF aware (in2out postrouting) """
+        nat_ip_vrf10 = "10.0.0.10"
+        nat_ip_vrf20 = "10.0.0.20"
+
+        self.vapi.ip_add_del_route(dst_address=self.pg3.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg3.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg3.sw_if_index,
+                                   table_id=10)
+        self.vapi.ip_add_del_route(dst_address=self.pg3.remote_ip4n,
+                                   dst_address_length=32,
+                                   next_hop_address=self.pg3.remote_ip4n,
+                                   next_hop_sw_if_index=self.pg3.sw_if_index,
+                                   table_id=20)
+
+        self.nat44_add_address(nat_ip_vrf10, vrf_id=10)
+        self.nat44_add_address(nat_ip_vrf20, vrf_id=20)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg4.sw_if_index)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg6.sw_if_index)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg3.sw_if_index,
+                                                         is_inside=0)
+
+        # in2out VRF 10
+        pkts = self.create_stream_in(self.pg4, self.pg3)
+        self.pg4.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=nat_ip_vrf10)
+
+        # out2in VRF 10
+        pkts = self.create_stream_out(self.pg3, dst_ip=nat_ip_vrf10)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg4.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg4)
+
+        # in2out VRF 20
+        pkts = self.create_stream_in(self.pg6, self.pg3)
+        self.pg6.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg3.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=nat_ip_vrf20)
+
+        # out2in VRF 20
+        pkts = self.create_stream_out(self.pg3, dst_ip=nat_ip_vrf20)
+        self.pg3.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg6)
+
+    def test_output_feature_hairpinning(self):
+        """ NAT44 interface output feature hairpinning (in2out postrouting) """
+        host = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        host_in_port = 1234
+        host_out_port = 0
+        server_in_port = 5678
+        server_out_port = 8765
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg1.sw_if_index,
+                                                         is_inside=0)
+
+        # add static mapping for server
+        self.nat44_add_static_mapping(server.ip4, self.nat_addr,
+                                      server_in_port, server_out_port,
+                                      proto=IP_PROTOS.tcp)
+
+        # send packet from host to server
+        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
+             IP(src=host.ip4, dst=self.nat_addr) /
+             TCP(sport=host_in_port, dport=server_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.nat_addr)
+            self.assertEqual(ip.dst, server.ip4)
+            self.assertNotEqual(tcp.sport, host_in_port)
+            self.assertEqual(tcp.dport, server_in_port)
+            self.check_tcp_checksum(p)
+            host_out_port = tcp.sport
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # send reply from server to host
+        p = (Ether(src=server.mac, dst=self.pg0.local_mac) /
+             IP(src=server.ip4, dst=self.nat_addr) /
+             TCP(sport=server_in_port, dport=host_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.nat_addr)
+            self.assertEqual(ip.dst, host.ip4)
+            self.assertEqual(tcp.sport, server_out_port)
+            self.assertEqual(tcp.dport, host_in_port)
+            self.check_tcp_checksum(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:"), p)
+            raise
+
+    def tearDown(self):
+        super(TestNAT44, self).tearDown()
+        if not self.vpp_dead:
+            self.logger.info(self.vapi.cli("show nat44 verbose"))
+            self.clear_nat44()
+
+
+class TestDeterministicNAT(MethodHolder):
+    """ Deterministic NAT Test Cases """
+
+    @classmethod
+    def setUpConstants(cls):
+        super(TestDeterministicNAT, cls).setUpConstants()
+        cls.vpp_cmdline.extend(["nat", "{", "deterministic", "}"])
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestDeterministicNAT, cls).setUpClass()
+
+        try:
+            cls.tcp_port_in = 6303
+            cls.tcp_external_port = 6303
+            cls.udp_port_in = 6304
+            cls.udp_external_port = 6304
+            cls.icmp_id_in = 6305
+            cls.nat_addr = '10.0.0.3'
+
+            cls.create_pg_interfaces(range(3))
+            cls.interfaces = list(cls.pg_interfaces)
+
+            for i in cls.interfaces:
+                i.admin_up()
+                i.config_ip4()
+                i.resolve_arp()
+
+            cls.pg0.generate_remote_hosts(2)
+            cls.pg0.configure_ipv4_neighbors()
+
+        except Exception:
+            super(TestDeterministicNAT, cls).tearDownClass()
+            raise
+
+    def create_stream_in(self, in_if, out_if, ttl=64):
+        """
+        Create packet stream for inside network
+
+        :param in_if: Inside interface
+        :param out_if: Outside interface
+        :param ttl: TTL of generated packets
+        """
+        pkts = []
+        # TCP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
+             TCP(sport=self.tcp_port_in, dport=self.tcp_external_port))
+        pkts.append(p)
+
+        # UDP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
+             UDP(sport=self.udp_port_in, dport=self.udp_external_port))
+        pkts.append(p)
+
+        # ICMP
+        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
+             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
+             ICMP(id=self.icmp_id_in, type='echo-request'))
+        pkts.append(p)
+
+        return pkts
+
+    def create_stream_out(self, out_if, dst_ip=None, ttl=64):
+        """
+        Create packet stream for outside network
+
+        :param out_if: Outside interface
+        :param dst_ip: Destination IP address (Default use global NAT address)
+        :param ttl: TTL of generated packets
+        """
+        if dst_ip is None:
+            dst_ip = self.nat_addr
+        pkts = []
+        # TCP
+        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
+             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
+             TCP(dport=self.tcp_port_out, sport=self.tcp_external_port))
+        pkts.append(p)
+
+        # UDP
+        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
+             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
+             UDP(dport=self.udp_port_out, sport=self.udp_external_port))
+        pkts.append(p)
+
+        # ICMP
+        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
+             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
+             ICMP(id=self.icmp_external_id, type='echo-reply'))
+        pkts.append(p)
+
+        return pkts
+
+    def verify_capture_out(self, capture, nat_ip=None, packet_num=3):
+        """
+        Verify captured packets on outside network
+
+        :param capture: Captured packets
+        :param nat_ip: Translated IP address (Default use global NAT address)
+        :param same_port: Sorce port number is not translated (Default False)
+        :param packet_num: Expected number of packets (Default 3)
+        """
+        if nat_ip is None:
+            nat_ip = self.nat_addr
+        self.assertEqual(packet_num, len(capture))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, nat_ip)
+                if packet.haslayer(TCP):
+                    self.tcp_port_out = packet[TCP].sport
+                elif packet.haslayer(UDP):
+                    self.udp_port_out = packet[UDP].sport
+                else:
+                    self.icmp_external_id = packet[ICMP].id
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet "
+                                      "(outside network):", packet))
+                raise
+
+    def initiate_tcp_session(self, in_if, out_if):
+        """
+        Initiates TCP session
+
+        :param in_if: Inside interface
+        :param out_if: Outside interface
+        """
+        try:
+            # SYN packet in->out
+            p = (Ether(src=in_if.remote_mac, dst=in_if.local_mac) /
+                 IP(src=in_if.remote_ip4, dst=out_if.remote_ip4) /
+                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
+                     flags="S"))
+            in_if.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = out_if.get_capture(1)
+            p = capture[0]
+            self.tcp_port_out = p[TCP].sport
+
+            # SYN + ACK packet out->in
+            p = (Ether(src=out_if.remote_mac, dst=out_if.local_mac) /
+                 IP(src=out_if.remote_ip4, dst=self.nat_addr) /
+                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
+                     flags="SA"))
+            out_if.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            in_if.get_capture(1)
+
+            # ACK packet in->out
+            p = (Ether(src=in_if.remote_mac, dst=in_if.local_mac) /
+                 IP(src=in_if.remote_ip4, dst=out_if.remote_ip4) /
+                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
+                     flags="A"))
+            in_if.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            out_if.get_capture(1)
+
+        except:
+            self.logger.error("TCP 3 way handshake failed")
+            raise
+
+    def verify_ipfix_max_entries_per_user(self, data):
+        """
+        Verify IPFIX maximum entries per user exceeded event
+
+        :param data: Decoded IPFIX data records
+        """
+        self.assertEqual(1, len(data))
+        record = data[0]
+        # natEvent
+        self.assertEqual(ord(record[230]), 13)
+        # natQuotaExceededEvent
+        self.assertEqual('\x03\x00\x00\x00', record[466])
+        # sourceIPv4Address
+        self.assertEqual(self.pg0.remote_ip4n, record[8])
+
+    def test_deterministic_mode(self):
+        """ NAT plugin run deterministic mode """
+        in_addr = '172.16.255.0'
+        out_addr = '172.17.255.50'
+        in_addr_t = '172.16.255.20'
+        in_addr_n = socket.inet_aton(in_addr)
+        out_addr_n = socket.inet_aton(out_addr)
+        in_addr_t_n = socket.inet_aton(in_addr_t)
+        in_plen = 24
+        out_plen = 32
+
+        nat_config = self.vapi.nat_show_config()
+        self.assertEqual(1, nat_config.deterministic)
+
+        self.vapi.nat_det_add_del_map(in_addr_n, in_plen, out_addr_n, out_plen)
+
+        rep1 = self.vapi.nat_det_forward(in_addr_t_n)
+        self.assertEqual(rep1.out_addr[:4], out_addr_n)
+        rep2 = self.vapi.nat_det_reverse(out_addr_n, rep1.out_port_hi)
+        self.assertEqual(rep2.in_addr[:4], in_addr_t_n)
+
+        deterministic_mappings = self.vapi.nat_det_map_dump()
+        self.assertEqual(len(deterministic_mappings), 1)
+        dsm = deterministic_mappings[0]
+        self.assertEqual(in_addr_n, dsm.in_addr[:4])
+        self.assertEqual(in_plen, dsm.in_plen)
+        self.assertEqual(out_addr_n, dsm.out_addr[:4])
+        self.assertEqual(out_plen, dsm.out_plen)
+
+        self.clear_nat_det()
+        deterministic_mappings = self.vapi.nat_det_map_dump()
+        self.assertEqual(len(deterministic_mappings), 0)
+
+    def test_set_timeouts(self):
+        """ Set deterministic NAT timeouts """
+        timeouts_before = self.vapi.nat_det_get_timeouts()
+
+        self.vapi.nat_det_set_timeouts(timeouts_before.udp + 10,
+                                       timeouts_before.tcp_established + 10,
+                                       timeouts_before.tcp_transitory + 10,
+                                       timeouts_before.icmp + 10)
+
+        timeouts_after = self.vapi.nat_det_get_timeouts()
+
+        self.assertNotEqual(timeouts_before.udp, timeouts_after.udp)
+        self.assertNotEqual(timeouts_before.icmp, timeouts_after.icmp)
+        self.assertNotEqual(timeouts_before.tcp_established,
+                            timeouts_after.tcp_established)
+        self.assertNotEqual(timeouts_before.tcp_transitory,
+                            timeouts_after.tcp_transitory)
+
+    def test_det_in(self):
+        """ Deterministic NAT translation test (TCP, UDP, ICMP) """
+
+        nat_ip = "10.0.0.10"
+
+        self.vapi.nat_det_add_del_map(self.pg0.remote_ip4n,
+                                      32,
+                                      socket.inet_aton(nat_ip),
+                                      32)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1, nat_ip)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in(capture, self.pg0)
+
+        # session dump test
+        sessions = self.vapi.nat_det_session_dump(self.pg0.remote_ip4n)
+        self.assertEqual(len(sessions), 3)
+
+        # TCP session
+        s = sessions[0]
+        self.assertEqual(s.ext_addr[:4], self.pg1.remote_ip4n)
+        self.assertEqual(s.in_port, self.tcp_port_in)
+        self.assertEqual(s.out_port, self.tcp_port_out)
+        self.assertEqual(s.ext_port, self.tcp_external_port)
+
+        # UDP session
+        s = sessions[1]
+        self.assertEqual(s.ext_addr[:4], self.pg1.remote_ip4n)
+        self.assertEqual(s.in_port, self.udp_port_in)
+        self.assertEqual(s.out_port, self.udp_port_out)
+        self.assertEqual(s.ext_port, self.udp_external_port)
+
+        # ICMP session
+        s = sessions[2]
+        self.assertEqual(s.ext_addr[:4], self.pg1.remote_ip4n)
+        self.assertEqual(s.in_port, self.icmp_id_in)
+        self.assertEqual(s.out_port, self.icmp_external_id)
+
+    def test_multiple_users(self):
+        """ Deterministic NAT multiple users """
+
+        nat_ip = "10.0.0.10"
+        port_in = 80
+        external_port = 6303
+
+        host0 = self.pg0.remote_hosts[0]
+        host1 = self.pg0.remote_hosts[1]
+
+        self.vapi.nat_det_add_del_map(host0.ip4n,
+                                      24,
+                                      socket.inet_aton(nat_ip),
+                                      32)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        # host0 to out
+        p = (Ether(src=host0.mac, dst=self.pg0.local_mac) /
+             IP(src=host0.ip4, dst=self.pg1.remote_ip4) /
+             TCP(sport=port_in, dport=external_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, nat_ip)
+            self.assertEqual(ip.dst, self.pg1.remote_ip4)
+            self.assertEqual(tcp.dport, external_port)
+            port_out0 = tcp.sport
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # host1 to out
+        p = (Ether(src=host1.mac, dst=self.pg0.local_mac) /
+             IP(src=host1.ip4, dst=self.pg1.remote_ip4) /
+             TCP(sport=port_in, dport=external_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, nat_ip)
+            self.assertEqual(ip.dst, self.pg1.remote_ip4)
+            self.assertEqual(tcp.dport, external_port)
+            port_out1 = tcp.sport
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        dms = self.vapi.nat_det_map_dump()
+        self.assertEqual(1, len(dms))
+        self.assertEqual(2, dms[0].ses_num)
+
+        # out to host0
+        p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+             IP(src=self.pg1.remote_ip4, dst=nat_ip) /
+             TCP(sport=external_port, dport=port_out0))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg1.remote_ip4)
+            self.assertEqual(ip.dst, host0.ip4)
+            self.assertEqual(tcp.dport, port_in)
+            self.assertEqual(tcp.sport, external_port)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # out to host1
+        p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+             IP(src=self.pg1.remote_ip4, dst=nat_ip) /
+             TCP(sport=external_port, dport=port_out1))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg1.remote_ip4)
+            self.assertEqual(ip.dst, host1.ip4)
+            self.assertEqual(tcp.dport, port_in)
+            self.assertEqual(tcp.sport, external_port)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet", p))
+            raise
+
+        # session close api test
+        self.vapi.nat_det_close_session_out(socket.inet_aton(nat_ip),
+                                            port_out1,
+                                            self.pg1.remote_ip4n,
+                                            external_port)
+        dms = self.vapi.nat_det_map_dump()
+        self.assertEqual(dms[0].ses_num, 1)
+
+        self.vapi.nat_det_close_session_in(host0.ip4n,
+                                           port_in,
+                                           self.pg1.remote_ip4n,
+                                           external_port)
+        dms = self.vapi.nat_det_map_dump()
+        self.assertEqual(dms[0].ses_num, 0)
+
+    def test_tcp_session_close_detection_in(self):
+        """ Deterministic NAT TCP session close from inside network """
+        self.vapi.nat_det_add_del_map(self.pg0.remote_ip4n,
+                                      32,
+                                      socket.inet_aton(self.nat_addr),
+                                      32)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        self.initiate_tcp_session(self.pg0, self.pg1)
+
+        # close the session from inside
+        try:
+            # FIN packet in -> out
+            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
+                     flags="F"))
+            self.pg0.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            self.pg1.get_capture(1)
+
+            pkts = []
+
+            # ACK packet out -> in
+            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                 IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
+                     flags="A"))
+            pkts.append(p)
+
+            # FIN packet out -> in
+            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                 IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
+                     flags="F"))
+            pkts.append(p)
+
+            self.pg1.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            self.pg0.get_capture(2)
+
+            # ACK packet in -> out
+            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
+                     flags="A"))
+            self.pg0.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            self.pg1.get_capture(1)
+
+            # Check if deterministic NAT44 closed the session
+            dms = self.vapi.nat_det_map_dump()
+            self.assertEqual(0, dms[0].ses_num)
+        except:
+            self.logger.error("TCP session termination failed")
+            raise
+
+    def test_tcp_session_close_detection_out(self):
+        """ Deterministic NAT TCP session close from outside network """
+        self.vapi.nat_det_add_del_map(self.pg0.remote_ip4n,
+                                      32,
+                                      socket.inet_aton(self.nat_addr),
+                                      32)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        self.initiate_tcp_session(self.pg0, self.pg1)
+
+        # close the session from outside
+        try:
+            # FIN packet out -> in
+            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                 IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
+                     flags="F"))
+            self.pg1.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            self.pg0.get_capture(1)
+
+            pkts = []
+
+            # ACK packet in -> out
+            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
+                     flags="A"))
+            pkts.append(p)
+
+            # ACK packet in -> out
+            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
+                     flags="F"))
+            pkts.append(p)
+
+            self.pg0.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            self.pg1.get_capture(2)
+
+            # ACK packet out -> in
+            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                 IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
+                     flags="A"))
+            self.pg1.add_stream(p)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            self.pg0.get_capture(1)
+
+            # Check if deterministic NAT44 closed the session
+            dms = self.vapi.nat_det_map_dump()
+            self.assertEqual(0, dms[0].ses_num)
+        except:
+            self.logger.error("TCP session termination failed")
+            raise
+
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
+    def test_session_timeout(self):
+        """ Deterministic NAT session timeouts """
+        self.vapi.nat_det_add_del_map(self.pg0.remote_ip4n,
+                                      32,
+                                      socket.inet_aton(self.nat_addr),
+                                      32)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+
+        self.initiate_tcp_session(self.pg0, self.pg1)
+        self.vapi.nat_det_set_timeouts(5, 5, 5, 5)
+        pkts = self.create_stream_in(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        sleep(15)
+
+        dms = self.vapi.nat_det_map_dump()
+        self.assertEqual(0, dms[0].ses_num)
+
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
+    def test_session_limit_per_user(self):
+        """ Deterministic NAT maximum sessions per user limit """
+        self.vapi.nat_det_add_del_map(self.pg0.remote_ip4n,
+                                      32,
+                                      socket.inet_aton(self.nat_addr),
+                                      32)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.set_ipfix_exporter(collector_address=self.pg2.remote_ip4n,
+                                     src_address=self.pg2.local_ip4n,
+                                     path_mtu=512,
+                                     template_interval=10)
+        self.vapi.nat_ipfix()
+
+        pkts = []
+        for port in range(1025, 2025):
+            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+                 UDP(sport=port, dport=port))
+            pkts.append(p)
+
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             UDP(sport=3001, dport=3002))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.assert_nothing_captured()
+
+        # verify ICMP error packet
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        self.assertTrue(p.haslayer(ICMP))
+        icmp = p[ICMP]
+        self.assertEqual(icmp.type, 3)
+        self.assertEqual(icmp.code, 1)
+        self.assertTrue(icmp.haslayer(IPerror))
+        inner_ip = icmp[IPerror]
+        self.assertEqual(inner_ip[UDPerror].sport, 3001)
+        self.assertEqual(inner_ip[UDPerror].dport, 3002)
+
+        dms = self.vapi.nat_det_map_dump()
+
+        self.assertEqual(1000, dms[0].ses_num)
+
+        # verify IPFIX logging
+        self.vapi.cli("ipfix flush")  # FIXME this should be an API call
+        sleep(1)
+        capture = self.pg2.get_capture(2)
+        ipfix = IPFIXDecoder()
+        # first load template
+        for p in capture:
+            self.assertTrue(p.haslayer(IPFIX))
+            if p.haslayer(Template):
+                ipfix.add_template(p.getlayer(Template))
+        # verify events in data set
+        for p in capture:
+            if p.haslayer(Data):
+                data = ipfix.decode_data_set(p.getlayer(Set))
+                self.verify_ipfix_max_entries_per_user(data)
+
+    def clear_nat_det(self):
+        """
+        Clear deterministic NAT configuration.
+        """
+        self.vapi.nat_ipfix(enable=0)
+        self.vapi.nat_det_set_timeouts()
+        deterministic_mappings = self.vapi.nat_det_map_dump()
+        for dsm in deterministic_mappings:
+            self.vapi.nat_det_add_del_map(dsm.in_addr,
+                                          dsm.in_plen,
+                                          dsm.out_addr,
+                                          dsm.out_plen,
+                                          is_add=0)
+
+        interfaces = self.vapi.nat44_interface_dump()
+        for intf in interfaces:
+            self.vapi.nat44_interface_add_del_feature(intf.sw_if_index,
+                                                      intf.is_inside,
+                                                      is_add=0)
+
+    def tearDown(self):
+        super(TestDeterministicNAT, self).tearDown()
+        if not self.vpp_dead:
+            self.logger.info(self.vapi.cli("show nat44 detail"))
+            self.clear_nat_det()
+
+
+class TestNAT64(MethodHolder):
+    """ NAT64 Test Cases """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestNAT64, cls).setUpClass()
+
+        try:
+            cls.tcp_port_in = 6303
+            cls.tcp_port_out = 6303
+            cls.udp_port_in = 6304
+            cls.udp_port_out = 6304
+            cls.icmp_id_in = 6305
+            cls.icmp_id_out = 6305
+            cls.nat_addr = '10.0.0.3'
+            cls.nat_addr_n = socket.inet_pton(socket.AF_INET, cls.nat_addr)
+            cls.vrf1_id = 10
+            cls.vrf1_nat_addr = '10.0.10.3'
+            cls.vrf1_nat_addr_n = socket.inet_pton(socket.AF_INET,
+                                                   cls.vrf1_nat_addr)
+
+            cls.create_pg_interfaces(range(3))
+            cls.ip6_interfaces = list(cls.pg_interfaces[0:1])
+            cls.ip6_interfaces.append(cls.pg_interfaces[2])
+            cls.ip4_interfaces = list(cls.pg_interfaces[1:2])
+
+            cls.pg_interfaces[2].set_table_ip6(cls.vrf1_id)
+
+            cls.pg0.generate_remote_hosts(2)
+
+            for i in cls.ip6_interfaces:
+                i.admin_up()
+                i.config_ip6()
+                i.configure_ipv6_neighbors()
+
+            for i in cls.ip4_interfaces:
+                i.admin_up()
+                i.config_ip4()
+                i.resolve_arp()
+
+        except Exception:
+            super(TestNAT64, cls).tearDownClass()
+            raise
+
+    def test_pool(self):
+        """ Add/delete address to NAT64 pool """
+        nat_addr = socket.inet_pton(socket.AF_INET, '1.2.3.4')
+
+        self.vapi.nat64_add_del_pool_addr_range(nat_addr, nat_addr)
+
+        addresses = self.vapi.nat64_pool_addr_dump()
+        self.assertEqual(len(addresses), 1)
+        self.assertEqual(addresses[0].address, nat_addr)
+
+        self.vapi.nat64_add_del_pool_addr_range(nat_addr, nat_addr, is_add=0)
+
+        addresses = self.vapi.nat64_pool_addr_dump()
+        self.assertEqual(len(addresses), 0)
+
+    def test_interface(self):
+        """ Enable/disable NAT64 feature on the interface """
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        interfaces = self.vapi.nat64_interface_dump()
+        self.assertEqual(len(interfaces), 2)
+        pg0_found = False
+        pg1_found = False
+        for intf in interfaces:
+            if intf.sw_if_index == self.pg0.sw_if_index:
+                self.assertEqual(intf.is_inside, 1)
+                pg0_found = True
+            elif intf.sw_if_index == self.pg1.sw_if_index:
+                self.assertEqual(intf.is_inside, 0)
+                pg1_found = True
+        self.assertTrue(pg0_found)
+        self.assertTrue(pg1_found)
+
+        features = self.vapi.cli("show interface features pg0")
+        self.assertNotEqual(features.find('nat64-in2out'), -1)
+        features = self.vapi.cli("show interface features pg1")
+        self.assertNotEqual(features.find('nat64-out2in'), -1)
+
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index, is_add=0)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_add=0)
+
+        interfaces = self.vapi.nat64_interface_dump()
+        self.assertEqual(len(interfaces), 0)
+
+    def test_static_bib(self):
+        """ Add/delete static BIB entry """
+        in_addr = socket.inet_pton(socket.AF_INET6,
+                                   '2001:db8:85a3::8a2e:370:7334')
+        out_addr = socket.inet_pton(socket.AF_INET, '10.1.1.3')
+        in_port = 1234
+        out_port = 5678
+        proto = IP_PROTOS.tcp
+
+        self.vapi.nat64_add_del_static_bib(in_addr,
+                                           out_addr,
+                                           in_port,
+                                           out_port,
+                                           proto)
+        bib = self.vapi.nat64_bib_dump(IP_PROTOS.tcp)
+        static_bib_num = 0
+        for bibe in bib:
+            if bibe.is_static:
+                static_bib_num += 1
+                self.assertEqual(bibe.i_addr, in_addr)
+                self.assertEqual(bibe.o_addr, out_addr)
+                self.assertEqual(bibe.i_port, in_port)
+                self.assertEqual(bibe.o_port, out_port)
+        self.assertEqual(static_bib_num, 1)
+
+        self.vapi.nat64_add_del_static_bib(in_addr,
+                                           out_addr,
+                                           in_port,
+                                           out_port,
+                                           proto,
+                                           is_add=0)
+        bib = self.vapi.nat64_bib_dump(IP_PROTOS.tcp)
+        static_bib_num = 0
+        for bibe in bib:
+            if bibe.is_static:
+                static_bib_num += 1
+        self.assertEqual(static_bib_num, 0)
+
+    def test_set_timeouts(self):
+        """ Set NAT64 timeouts """
+        # verify default values
+        timeouts = self.vapi.nat64_get_timeouts()
+        self.assertEqual(timeouts.udp, 300)
+        self.assertEqual(timeouts.icmp, 60)
+        self.assertEqual(timeouts.tcp_trans, 240)
+        self.assertEqual(timeouts.tcp_est, 7440)
+        self.assertEqual(timeouts.tcp_incoming_syn, 6)
+
+        # set and verify custom values
+        self.vapi.nat64_set_timeouts(udp=200, icmp=30, tcp_trans=250,
+                                     tcp_est=7450, tcp_incoming_syn=10)
+        timeouts = self.vapi.nat64_get_timeouts()
+        self.assertEqual(timeouts.udp, 200)
+        self.assertEqual(timeouts.icmp, 30)
+        self.assertEqual(timeouts.tcp_trans, 250)
+        self.assertEqual(timeouts.tcp_est, 7450)
+        self.assertEqual(timeouts.tcp_incoming_syn, 10)
+
+    def test_dynamic(self):
+        """ NAT64 dynamic translation test """
+        self.tcp_port_in = 6303
+        self.udp_port_in = 6304
+        self.icmp_id_in = 6305
+
+        ses_num_start = self.nat64_get_ses_num()
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        # in2out
+        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=self.nat_addr,
+                                dst_ip=self.pg1.remote_ip4)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        ip = IPv6(src=''.join(['64:ff9b::', self.pg1.remote_ip4]))
+        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg0.remote_ip6)
+
+        # in2out
+        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=self.nat_addr,
+                                dst_ip=self.pg1.remote_ip4)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg0.remote_ip6)
+
+        ses_num_end = self.nat64_get_ses_num()
+
+        self.assertEqual(ses_num_end - ses_num_start, 3)
+
+        # tenant with specific VRF
+        self.vapi.nat64_add_del_pool_addr_range(self.vrf1_nat_addr_n,
+                                                self.vrf1_nat_addr_n,
+                                                vrf_id=self.vrf1_id)
+        self.vapi.nat64_add_del_interface(self.pg2.sw_if_index)
+
+        pkts = self.create_stream_in_ip6(self.pg2, self.pg1)
+        self.pg2.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=self.vrf1_nat_addr,
+                                dst_ip=self.pg1.remote_ip4)
+
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.vrf1_nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg2.remote_ip6)
+
+    def test_static(self):
+        """ NAT64 static translation test """
+        self.tcp_port_in = 60303
+        self.udp_port_in = 60304
+        self.icmp_id_in = 60305
+        self.tcp_port_out = 60303
+        self.udp_port_out = 60304
+        self.icmp_id_out = 60305
+
+        ses_num_start = self.nat64_get_ses_num()
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        self.vapi.nat64_add_del_static_bib(self.pg0.remote_ip6n,
+                                           self.nat_addr_n,
+                                           self.tcp_port_in,
+                                           self.tcp_port_out,
+                                           IP_PROTOS.tcp)
+        self.vapi.nat64_add_del_static_bib(self.pg0.remote_ip6n,
+                                           self.nat_addr_n,
+                                           self.udp_port_in,
+                                           self.udp_port_out,
+                                           IP_PROTOS.udp)
+        self.vapi.nat64_add_del_static_bib(self.pg0.remote_ip6n,
+                                           self.nat_addr_n,
+                                           self.icmp_id_in,
+                                           self.icmp_id_out,
+                                           IP_PROTOS.icmp)
+
+        # in2out
+        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=self.nat_addr,
+                                dst_ip=self.pg1.remote_ip4, same_port=True)
+
+        # out2in
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        ip = IPv6(src=''.join(['64:ff9b::', self.pg1.remote_ip4]))
+        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg0.remote_ip6)
+
+        ses_num_end = self.nat64_get_ses_num()
+
+        self.assertEqual(ses_num_end - ses_num_start, 3)
+
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
+    def test_session_timeout(self):
+        """ NAT64 session timeout """
+        self.icmp_id_in = 1234
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+        self.vapi.nat64_set_timeouts(icmp=5)
+
+        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+
+        ses_num_before_timeout = self.nat64_get_ses_num()
+
+        sleep(15)
+
+        # ICMP session after timeout
+        ses_num_after_timeout = self.nat64_get_ses_num()
+        self.assertNotEqual(ses_num_before_timeout, ses_num_after_timeout)
+
+    def test_icmp_error(self):
+        """ NAT64 ICMP Error message translation """
+        self.tcp_port_in = 6303
+        self.udp_port_in = 6304
+        self.icmp_id_in = 6305
+
+        ses_num_start = self.nat64_get_ses_num()
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        # send some packets to create sessions
+        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture_ip4 = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture_ip4,
+                                nat_ip=self.nat_addr,
+                                dst_ip=self.pg1.remote_ip4)
+
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture_ip6 = self.pg0.get_capture(len(pkts))
+        ip = IPv6(src=''.join(['64:ff9b::', self.pg1.remote_ip4]))
+        self.verify_capture_in_ip6(capture_ip6, ip[IPv6].src,
+                                   self.pg0.remote_ip6)
+
+        # in2out
+        pkts = [Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+                IPv6(src=self.pg0.remote_ip6, dst=ip[IPv6].src) /
+                ICMPv6DestUnreach(code=1) /
+                packet[IPv6] for packet in capture_ip6]
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IP].src, self.nat_addr)
+                self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+                self.assertEqual(packet[ICMP].type, 3)
+                self.assertEqual(packet[ICMP].code, 13)
+                inner = packet[IPerror]
+                self.assertEqual(inner.src, self.pg1.remote_ip4)
+                self.assertEqual(inner.dst, self.nat_addr)
+                self.check_icmp_checksum(packet)
+                if inner.haslayer(TCPerror):
+                    self.assertEqual(inner[TCPerror].dport, self.tcp_port_out)
+                elif inner.haslayer(UDPerror):
+                    self.assertEqual(inner[UDPerror].dport, self.udp_port_out)
+                else:
+                    self.assertEqual(inner[ICMPerror].id, self.icmp_id_out)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+        # out2in
+        pkts = [Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+                IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+                ICMP(type=3, code=13) /
+                packet[IP] for packet in capture_ip4]
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IPv6].src, ip.src)
+                self.assertEqual(packet[IPv6].dst, self.pg0.remote_ip6)
+                icmp = packet[ICMPv6DestUnreach]
+                self.assertEqual(icmp.code, 1)
+                inner = icmp[IPerror6]
+                self.assertEqual(inner.src, self.pg0.remote_ip6)
+                self.assertEqual(inner.dst, ip.src)
+                self.check_icmpv6_checksum(packet)
+                if inner.haslayer(TCPerror):
+                    self.assertEqual(inner[TCPerror].sport, self.tcp_port_in)
+                elif inner.haslayer(UDPerror):
+                    self.assertEqual(inner[UDPerror].sport, self.udp_port_in)
+                else:
+                    self.assertEqual(inner[ICMPv6EchoRequest].id,
+                                     self.icmp_id_in)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+    def test_hairpinning(self):
+        """ NAT64 hairpinning """
+
+        client = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        server_tcp_in_port = 22
+        server_tcp_out_port = 4022
+        server_udp_in_port = 23
+        server_udp_out_port = 4023
+        client_tcp_in_port = 1234
+        client_udp_in_port = 1235
+        client_tcp_out_port = 0
+        client_udp_out_port = 0
+        ip = IPv6(src=''.join(['64:ff9b::', self.nat_addr]))
+        nat_addr_ip6 = ip.src
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        self.vapi.nat64_add_del_static_bib(server.ip6n,
+                                           self.nat_addr_n,
+                                           server_tcp_in_port,
+                                           server_tcp_out_port,
+                                           IP_PROTOS.tcp)
+        self.vapi.nat64_add_del_static_bib(server.ip6n,
+                                           self.nat_addr_n,
+                                           server_udp_in_port,
+                                           server_udp_out_port,
+                                           IP_PROTOS.udp)
+
+        # client to server
+        pkts = []
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=client.ip6, dst=nat_addr_ip6) /
+             TCP(sport=client_tcp_in_port, dport=server_tcp_out_port))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=client.ip6, dst=nat_addr_ip6) /
+             UDP(sport=client_udp_in_port, dport=server_udp_out_port))
+        pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IPv6].src, nat_addr_ip6)
+                self.assertEqual(packet[IPv6].dst, server.ip6)
+                if packet.haslayer(TCP):
+                    self.assertNotEqual(packet[TCP].sport, client_tcp_in_port)
+                    self.assertEqual(packet[TCP].dport, server_tcp_in_port)
+                    self.check_tcp_checksum(packet)
+                    client_tcp_out_port = packet[TCP].sport
+                else:
+                    self.assertNotEqual(packet[UDP].sport, client_udp_in_port)
+                    self.assertEqual(packet[UDP].dport, server_udp_in_port)
+                    self.check_udp_checksum(packet)
+                    client_udp_out_port = packet[UDP].sport
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+        # server to client
+        pkts = []
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=server.ip6, dst=nat_addr_ip6) /
+             TCP(sport=server_tcp_in_port, dport=client_tcp_out_port))
+        pkts.append(p)
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=server.ip6, dst=nat_addr_ip6) /
+             UDP(sport=server_udp_in_port, dport=client_udp_out_port))
+        pkts.append(p)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IPv6].src, nat_addr_ip6)
+                self.assertEqual(packet[IPv6].dst, client.ip6)
+                if packet.haslayer(TCP):
+                    self.assertEqual(packet[TCP].sport, server_tcp_out_port)
+                    self.assertEqual(packet[TCP].dport, client_tcp_in_port)
+                    self.check_tcp_checksum(packet)
+                else:
+                    self.assertEqual(packet[UDP].sport, server_udp_out_port)
+                    self.assertEqual(packet[UDP].dport, client_udp_in_port)
+                    self.check_udp_checksum(packet)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+        # ICMP error
+        pkts = []
+        pkts = [Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+                IPv6(src=client.ip6, dst=nat_addr_ip6) /
+                ICMPv6DestUnreach(code=1) /
+                packet[IPv6] for packet in capture]
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for packet in capture:
+            try:
+                self.assertEqual(packet[IPv6].src, nat_addr_ip6)
+                self.assertEqual(packet[IPv6].dst, server.ip6)
+                icmp = packet[ICMPv6DestUnreach]
+                self.assertEqual(icmp.code, 1)
+                inner = icmp[IPerror6]
+                self.assertEqual(inner.src, server.ip6)
+                self.assertEqual(inner.dst, nat_addr_ip6)
+                self.check_icmpv6_checksum(packet)
+                if inner.haslayer(TCPerror):
+                    self.assertEqual(inner[TCPerror].sport, server_tcp_in_port)
+                    self.assertEqual(inner[TCPerror].dport,
+                                     client_tcp_out_port)
+                else:
+                    self.assertEqual(inner[UDPerror].sport, server_udp_in_port)
+                    self.assertEqual(inner[UDPerror].dport,
+                                     client_udp_out_port)
+            except:
+                self.logger.error(ppp("Unexpected or invalid packet:", packet))
+                raise
+
+    def test_prefix(self):
+        """ NAT64 Network-Specific Prefix """
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+        self.vapi.nat64_add_del_pool_addr_range(self.vrf1_nat_addr_n,
+                                                self.vrf1_nat_addr_n,
+                                                vrf_id=self.vrf1_id)
+        self.vapi.nat64_add_del_interface(self.pg2.sw_if_index)
+
+        # Add global prefix
+        global_pref64 = "2001:db8::"
+        global_pref64_n = socket.inet_pton(socket.AF_INET6, global_pref64)
+        global_pref64_len = 32
+        self.vapi.nat64_add_del_prefix(global_pref64_n, global_pref64_len)
+
+        prefix = self.vapi.nat64_prefix_dump()
+        self.assertEqual(len(prefix), 1)
+        self.assertEqual(prefix[0].prefix, global_pref64_n)
+        self.assertEqual(prefix[0].prefix_len, global_pref64_len)
+        self.assertEqual(prefix[0].vrf_id, 0)
+
+        # Add tenant specific prefix
+        vrf1_pref64 = "2001:db8:122:300::"
+        vrf1_pref64_n = socket.inet_pton(socket.AF_INET6, vrf1_pref64)
+        vrf1_pref64_len = 56
+        self.vapi.nat64_add_del_prefix(vrf1_pref64_n,
+                                       vrf1_pref64_len,
+                                       vrf_id=self.vrf1_id)
+        prefix = self.vapi.nat64_prefix_dump()
+        self.assertEqual(len(prefix), 2)
+
+        # Global prefix
+        pkts = self.create_stream_in_ip6(self.pg0,
+                                         self.pg1,
+                                         pref=global_pref64,
+                                         plen=global_pref64_len)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=self.nat_addr,
+                                dst_ip=self.pg1.remote_ip4)
+
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        dst_ip = self.compose_ip6(self.pg1.remote_ip4,
+                                  global_pref64,
+                                  global_pref64_len)
+        self.verify_capture_in_ip6(capture, dst_ip, self.pg0.remote_ip6)
+
+        # Tenant specific prefix
+        pkts = self.create_stream_in_ip6(self.pg2,
+                                         self.pg1,
+                                         pref=vrf1_pref64,
+                                         plen=vrf1_pref64_len)
+        self.pg2.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(len(pkts))
+        self.verify_capture_out(capture, nat_ip=self.vrf1_nat_addr,
+                                dst_ip=self.pg1.remote_ip4)
+
+        pkts = self.create_stream_out(self.pg1, dst_ip=self.vrf1_nat_addr)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg2.get_capture(len(pkts))
+        dst_ip = self.compose_ip6(self.pg1.remote_ip4,
+                                  vrf1_pref64,
+                                  vrf1_pref64_len)
+        self.verify_capture_in_ip6(capture, dst_ip, self.pg2.remote_ip6)
+
+    def test_unknown_proto(self):
+        """ NAT64 translate packet with unknown protocol """
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+        remote_ip6 = self.compose_ip6(self.pg1.remote_ip4, '64:ff9b::', 96)
+
+        # in2out
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=self.pg0.remote_ip6, dst=remote_ip6) /
+             TCP(sport=self.tcp_port_in, dport=20))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=self.pg0.remote_ip6, dst=remote_ip6, nh=47) /
+             GRE() /
+             IP(src=self.pg2.local_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.nat_addr)
+            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # out2in
+        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+             IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg2.local_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IPv6].src, remote_ip6)
+            self.assertEqual(packet[IPv6].dst, self.pg0.remote_ip6)
+            self.assertEqual(packet[IPv6].nh, 47)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def test_hairpinning_unknown_proto(self):
+        """ NAT64 translate packet with unknown protocol - hairpinning """
+
+        client = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        server_tcp_in_port = 22
+        server_tcp_out_port = 4022
+        client_tcp_in_port = 1234
+        client_tcp_out_port = 1235
+        server_nat_ip = "10.0.0.100"
+        client_nat_ip = "10.0.0.110"
+        server_nat_ip_n = socket.inet_pton(socket.AF_INET, server_nat_ip)
+        client_nat_ip_n = socket.inet_pton(socket.AF_INET, client_nat_ip)
+        server_nat_ip6 = self.compose_ip6(server_nat_ip, '64:ff9b::', 96)
+        client_nat_ip6 = self.compose_ip6(client_nat_ip, '64:ff9b::', 96)
+
+        self.vapi.nat64_add_del_pool_addr_range(server_nat_ip_n,
+                                                client_nat_ip_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        self.vapi.nat64_add_del_static_bib(server.ip6n,
+                                           server_nat_ip_n,
+                                           server_tcp_in_port,
+                                           server_tcp_out_port,
+                                           IP_PROTOS.tcp)
+
+        self.vapi.nat64_add_del_static_bib(server.ip6n,
+                                           server_nat_ip_n,
+                                           0,
+                                           0,
+                                           IP_PROTOS.gre)
+
+        self.vapi.nat64_add_del_static_bib(client.ip6n,
+                                           client_nat_ip_n,
+                                           client_tcp_in_port,
+                                           client_tcp_out_port,
+                                           IP_PROTOS.tcp)
+
+        # client to server
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=client.ip6, dst=server_nat_ip6) /
+             TCP(sport=client_tcp_in_port, dport=server_tcp_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=client.ip6, dst=server_nat_ip6, nh=IP_PROTOS.gre) /
+             GRE() /
+             IP(src=self.pg2.local_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IPv6].src, client_nat_ip6)
+            self.assertEqual(packet[IPv6].dst, server.ip6)
+            self.assertEqual(packet[IPv6].nh, IP_PROTOS.gre)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # server to client
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=server.ip6, dst=client_nat_ip6, nh=IP_PROTOS.gre) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg2.local_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IPv6].src, server_nat_ip6)
+            self.assertEqual(packet[IPv6].dst, client.ip6)
+            self.assertEqual(packet[IPv6].nh, IP_PROTOS.gre)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def nat64_get_ses_num(self):
+        """
+        Return number of active NAT64 sessions.
+        """
+        st = self.vapi.nat64_st_dump()
+        return len(st)
+
+    def clear_nat64(self):
+        """
+        Clear NAT64 configuration.
+        """
+        self.vapi.nat64_set_timeouts()
+
+        interfaces = self.vapi.nat64_interface_dump()
+        for intf in interfaces:
+            self.vapi.nat64_add_del_interface(intf.sw_if_index,
+                                              intf.is_inside,
+                                              is_add=0)
+
+        bib = self.vapi.nat64_bib_dump(IP_PROTOS.tcp)
+        for bibe in bib:
+            if bibe.is_static:
+                self.vapi.nat64_add_del_static_bib(bibe.i_addr,
+                                                   bibe.o_addr,
+                                                   bibe.i_port,
+                                                   bibe.o_port,
+                                                   bibe.proto,
+                                                   bibe.vrf_id,
+                                                   is_add=0)
+
+        bib = self.vapi.nat64_bib_dump(IP_PROTOS.udp)
+        for bibe in bib:
+            if bibe.is_static:
+                self.vapi.nat64_add_del_static_bib(bibe.i_addr,
+                                                   bibe.o_addr,
+                                                   bibe.i_port,
+                                                   bibe.o_port,
+                                                   bibe.proto,
+                                                   bibe.vrf_id,
+                                                   is_add=0)
+
+        bib = self.vapi.nat64_bib_dump(IP_PROTOS.icmp)
+        for bibe in bib:
+            if bibe.is_static:
+                self.vapi.nat64_add_del_static_bib(bibe.i_addr,
+                                                   bibe.o_addr,
+                                                   bibe.i_port,
+                                                   bibe.o_port,
+                                                   bibe.proto,
+                                                   bibe.vrf_id,
+                                                   is_add=0)
+
+        adresses = self.vapi.nat64_pool_addr_dump()
+        for addr in adresses:
+            self.vapi.nat64_add_del_pool_addr_range(addr.address,
+                                                    addr.address,
+                                                    vrf_id=addr.vrf_id,
+                                                    is_add=0)
+
+        prefixes = self.vapi.nat64_prefix_dump()
+        for prefix in prefixes:
+            self.vapi.nat64_add_del_prefix(prefix.prefix,
+                                           prefix.prefix_len,
+                                           vrf_id=prefix.vrf_id,
+                                           is_add=0)
+
+    def tearDown(self):
+        super(TestNAT64, self).tearDown()
+        if not self.vpp_dead:
+            self.logger.info(self.vapi.cli("show nat64 pool"))
+            self.logger.info(self.vapi.cli("show nat64 interfaces"))
+            self.logger.info(self.vapi.cli("show nat64 prefix"))
+            self.logger.info(self.vapi.cli("show nat64 bib all"))
+            self.logger.info(self.vapi.cli("show nat64 session table all"))
+            self.clear_nat64()
+
+if __name__ == '__main__':
+    unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_snat.py b/test/test_snat.py
deleted file mode 100644
index eb47bbb8..00000000
--- a/test/test_snat.py
+++ /dev/null
@@ -1,3718 +0,0 @@
-#!/usr/bin/env python
-
-import socket
-import unittest
-import struct
-
-from framework import VppTestCase, VppTestRunner, running_extended_tests
-from scapy.layers.inet import IP, TCP, UDP, ICMP
-from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
-from scapy.layers.inet6 import IPv6, ICMPv6EchoRequest, ICMPv6EchoReply
-from scapy.layers.inet6 import ICMPv6DestUnreach, IPerror6
-from scapy.layers.l2 import Ether, ARP, GRE
-from scapy.data import IP_PROTOS
-from scapy.packet import bind_layers
-from util import ppp
-from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
-from time import sleep
-
-
-class MethodHolder(VppTestCase):
-    """ SNAT create capture and verify method holder """
-
-    @classmethod
-    def setUpClass(cls):
-        super(MethodHolder, cls).setUpClass()
-
-    def tearDown(self):
-        super(MethodHolder, self).tearDown()
-
-    def check_ip_checksum(self, pkt):
-        """
-        Check IP checksum of the packet
-
-        :param pkt: Packet to check IP checksum
-        """
-        new = pkt.__class__(str(pkt))
-        del new['IP'].chksum
-        new = new.__class__(str(new))
-        self.assertEqual(new['IP'].chksum, pkt['IP'].chksum)
-
-    def check_tcp_checksum(self, pkt):
-        """
-        Check TCP checksum in IP packet
-
-        :param pkt: Packet to check TCP checksum
-        """
-        new = pkt.__class__(str(pkt))
-        del new['TCP'].chksum
-        new = new.__class__(str(new))
-        self.assertEqual(new['TCP'].chksum, pkt['TCP'].chksum)
-
-    def check_udp_checksum(self, pkt):
-        """
-        Check UDP checksum in IP packet
-
-        :param pkt: Packet to check UDP checksum
-        """
-        new = pkt.__class__(str(pkt))
-        del new['UDP'].chksum
-        new = new.__class__(str(new))
-        self.assertEqual(new['UDP'].chksum, pkt['UDP'].chksum)
-
-    def check_icmp_errror_embedded(self, pkt):
-        """
-        Check ICMP error embeded packet checksum
-
-        :param pkt: Packet to check ICMP error embeded packet checksum
-        """
-        if pkt.haslayer(IPerror):
-            new = pkt.__class__(str(pkt))
-            del new['IPerror'].chksum
-            new = new.__class__(str(new))
-            self.assertEqual(new['IPerror'].chksum, pkt['IPerror'].chksum)
-
-        if pkt.haslayer(TCPerror):
-            new = pkt.__class__(str(pkt))
-            del new['TCPerror'].chksum
-            new = new.__class__(str(new))
-            self.assertEqual(new['TCPerror'].chksum, pkt['TCPerror'].chksum)
-
-        if pkt.haslayer(UDPerror):
-            if pkt['UDPerror'].chksum != 0:
-                new = pkt.__class__(str(pkt))
-                del new['UDPerror'].chksum
-                new = new.__class__(str(new))
-                self.assertEqual(new['UDPerror'].chksum,
-                                 pkt['UDPerror'].chksum)
-
-        if pkt.haslayer(ICMPerror):
-            del new['ICMPerror'].chksum
-            new = new.__class__(str(new))
-            self.assertEqual(new['ICMPerror'].chksum, pkt['ICMPerror'].chksum)
-
-    def check_icmp_checksum(self, pkt):
-        """
-        Check ICMP checksum in IPv4 packet
-
-        :param pkt: Packet to check ICMP checksum
-        """
-        new = pkt.__class__(str(pkt))
-        del new['ICMP'].chksum
-        new = new.__class__(str(new))
-        self.assertEqual(new['ICMP'].chksum, pkt['ICMP'].chksum)
-        if pkt.haslayer(IPerror):
-            self.check_icmp_errror_embedded(pkt)
-
-    def check_icmpv6_checksum(self, pkt):
-        """
-        Check ICMPv6 checksum in IPv4 packet
-
-        :param pkt: Packet to check ICMPv6 checksum
-        """
-        new = pkt.__class__(str(pkt))
-        if pkt.haslayer(ICMPv6DestUnreach):
-            del new['ICMPv6DestUnreach'].cksum
-            new = new.__class__(str(new))
-            self.assertEqual(new['ICMPv6DestUnreach'].cksum,
-                             pkt['ICMPv6DestUnreach'].cksum)
-            self.check_icmp_errror_embedded(pkt)
-        if pkt.haslayer(ICMPv6EchoRequest):
-            del new['ICMPv6EchoRequest'].cksum
-            new = new.__class__(str(new))
-            self.assertEqual(new['ICMPv6EchoRequest'].cksum,
-                             pkt['ICMPv6EchoRequest'].cksum)
-        if pkt.haslayer(ICMPv6EchoReply):
-            del new['ICMPv6EchoReply'].cksum
-            new = new.__class__(str(new))
-            self.assertEqual(new['ICMPv6EchoReply'].cksum,
-                             pkt['ICMPv6EchoReply'].cksum)
-
-    def create_stream_in(self, in_if, out_if, ttl=64):
-        """
-        Create packet stream for inside network
-
-        :param in_if: Inside interface
-        :param out_if: Outside interface
-        :param ttl: TTL of generated packets
-        """
-        pkts = []
-        # TCP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
-             TCP(sport=self.tcp_port_in, dport=20))
-        pkts.append(p)
-
-        # UDP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
-             UDP(sport=self.udp_port_in, dport=20))
-        pkts.append(p)
-
-        # ICMP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
-             ICMP(id=self.icmp_id_in, type='echo-request'))
-        pkts.append(p)
-
-        return pkts
-
-    def compose_ip6(self, ip4, pref, plen):
-        """
-        Compose IPv4-embedded IPv6 addresses
-
-        :param ip4: IPv4 address
-        :param pref: IPv6 prefix
-        :param plen: IPv6 prefix length
-        :returns: IPv4-embedded IPv6 addresses
-        """
-        pref_n = list(socket.inet_pton(socket.AF_INET6, pref))
-        ip4_n = list(socket.inet_pton(socket.AF_INET, ip4))
-        if plen == 32:
-            pref_n[4] = ip4_n[0]
-            pref_n[5] = ip4_n[1]
-            pref_n[6] = ip4_n[2]
-            pref_n[7] = ip4_n[3]
-        elif plen == 40:
-            pref_n[5] = ip4_n[0]
-            pref_n[6] = ip4_n[1]
-            pref_n[7] = ip4_n[2]
-            pref_n[9] = ip4_n[3]
-        elif plen == 48:
-            pref_n[6] = ip4_n[0]
-            pref_n[7] = ip4_n[1]
-            pref_n[9] = ip4_n[2]
-            pref_n[10] = ip4_n[3]
-        elif plen == 56:
-            pref_n[7] = ip4_n[0]
-            pref_n[9] = ip4_n[1]
-            pref_n[10] = ip4_n[2]
-            pref_n[11] = ip4_n[3]
-        elif plen == 64:
-            pref_n[9] = ip4_n[0]
-            pref_n[10] = ip4_n[1]
-            pref_n[11] = ip4_n[2]
-            pref_n[12] = ip4_n[3]
-        elif plen == 96:
-            pref_n[12] = ip4_n[0]
-            pref_n[13] = ip4_n[1]
-            pref_n[14] = ip4_n[2]
-            pref_n[15] = ip4_n[3]
-        return socket.inet_ntop(socket.AF_INET6, ''.join(pref_n))
-
-    def create_stream_in_ip6(self, in_if, out_if, hlim=64, pref=None, plen=0):
-        """
-        Create IPv6 packet stream for inside network
-
-        :param in_if: Inside interface
-        :param out_if: Outside interface
-        :param ttl: Hop Limit of generated packets
-        :param pref: NAT64 prefix
-        :param plen: NAT64 prefix length
-        """
-        pkts = []
-        if pref is None:
-            dst = ''.join(['64:ff9b::', out_if.remote_ip4])
-        else:
-            dst = self.compose_ip6(out_if.remote_ip4, pref, plen)
-
-        # TCP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IPv6(src=in_if.remote_ip6, dst=dst, hlim=hlim) /
-             TCP(sport=self.tcp_port_in, dport=20))
-        pkts.append(p)
-
-        # UDP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IPv6(src=in_if.remote_ip6, dst=dst, hlim=hlim) /
-             UDP(sport=self.udp_port_in, dport=20))
-        pkts.append(p)
-
-        # ICMP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IPv6(src=in_if.remote_ip6, dst=dst, hlim=hlim) /
-             ICMPv6EchoRequest(id=self.icmp_id_in))
-        pkts.append(p)
-
-        return pkts
-
-    def create_stream_out(self, out_if, dst_ip=None, ttl=64):
-        """
-        Create packet stream for outside network
-
-        :param out_if: Outside interface
-        :param dst_ip: Destination IP address (Default use global SNAT address)
-        :param ttl: TTL of generated packets
-        """
-        if dst_ip is None:
-            dst_ip = self.snat_addr
-        pkts = []
-        # TCP
-        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
-             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
-             TCP(dport=self.tcp_port_out, sport=20))
-        pkts.append(p)
-
-        # UDP
-        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
-             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
-             UDP(dport=self.udp_port_out, sport=20))
-        pkts.append(p)
-
-        # ICMP
-        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
-             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
-             ICMP(id=self.icmp_id_out, type='echo-reply'))
-        pkts.append(p)
-
-        return pkts
-
-    def verify_capture_out(self, capture, nat_ip=None, same_port=False,
-                           packet_num=3, dst_ip=None):
-        """
-        Verify captured packets on outside network
-
-        :param capture: Captured packets
-        :param nat_ip: Translated IP address (Default use global SNAT address)
-        :param same_port: Sorce port number is not translated (Default False)
-        :param packet_num: Expected number of packets (Default 3)
-        :param dst_ip: Destination IP address (Default do not verify)
-        """
-        if nat_ip is None:
-            nat_ip = self.snat_addr
-        self.assertEqual(packet_num, len(capture))
-        for packet in capture:
-            try:
-                self.check_ip_checksum(packet)
-                self.assertEqual(packet[IP].src, nat_ip)
-                if dst_ip is not None:
-                    self.assertEqual(packet[IP].dst, dst_ip)
-                if packet.haslayer(TCP):
-                    if same_port:
-                        self.assertEqual(packet[TCP].sport, self.tcp_port_in)
-                    else:
-                        self.assertNotEqual(
-                            packet[TCP].sport, self.tcp_port_in)
-                    self.tcp_port_out = packet[TCP].sport
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    if same_port:
-                        self.assertEqual(packet[UDP].sport, self.udp_port_in)
-                    else:
-                        self.assertNotEqual(
-                            packet[UDP].sport, self.udp_port_in)
-                    self.udp_port_out = packet[UDP].sport
-                else:
-                    if same_port:
-                        self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-                    else:
-                        self.assertNotEqual(packet[ICMP].id, self.icmp_id_in)
-                    self.icmp_id_out = packet[ICMP].id
-                    self.check_icmp_checksum(packet)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(outside network):", packet))
-                raise
-
-    def verify_capture_in(self, capture, in_if, packet_num=3):
-        """
-        Verify captured packets on inside network
-
-        :param capture: Captured packets
-        :param in_if: Inside interface
-        :param packet_num: Expected number of packets (Default 3)
-        """
-        self.assertEqual(packet_num, len(capture))
-        for packet in capture:
-            try:
-                self.check_ip_checksum(packet)
-                self.assertEqual(packet[IP].dst, in_if.remote_ip4)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
-                else:
-                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-                    self.check_icmp_checksum(packet)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(inside network):", packet))
-                raise
-
-    def verify_capture_in_ip6(self, capture, src_ip, dst_ip, packet_num=3):
-        """
-        Verify captured IPv6 packets on inside network
-
-        :param capture: Captured packets
-        :param src_ip: Source IP
-        :param dst_ip: Destination IP address
-        :param packet_num: Expected number of packets (Default 3)
-        """
-        self.assertEqual(packet_num, len(capture))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IPv6].src, src_ip)
-                self.assertEqual(packet[IPv6].dst, dst_ip)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
-                    self.check_udp_checksum(packet)
-                else:
-                    self.assertEqual(packet[ICMPv6EchoReply].id,
-                                     self.icmp_id_in)
-                    self.check_icmpv6_checksum(packet)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(inside network):", packet))
-                raise
-
-    def verify_capture_no_translation(self, capture, ingress_if, egress_if):
-        """
-        Verify captured packet that don't have to be translated
-
-        :param capture: Captured packets
-        :param ingress_if: Ingress interface
-        :param egress_if: Egress interface
-        """
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, ingress_if.remote_ip4)
-                self.assertEqual(packet[IP].dst, egress_if.remote_ip4)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].sport, self.tcp_port_in)
-                elif packet.haslayer(UDP):
-                    self.assertEqual(packet[UDP].sport, self.udp_port_in)
-                else:
-                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(inside network):", packet))
-                raise
-
-    def verify_capture_out_with_icmp_errors(self, capture, src_ip=None,
-                                            packet_num=3, icmp_type=11):
-        """
-        Verify captured packets with ICMP errors on outside network
-
-        :param capture: Captured packets
-        :param src_ip: Translated IP address or IP address of VPP
-                       (Default use global SNAT address)
-        :param packet_num: Expected number of packets (Default 3)
-        :param icmp_type: Type of error ICMP packet
-                          we are expecting (Default 11)
-        """
-        if src_ip is None:
-            src_ip = self.snat_addr
-        self.assertEqual(packet_num, len(capture))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, src_ip)
-                self.assertTrue(packet.haslayer(ICMP))
-                icmp = packet[ICMP]
-                self.assertEqual(icmp.type, icmp_type)
-                self.assertTrue(icmp.haslayer(IPerror))
-                inner_ip = icmp[IPerror]
-                if inner_ip.haslayer(TCPerror):
-                    self.assertEqual(inner_ip[TCPerror].dport,
-                                     self.tcp_port_out)
-                elif inner_ip.haslayer(UDPerror):
-                    self.assertEqual(inner_ip[UDPerror].dport,
-                                     self.udp_port_out)
-                else:
-                    self.assertEqual(inner_ip[ICMPerror].id, self.icmp_id_out)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(outside network):", packet))
-                raise
-
-    def verify_capture_in_with_icmp_errors(self, capture, in_if, packet_num=3,
-                                           icmp_type=11):
-        """
-        Verify captured packets with ICMP errors on inside network
-
-        :param capture: Captured packets
-        :param in_if: Inside interface
-        :param packet_num: Expected number of packets (Default 3)
-        :param icmp_type: Type of error ICMP packet
-                          we are expecting (Default 11)
-        """
-        self.assertEqual(packet_num, len(capture))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].dst, in_if.remote_ip4)
-                self.assertTrue(packet.haslayer(ICMP))
-                icmp = packet[ICMP]
-                self.assertEqual(icmp.type, icmp_type)
-                self.assertTrue(icmp.haslayer(IPerror))
-                inner_ip = icmp[IPerror]
-                if inner_ip.haslayer(TCPerror):
-                    self.assertEqual(inner_ip[TCPerror].sport,
-                                     self.tcp_port_in)
-                elif inner_ip.haslayer(UDPerror):
-                    self.assertEqual(inner_ip[UDPerror].sport,
-                                     self.udp_port_in)
-                else:
-                    self.assertEqual(inner_ip[ICMPerror].id, self.icmp_id_in)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(inside network):", packet))
-                raise
-
-    def verify_ipfix_nat44_ses(self, data):
-        """
-        Verify IPFIX NAT44 session create/delete event
-
-        :param data: Decoded IPFIX data records
-        """
-        nat44_ses_create_num = 0
-        nat44_ses_delete_num = 0
-        self.assertEqual(6, len(data))
-        for record in data:
-            # natEvent
-            self.assertIn(ord(record[230]), [4, 5])
-            if ord(record[230]) == 4:
-                nat44_ses_create_num += 1
-            else:
-                nat44_ses_delete_num += 1
-            # sourceIPv4Address
-            self.assertEqual(self.pg0.remote_ip4n, record[8])
-            # postNATSourceIPv4Address
-            self.assertEqual(socket.inet_pton(socket.AF_INET, self.snat_addr),
-                             record[225])
-            # ingressVRFID
-            self.assertEqual(struct.pack("!I", 0), record[234])
-            # protocolIdentifier/sourceTransportPort/postNAPTSourceTransportPort
-            if IP_PROTOS.icmp == ord(record[4]):
-                self.assertEqual(struct.pack("!H", self.icmp_id_in), record[7])
-                self.assertEqual(struct.pack("!H", self.icmp_id_out),
-                                 record[227])
-            elif IP_PROTOS.tcp == ord(record[4]):
-                self.assertEqual(struct.pack("!H", self.tcp_port_in),
-                                 record[7])
-                self.assertEqual(struct.pack("!H", self.tcp_port_out),
-                                 record[227])
-            elif IP_PROTOS.udp == ord(record[4]):
-                self.assertEqual(struct.pack("!H", self.udp_port_in),
-                                 record[7])
-                self.assertEqual(struct.pack("!H", self.udp_port_out),
-                                 record[227])
-            else:
-                self.fail("Invalid protocol")
-        self.assertEqual(3, nat44_ses_create_num)
-        self.assertEqual(3, nat44_ses_delete_num)
-
-    def verify_ipfix_addr_exhausted(self, data):
-        """
-        Verify IPFIX NAT addresses event
-
-        :param data: Decoded IPFIX data records
-        """
-        self.assertEqual(1, len(data))
-        record = data[0]
-        # natEvent
-        self.assertEqual(ord(record[230]), 3)
-        # natPoolID
-        self.assertEqual(struct.pack("!I", 0), record[283])
-
-
-class TestSNAT(MethodHolder):
-    """ SNAT Test Cases """
-
-    @classmethod
-    def setUpClass(cls):
-        super(TestSNAT, cls).setUpClass()
-
-        try:
-            cls.tcp_port_in = 6303
-            cls.tcp_port_out = 6303
-            cls.udp_port_in = 6304
-            cls.udp_port_out = 6304
-            cls.icmp_id_in = 6305
-            cls.icmp_id_out = 6305
-            cls.snat_addr = '10.0.0.3'
-            cls.ipfix_src_port = 4739
-            cls.ipfix_domain_id = 1
-
-            cls.create_pg_interfaces(range(9))
-            cls.interfaces = list(cls.pg_interfaces[0:4])
-
-            for i in cls.interfaces:
-                i.admin_up()
-                i.config_ip4()
-                i.resolve_arp()
-
-            cls.pg0.generate_remote_hosts(3)
-            cls.pg0.configure_ipv4_neighbors()
-
-            cls.overlapping_interfaces = list(list(cls.pg_interfaces[4:7]))
-
-            cls.pg4._local_ip4 = "172.16.255.1"
-            cls.pg4._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
-            cls.pg4._remote_hosts[0]._ip4 = "172.16.255.2"
-            cls.pg4.set_table_ip4(10)
-            cls.pg5._local_ip4 = "172.17.255.3"
-            cls.pg5._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
-            cls.pg5._remote_hosts[0]._ip4 = "172.17.255.4"
-            cls.pg5.set_table_ip4(10)
-            cls.pg6._local_ip4 = "172.16.255.1"
-            cls.pg6._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
-            cls.pg6._remote_hosts[0]._ip4 = "172.16.255.2"
-            cls.pg6.set_table_ip4(20)
-            for i in cls.overlapping_interfaces:
-                i.config_ip4()
-                i.admin_up()
-                i.resolve_arp()
-
-            cls.pg7.admin_up()
-            cls.pg8.admin_up()
-
-        except Exception:
-            super(TestSNAT, cls).tearDownClass()
-            raise
-
-    def clear_snat(self):
-        """
-        Clear SNAT configuration.
-        """
-        # I found no elegant way to do this
-        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg7.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg7.sw_if_index,
-                                   is_add=0)
-        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg8.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg8.sw_if_index,
-                                   is_add=0)
-
-        for intf in [self.pg7, self.pg8]:
-            neighbors = self.vapi.ip_neighbor_dump(intf.sw_if_index)
-            for n in neighbors:
-                self.vapi.ip_neighbor_add_del(intf.sw_if_index,
-                                              n.mac_address,
-                                              n.ip_address,
-                                              is_add=0)
-
-        if self.pg7.has_ip4_config:
-            self.pg7.unconfig_ip4()
-
-        interfaces = self.vapi.snat_interface_addr_dump()
-        for intf in interfaces:
-            self.vapi.snat_add_interface_addr(intf.sw_if_index, is_add=0)
-
-        self.vapi.snat_ipfix(enable=0, src_port=self.ipfix_src_port,
-                             domain_id=self.ipfix_domain_id)
-        self.ipfix_src_port = 4739
-        self.ipfix_domain_id = 1
-
-        interfaces = self.vapi.snat_interface_dump()
-        for intf in interfaces:
-            self.vapi.snat_interface_add_del_feature(intf.sw_if_index,
-                                                     intf.is_inside,
-                                                     is_add=0)
-
-        interfaces = self.vapi.snat_interface_output_feature_dump()
-        for intf in interfaces:
-            self.vapi.snat_interface_add_del_output_feature(intf.sw_if_index,
-                                                            intf.is_inside,
-                                                            is_add=0)
-
-        static_mappings = self.vapi.snat_static_mapping_dump()
-        for sm in static_mappings:
-            self.vapi.snat_add_static_mapping(sm.local_ip_address,
-                                              sm.external_ip_address,
-                                              local_port=sm.local_port,
-                                              external_port=sm.external_port,
-                                              addr_only=sm.addr_only,
-                                              vrf_id=sm.vrf_id,
-                                              protocol=sm.protocol,
-                                              is_add=0)
-
-        adresses = self.vapi.snat_address_dump()
-        for addr in adresses:
-            self.vapi.snat_add_address_range(addr.ip_address,
-                                             addr.ip_address,
-                                             is_add=0)
-
-    def snat_add_static_mapping(self, local_ip, external_ip='0.0.0.0',
-                                local_port=0, external_port=0, vrf_id=0,
-                                is_add=1, external_sw_if_index=0xFFFFFFFF,
-                                proto=0):
-        """
-        Add/delete S-NAT static mapping
-
-        :param local_ip: Local IP address
-        :param external_ip: External IP address
-        :param local_port: Local port number (Optional)
-        :param external_port: External port number (Optional)
-        :param vrf_id: VRF ID (Default 0)
-        :param is_add: 1 if add, 0 if delete (Default add)
-        :param external_sw_if_index: External interface instead of IP address
-        :param proto: IP protocol (Mandatory if port specified)
-        """
-        addr_only = 1
-        if local_port and external_port:
-            addr_only = 0
-        l_ip = socket.inet_pton(socket.AF_INET, local_ip)
-        e_ip = socket.inet_pton(socket.AF_INET, external_ip)
-        self.vapi.snat_add_static_mapping(
-            l_ip,
-            e_ip,
-            external_sw_if_index,
-            local_port,
-            external_port,
-            addr_only,
-            vrf_id,
-            proto,
-            is_add)
-
-    def snat_add_address(self, ip, is_add=1, vrf_id=0xFFFFFFFF):
-        """
-        Add/delete S-NAT address
-
-        :param ip: IP address
-        :param is_add: 1 if add, 0 if delete (Default add)
-        """
-        snat_addr = socket.inet_pton(socket.AF_INET, ip)
-        self.vapi.snat_add_address_range(snat_addr, snat_addr, is_add,
-                                         vrf_id=vrf_id)
-
-    def test_dynamic(self):
-        """ SNAT dynamic translation test """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-    def test_dynamic_icmp_errors_in2out_ttl_1(self):
-        """ SNAT handling of client packets with TTL=1 """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # Client side - generate traffic
-        pkts = self.create_stream_in(self.pg0, self.pg1, ttl=1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Client side - verify ICMP type 11 packets
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in_with_icmp_errors(capture, self.pg0)
-
-    def test_dynamic_icmp_errors_out2in_ttl_1(self):
-        """ SNAT handling of server packets with TTL=1 """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # Client side - create sessions
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Server side - generate traffic
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-        pkts = self.create_stream_out(self.pg1, ttl=1)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Server side - verify ICMP type 11 packets
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out_with_icmp_errors(capture,
-                                                 src_ip=self.pg1.local_ip4)
-
-    def test_dynamic_icmp_errors_in2out_ttl_2(self):
-        """ SNAT handling of error responses to client packets with TTL=2 """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # Client side - generate traffic
-        pkts = self.create_stream_in(self.pg0, self.pg1, ttl=2)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Server side - simulate ICMP type 11 response
-        capture = self.pg1.get_capture(len(pkts))
-        pkts = [Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-                IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
-                ICMP(type=11) / packet[IP] for packet in capture]
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Client side - verify ICMP type 11 packets
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in_with_icmp_errors(capture, self.pg0)
-
-    def test_dynamic_icmp_errors_out2in_ttl_2(self):
-        """ SNAT handling of error responses to server packets with TTL=2 """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # Client side - create sessions
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Server side - generate traffic
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-        pkts = self.create_stream_out(self.pg1, ttl=2)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Client side - simulate ICMP type 11 response
-        capture = self.pg0.get_capture(len(pkts))
-        pkts = [Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-                IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                ICMP(type=11) / packet[IP] for packet in capture]
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # Server side - verify ICMP type 11 packets
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out_with_icmp_errors(capture)
-
-    def test_ping_out_interface_from_outside(self):
-        """ Ping SNAT out interface from outside network """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-             IP(src=self.pg1.remote_ip4, dst=self.pg1.local_ip4) /
-             ICMP(id=self.icmp_id_out, type='echo-request'))
-        pkts = [p]
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.assertEqual(1, len(capture))
-        packet = capture[0]
-        try:
-            self.assertEqual(packet[IP].src, self.pg1.local_ip4)
-            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
-            self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-            self.assertEqual(packet[ICMP].type, 0)  # echo reply
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet "
-                                  "(outside network):", packet))
-            raise
-
-    def test_ping_internal_host_from_outside(self):
-        """ Ping internal host from outside network """
-
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # out2in
-        pkt = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-               IP(src=self.pg1.remote_ip4, dst=self.snat_addr, ttl=64) /
-               ICMP(id=self.icmp_id_out, type='echo-request'))
-        self.pg1.add_stream(pkt)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        self.verify_capture_in(capture, self.pg0, packet_num=1)
-        self.assert_equal(capture[0][IP].proto, IP_PROTOS.icmp)
-
-        # in2out
-        pkt = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-               IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4, ttl=64) /
-               ICMP(id=self.icmp_id_in, type='echo-reply'))
-        self.pg0.add_stream(pkt)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(1)
-        self.verify_capture_out(capture, same_port=True, packet_num=1)
-        self.assert_equal(capture[0][IP].proto, IP_PROTOS.icmp)
-
-    def test_static_in(self):
-        """ SNAT 1:1 NAT initialized from inside network """
-
-        nat_ip = "10.0.0.10"
-        self.tcp_port_out = 6303
-        self.udp_port_out = 6304
-        self.icmp_id_out = 6305
-
-        self.snat_add_static_mapping(self.pg0.remote_ip4, nat_ip)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip, True)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1, nat_ip)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-    def test_static_out(self):
-        """ SNAT 1:1 NAT initialized from outside network """
-
-        nat_ip = "10.0.0.20"
-        self.tcp_port_out = 6303
-        self.udp_port_out = 6304
-        self.icmp_id_out = 6305
-
-        self.snat_add_static_mapping(self.pg0.remote_ip4, nat_ip)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1, nat_ip)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip, True)
-
-    def test_static_with_port_in(self):
-        """ SNAT 1:1 NAT with port initialized from inside network """
-
-        self.tcp_port_out = 3606
-        self.udp_port_out = 3607
-        self.icmp_id_out = 3608
-
-        self.snat_add_address(self.snat_addr)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr,
-                                     self.tcp_port_in, self.tcp_port_out,
-                                     proto=IP_PROTOS.tcp)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr,
-                                     self.udp_port_in, self.udp_port_out,
-                                     proto=IP_PROTOS.udp)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr,
-                                     self.icmp_id_in, self.icmp_id_out,
-                                     proto=IP_PROTOS.icmp)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-    def test_static_with_port_out(self):
-        """ SNAT 1:1 NAT with port initialized from outside network """
-
-        self.tcp_port_out = 30606
-        self.udp_port_out = 30607
-        self.icmp_id_out = 30608
-
-        self.snat_add_address(self.snat_addr)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr,
-                                     self.tcp_port_in, self.tcp_port_out,
-                                     proto=IP_PROTOS.tcp)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr,
-                                     self.udp_port_in, self.udp_port_out,
-                                     proto=IP_PROTOS.udp)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, self.snat_addr,
-                                     self.icmp_id_in, self.icmp_id_out,
-                                     proto=IP_PROTOS.icmp)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-    def test_static_vrf_aware(self):
-        """ SNAT 1:1 NAT VRF awareness """
-
-        nat_ip1 = "10.0.0.30"
-        nat_ip2 = "10.0.0.40"
-        self.tcp_port_out = 6303
-        self.udp_port_out = 6304
-        self.icmp_id_out = 6305
-
-        self.snat_add_static_mapping(self.pg4.remote_ip4, nat_ip1,
-                                     vrf_id=10)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, nat_ip2,
-                                     vrf_id=10)
-        self.vapi.snat_interface_add_del_feature(self.pg3.sw_if_index,
-                                                 is_inside=0)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg4.sw_if_index)
-
-        # inside interface VRF match SNAT static mapping VRF
-        pkts = self.create_stream_in(self.pg4, self.pg3)
-        self.pg4.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip1, True)
-
-        # inside interface VRF don't match SNAT static mapping VRF (packets
-        # are dropped)
-        pkts = self.create_stream_in(self.pg0, self.pg3)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        self.pg3.assert_nothing_captured()
-
-    def test_multiple_inside_interfaces(self):
-        """ SNAT multiple inside interfaces (non-overlapping address space) """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg3.sw_if_index,
-                                                 is_inside=0)
-
-        # between two S-NAT inside interfaces (no translation)
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_no_translation(capture, self.pg0, self.pg1)
-
-        # from S-NAT inside to interface without S-NAT feature (no translation)
-        pkts = self.create_stream_in(self.pg0, self.pg2)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_no_translation(capture, self.pg0, self.pg2)
-
-        # in2out 1st interface
-        pkts = self.create_stream_in(self.pg0, self.pg3)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in 1st interface
-        pkts = self.create_stream_out(self.pg3)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-        # in2out 2nd interface
-        pkts = self.create_stream_in(self.pg1, self.pg3)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in 2nd interface
-        pkts = self.create_stream_out(self.pg3)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg1)
-
-    def test_inside_overlapping_interfaces(self):
-        """ SNAT multiple inside interfaces with overlapping address space """
-
-        static_nat_ip = "10.0.0.10"
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg3.sw_if_index,
-                                                 is_inside=0)
-        self.vapi.snat_interface_add_del_feature(self.pg4.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg5.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg6.sw_if_index)
-        self.snat_add_static_mapping(self.pg6.remote_ip4, static_nat_ip,
-                                     vrf_id=20)
-
-        # between S-NAT inside interfaces with same VRF (no translation)
-        pkts = self.create_stream_in(self.pg4, self.pg5)
-        self.pg4.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg5.get_capture(len(pkts))
-        self.verify_capture_no_translation(capture, self.pg4, self.pg5)
-
-        # between S-NAT inside interfaces with different VRF (hairpinning)
-        p = (Ether(src=self.pg4.remote_mac, dst=self.pg4.local_mac) /
-             IP(src=self.pg4.remote_ip4, dst=static_nat_ip) /
-             TCP(sport=1234, dport=5678))
-        self.pg4.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg6.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.snat_addr)
-            self.assertEqual(ip.dst, self.pg6.remote_ip4)
-            self.assertNotEqual(tcp.sport, 1234)
-            self.assertEqual(tcp.dport, 5678)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", p))
-            raise
-
-        # in2out 1st interface
-        pkts = self.create_stream_in(self.pg4, self.pg3)
-        self.pg4.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in 1st interface
-        pkts = self.create_stream_out(self.pg3)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg4.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg4)
-
-        # in2out 2nd interface
-        pkts = self.create_stream_in(self.pg5, self.pg3)
-        self.pg5.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in 2nd interface
-        pkts = self.create_stream_out(self.pg3)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg5.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg5)
-
-        # pg5 session dump
-        addresses = self.vapi.snat_address_dump()
-        self.assertEqual(len(addresses), 1)
-        sessions = self.vapi.snat_user_session_dump(self.pg5.remote_ip4n, 10)
-        self.assertEqual(len(sessions), 3)
-        for session in sessions:
-            self.assertFalse(session.is_static)
-            self.assertEqual(session.inside_ip_address[0:4],
-                             self.pg5.remote_ip4n)
-            self.assertEqual(session.outside_ip_address,
-                             addresses[0].ip_address)
-        self.assertEqual(sessions[0].protocol, IP_PROTOS.tcp)
-        self.assertEqual(sessions[1].protocol, IP_PROTOS.udp)
-        self.assertEqual(sessions[2].protocol, IP_PROTOS.icmp)
-        self.assertEqual(sessions[0].inside_port, self.tcp_port_in)
-        self.assertEqual(sessions[1].inside_port, self.udp_port_in)
-        self.assertEqual(sessions[2].inside_port, self.icmp_id_in)
-        self.assertEqual(sessions[0].outside_port, self.tcp_port_out)
-        self.assertEqual(sessions[1].outside_port, self.udp_port_out)
-        self.assertEqual(sessions[2].outside_port, self.icmp_id_out)
-
-        # in2out 3rd interface
-        pkts = self.create_stream_in(self.pg6, self.pg3)
-        self.pg6.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture, static_nat_ip, True)
-
-        # out2in 3rd interface
-        pkts = self.create_stream_out(self.pg3, static_nat_ip)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg6.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg6)
-
-        # general user and session dump verifications
-        users = self.vapi.snat_user_dump()
-        self.assertTrue(len(users) >= 3)
-        addresses = self.vapi.snat_address_dump()
-        self.assertEqual(len(addresses), 1)
-        for user in users:
-            sessions = self.vapi.snat_user_session_dump(user.ip_address,
-                                                        user.vrf_id)
-            for session in sessions:
-                self.assertEqual(user.ip_address, session.inside_ip_address)
-                self.assertTrue(session.total_bytes > session.total_pkts > 0)
-                self.assertTrue(session.protocol in
-                                [IP_PROTOS.tcp, IP_PROTOS.udp,
-                                 IP_PROTOS.icmp])
-
-        # pg4 session dump
-        sessions = self.vapi.snat_user_session_dump(self.pg4.remote_ip4n, 10)
-        self.assertTrue(len(sessions) >= 4)
-        for session in sessions:
-            self.assertFalse(session.is_static)
-            self.assertEqual(session.inside_ip_address[0:4],
-                             self.pg4.remote_ip4n)
-            self.assertEqual(session.outside_ip_address,
-                             addresses[0].ip_address)
-
-        # pg6 session dump
-        sessions = self.vapi.snat_user_session_dump(self.pg6.remote_ip4n, 20)
-        self.assertTrue(len(sessions) >= 3)
-        for session in sessions:
-            self.assertTrue(session.is_static)
-            self.assertEqual(session.inside_ip_address[0:4],
-                             self.pg6.remote_ip4n)
-            self.assertEqual(map(ord, session.outside_ip_address[0:4]),
-                             map(int, static_nat_ip.split('.')))
-            self.assertTrue(session.inside_port in
-                            [self.tcp_port_in, self.udp_port_in,
-                             self.icmp_id_in])
-
-    def test_hairpinning(self):
-        """ SNAT hairpinning - 1:1 NAT with port"""
-
-        host = self.pg0.remote_hosts[0]
-        server = self.pg0.remote_hosts[1]
-        host_in_port = 1234
-        host_out_port = 0
-        server_in_port = 5678
-        server_out_port = 8765
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-        # add static mapping for server
-        self.snat_add_static_mapping(server.ip4, self.snat_addr,
-                                     server_in_port, server_out_port,
-                                     proto=IP_PROTOS.tcp)
-
-        # send packet from host to server
-        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
-             IP(src=host.ip4, dst=self.snat_addr) /
-             TCP(sport=host_in_port, dport=server_out_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.snat_addr)
-            self.assertEqual(ip.dst, server.ip4)
-            self.assertNotEqual(tcp.sport, host_in_port)
-            self.assertEqual(tcp.dport, server_in_port)
-            self.check_tcp_checksum(p)
-            host_out_port = tcp.sport
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", p))
-            raise
-
-        # send reply from server to host
-        p = (Ether(src=server.mac, dst=self.pg0.local_mac) /
-             IP(src=server.ip4, dst=self.snat_addr) /
-             TCP(sport=server_in_port, dport=host_out_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.snat_addr)
-            self.assertEqual(ip.dst, host.ip4)
-            self.assertEqual(tcp.sport, server_out_port)
-            self.assertEqual(tcp.dport, host_in_port)
-            self.check_tcp_checksum(p)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:"), p)
-            raise
-
-    def test_hairpinning2(self):
-        """ SNAT hairpinning - 1:1 NAT"""
-
-        server1_nat_ip = "10.0.0.10"
-        server2_nat_ip = "10.0.0.11"
-        host = self.pg0.remote_hosts[0]
-        server1 = self.pg0.remote_hosts[1]
-        server2 = self.pg0.remote_hosts[2]
-        server_tcp_port = 22
-        server_udp_port = 20
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # add static mapping for servers
-        self.snat_add_static_mapping(server1.ip4, server1_nat_ip)
-        self.snat_add_static_mapping(server2.ip4, server2_nat_ip)
-
-        # host to server1
-        pkts = []
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=host.ip4, dst=server1_nat_ip) /
-             TCP(sport=self.tcp_port_in, dport=server_tcp_port))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=host.ip4, dst=server1_nat_ip) /
-             UDP(sport=self.udp_port_in, dport=server_udp_port))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=host.ip4, dst=server1_nat_ip) /
-             ICMP(id=self.icmp_id_in, type='echo-request'))
-        pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, self.snat_addr)
-                self.assertEqual(packet[IP].dst, server1.ip4)
-                if packet.haslayer(TCP):
-                    self.assertNotEqual(packet[TCP].sport, self.tcp_port_in)
-                    self.assertEqual(packet[TCP].dport, server_tcp_port)
-                    self.tcp_port_out = packet[TCP].sport
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    self.assertNotEqual(packet[UDP].sport, self.udp_port_in)
-                    self.assertEqual(packet[UDP].dport, server_udp_port)
-                    self.udp_port_out = packet[UDP].sport
-                else:
-                    self.assertNotEqual(packet[ICMP].id, self.icmp_id_in)
-                    self.icmp_id_out = packet[ICMP].id
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-        # server1 to host
-        pkts = []
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server1.ip4, dst=self.snat_addr) /
-             TCP(sport=server_tcp_port, dport=self.tcp_port_out))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server1.ip4, dst=self.snat_addr) /
-             UDP(sport=server_udp_port, dport=self.udp_port_out))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server1.ip4, dst=self.snat_addr) /
-             ICMP(id=self.icmp_id_out, type='echo-reply'))
-        pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, server1_nat_ip)
-                self.assertEqual(packet[IP].dst, host.ip4)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
-                    self.assertEqual(packet[TCP].sport, server_tcp_port)
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
-                    self.assertEqual(packet[UDP].sport, server_udp_port)
-                else:
-                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-        # server2 to server1
-        pkts = []
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server2.ip4, dst=server1_nat_ip) /
-             TCP(sport=self.tcp_port_in, dport=server_tcp_port))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server2.ip4, dst=server1_nat_ip) /
-             UDP(sport=self.udp_port_in, dport=server_udp_port))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server2.ip4, dst=server1_nat_ip) /
-             ICMP(id=self.icmp_id_in, type='echo-request'))
-        pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, server2_nat_ip)
-                self.assertEqual(packet[IP].dst, server1.ip4)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].sport, self.tcp_port_in)
-                    self.assertEqual(packet[TCP].dport, server_tcp_port)
-                    self.tcp_port_out = packet[TCP].sport
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    self.assertEqual(packet[UDP].sport, self.udp_port_in)
-                    self.assertEqual(packet[UDP].dport, server_udp_port)
-                    self.udp_port_out = packet[UDP].sport
-                else:
-                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-                    self.icmp_id_out = packet[ICMP].id
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-        # server1 to server2
-        pkts = []
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server1.ip4, dst=server2_nat_ip) /
-             TCP(sport=server_tcp_port, dport=self.tcp_port_out))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server1.ip4, dst=server2_nat_ip) /
-             UDP(sport=server_udp_port, dport=self.udp_port_out))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=server1.ip4, dst=server2_nat_ip) /
-             ICMP(id=self.icmp_id_out, type='echo-reply'))
-        pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, server1_nat_ip)
-                self.assertEqual(packet[IP].dst, server2.ip4)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].dport, self.tcp_port_in)
-                    self.assertEqual(packet[TCP].sport, server_tcp_port)
-                    self.check_tcp_checksum(packet)
-                elif packet.haslayer(UDP):
-                    self.assertEqual(packet[UDP].dport, self.udp_port_in)
-                    self.assertEqual(packet[UDP].sport, server_udp_port)
-                else:
-                    self.assertEqual(packet[ICMP].id, self.icmp_id_in)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-    def test_max_translations_per_user(self):
-        """ MAX translations per user - recycle the least recently used """
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # get maximum number of translations per user
-        snat_config = self.vapi.snat_show_config()
-
-        # send more than maximum number of translations per user packets
-        pkts_num = snat_config.max_translations_per_user + 5
-        pkts = []
-        for port in range(0, pkts_num):
-            p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                 TCP(sport=1025 + port))
-            pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-
-        # verify number of translated packet
-        self.pg1.get_capture(pkts_num)
-
-    def test_interface_addr(self):
-        """ Acquire SNAT addresses from interface """
-        self.vapi.snat_add_interface_addr(self.pg7.sw_if_index)
-
-        # no address in NAT pool
-        adresses = self.vapi.snat_address_dump()
-        self.assertEqual(0, len(adresses))
-
-        # configure interface address and check NAT address pool
-        self.pg7.config_ip4()
-        adresses = self.vapi.snat_address_dump()
-        self.assertEqual(1, len(adresses))
-        self.assertEqual(adresses[0].ip_address[0:4], self.pg7.local_ip4n)
-
-        # remove interface address and check NAT address pool
-        self.pg7.unconfig_ip4()
-        adresses = self.vapi.snat_address_dump()
-        self.assertEqual(0, len(adresses))
-
-    def test_interface_addr_static_mapping(self):
-        """ Static mapping with addresses from interface """
-        self.vapi.snat_add_interface_addr(self.pg7.sw_if_index)
-        self.snat_add_static_mapping('1.2.3.4',
-                                     external_sw_if_index=self.pg7.sw_if_index)
-
-        # static mappings with external interface
-        static_mappings = self.vapi.snat_static_mapping_dump()
-        self.assertEqual(1, len(static_mappings))
-        self.assertEqual(self.pg7.sw_if_index,
-                         static_mappings[0].external_sw_if_index)
-
-        # configure interface address and check static mappings
-        self.pg7.config_ip4()
-        static_mappings = self.vapi.snat_static_mapping_dump()
-        self.assertEqual(1, len(static_mappings))
-        self.assertEqual(static_mappings[0].external_ip_address[0:4],
-                         self.pg7.local_ip4n)
-        self.assertEqual(0xFFFFFFFF, static_mappings[0].external_sw_if_index)
-
-        # remove interface address and check static mappings
-        self.pg7.unconfig_ip4()
-        static_mappings = self.vapi.snat_static_mapping_dump()
-        self.assertEqual(0, len(static_mappings))
-
-    def test_ipfix_nat44_sess(self):
-        """ S-NAT IPFIX logging NAT44 session created/delted """
-        self.ipfix_domain_id = 10
-        self.ipfix_src_port = 20202
-        colector_port = 30303
-        bind_layers(UDP, IPFIX, dport=30303)
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-        self.vapi.set_ipfix_exporter(collector_address=self.pg3.remote_ip4n,
-                                     src_address=self.pg3.local_ip4n,
-                                     path_mtu=512,
-                                     template_interval=10,
-                                     collector_port=colector_port)
-        self.vapi.snat_ipfix(domain_id=self.ipfix_domain_id,
-                             src_port=self.ipfix_src_port)
-
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-        self.snat_add_address(self.snat_addr, is_add=0)
-        self.vapi.cli("ipfix flush")  # FIXME this should be an API call
-        capture = self.pg3.get_capture(3)
-        ipfix = IPFIXDecoder()
-        # first load template
-        for p in capture:
-            self.assertTrue(p.haslayer(IPFIX))
-            self.assertEqual(p[IP].src, self.pg3.local_ip4)
-            self.assertEqual(p[IP].dst, self.pg3.remote_ip4)
-            self.assertEqual(p[UDP].sport, self.ipfix_src_port)
-            self.assertEqual(p[UDP].dport, colector_port)
-            self.assertEqual(p[IPFIX].observationDomainID,
-                             self.ipfix_domain_id)
-            if p.haslayer(Template):
-                ipfix.add_template(p.getlayer(Template))
-        # verify events in data set
-        for p in capture:
-            if p.haslayer(Data):
-                data = ipfix.decode_data_set(p.getlayer(Set))
-                self.verify_ipfix_nat44_ses(data)
-
-    def test_ipfix_addr_exhausted(self):
-        """ S-NAT IPFIX logging NAT addresses exhausted """
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-        self.vapi.set_ipfix_exporter(collector_address=self.pg3.remote_ip4n,
-                                     src_address=self.pg3.local_ip4n,
-                                     path_mtu=512,
-                                     template_interval=10)
-        self.vapi.snat_ipfix(domain_id=self.ipfix_domain_id,
-                             src_port=self.ipfix_src_port)
-
-        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-             TCP(sport=3025))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(0)
-        self.vapi.cli("ipfix flush")  # FIXME this should be an API call
-        capture = self.pg3.get_capture(3)
-        ipfix = IPFIXDecoder()
-        # first load template
-        for p in capture:
-            self.assertTrue(p.haslayer(IPFIX))
-            self.assertEqual(p[IP].src, self.pg3.local_ip4)
-            self.assertEqual(p[IP].dst, self.pg3.remote_ip4)
-            self.assertEqual(p[UDP].sport, self.ipfix_src_port)
-            self.assertEqual(p[UDP].dport, 4739)
-            self.assertEqual(p[IPFIX].observationDomainID,
-                             self.ipfix_domain_id)
-            if p.haslayer(Template):
-                ipfix.add_template(p.getlayer(Template))
-        # verify events in data set
-        for p in capture:
-            if p.haslayer(Data):
-                data = ipfix.decode_data_set(p.getlayer(Set))
-                self.verify_ipfix_addr_exhausted(data)
-
-    def test_pool_addr_fib(self):
-        """ S-NAT add pool addresses to FIB """
-        static_addr = '10.0.0.10'
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, static_addr)
-
-        # SNAT address
-        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
-             ARP(op=ARP.who_has, pdst=self.snat_addr,
-                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(1)
-        self.assertTrue(capture[0].haslayer(ARP))
-        self.assertTrue(capture[0][ARP].op, ARP.is_at)
-
-        # 1:1 NAT address
-        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
-             ARP(op=ARP.who_has, pdst=static_addr,
-                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(1)
-        self.assertTrue(capture[0].haslayer(ARP))
-        self.assertTrue(capture[0][ARP].op, ARP.is_at)
-
-        # send ARP to non-SNAT interface
-        p = (Ether(src=self.pg2.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
-             ARP(op=ARP.who_has, pdst=self.snat_addr,
-                 psrc=self.pg2.remote_ip4, hwsrc=self.pg2.remote_mac))
-        self.pg2.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(0)
-
-        # remove addresses and verify
-        self.snat_add_address(self.snat_addr, is_add=0)
-        self.snat_add_static_mapping(self.pg0.remote_ip4, static_addr,
-                                     is_add=0)
-
-        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
-             ARP(op=ARP.who_has, pdst=self.snat_addr,
-                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(0)
-
-        p = (Ether(src=self.pg1.remote_mac, dst='ff:ff:ff:ff:ff:ff') /
-             ARP(op=ARP.who_has, pdst=static_addr,
-                 psrc=self.pg1.remote_ip4, hwsrc=self.pg1.remote_mac))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(0)
-
-    def test_vrf_mode(self):
-        """ S-NAT tenant VRF aware address pool mode """
-
-        vrf_id1 = 1
-        vrf_id2 = 2
-        nat_ip1 = "10.0.0.10"
-        nat_ip2 = "10.0.0.11"
-
-        self.pg0.unconfig_ip4()
-        self.pg1.unconfig_ip4()
-        self.pg0.set_table_ip4(vrf_id1)
-        self.pg1.set_table_ip4(vrf_id2)
-        self.pg0.config_ip4()
-        self.pg1.config_ip4()
-
-        self.snat_add_address(nat_ip1, vrf_id=vrf_id1)
-        self.snat_add_address(nat_ip2, vrf_id=vrf_id2)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg2.sw_if_index,
-                                                 is_inside=0)
-
-        # first VRF
-        pkts = self.create_stream_in(self.pg0, self.pg2)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip1)
-
-        # second VRF
-        pkts = self.create_stream_in(self.pg1, self.pg2)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip2)
-
-    def test_vrf_feature_independent(self):
-        """ S-NAT tenant VRF independent address pool mode """
-
-        nat_ip1 = "10.0.0.10"
-        nat_ip2 = "10.0.0.11"
-
-        self.snat_add_address(nat_ip1)
-        self.snat_add_address(nat_ip2)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg2.sw_if_index,
-                                                 is_inside=0)
-
-        # first VRF
-        pkts = self.create_stream_in(self.pg0, self.pg2)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip1)
-
-        # second VRF
-        pkts = self.create_stream_in(self.pg1, self.pg2)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip1)
-
-    def test_dynamic_ipless_interfaces(self):
-        """ SNAT interfaces without configured ip dynamic map """
-
-        self.vapi.ip_neighbor_add_del(self.pg7.sw_if_index,
-                                      self.pg7.remote_mac,
-                                      self.pg7.remote_ip4n,
-                                      is_static=1)
-        self.vapi.ip_neighbor_add_del(self.pg8.sw_if_index,
-                                      self.pg8.remote_mac,
-                                      self.pg8.remote_ip4n,
-                                      is_static=1)
-
-        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg7.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg7.sw_if_index)
-        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg8.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg8.sw_if_index)
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg7.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg8.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg7, self.pg8)
-        self.pg7.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg8.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg8, self.snat_addr)
-        self.pg8.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg7.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg7)
-
-    def test_static_ipless_interfaces(self):
-        """ SNAT 1:1 NAT interfaces without configured ip """
-
-        self.vapi.ip_neighbor_add_del(self.pg7.sw_if_index,
-                                      self.pg7.remote_mac,
-                                      self.pg7.remote_ip4n,
-                                      is_static=1)
-        self.vapi.ip_neighbor_add_del(self.pg8.sw_if_index,
-                                      self.pg8.remote_mac,
-                                      self.pg8.remote_ip4n,
-                                      is_static=1)
-
-        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg7.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg7.sw_if_index)
-        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg8.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg8.sw_if_index)
-
-        self.snat_add_static_mapping(self.pg7.remote_ip4, self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg7.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg8.sw_if_index,
-                                                 is_inside=0)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg8)
-        self.pg8.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg7.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg7)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg7, self.pg8)
-        self.pg7.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg8.get_capture(len(pkts))
-        self.verify_capture_out(capture, self.snat_addr, True)
-
-    def test_static_with_port_ipless_interfaces(self):
-        """ SNAT 1:1 NAT with port interfaces without configured ip """
-
-        self.tcp_port_out = 30606
-        self.udp_port_out = 30607
-        self.icmp_id_out = 30608
-
-        self.vapi.ip_neighbor_add_del(self.pg7.sw_if_index,
-                                      self.pg7.remote_mac,
-                                      self.pg7.remote_ip4n,
-                                      is_static=1)
-        self.vapi.ip_neighbor_add_del(self.pg8.sw_if_index,
-                                      self.pg8.remote_mac,
-                                      self.pg8.remote_ip4n,
-                                      is_static=1)
-
-        self.vapi.ip_add_del_route(dst_address=self.pg7.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg7.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg7.sw_if_index)
-        self.vapi.ip_add_del_route(dst_address=self.pg8.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg8.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg8.sw_if_index)
-
-        self.snat_add_address(self.snat_addr)
-        self.snat_add_static_mapping(self.pg7.remote_ip4, self.snat_addr,
-                                     self.tcp_port_in, self.tcp_port_out,
-                                     proto=IP_PROTOS.tcp)
-        self.snat_add_static_mapping(self.pg7.remote_ip4, self.snat_addr,
-                                     self.udp_port_in, self.udp_port_out,
-                                     proto=IP_PROTOS.udp)
-        self.snat_add_static_mapping(self.pg7.remote_ip4, self.snat_addr,
-                                     self.icmp_id_in, self.icmp_id_out,
-                                     proto=IP_PROTOS.icmp)
-        self.vapi.snat_interface_add_del_feature(self.pg7.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg8.sw_if_index,
-                                                 is_inside=0)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg8)
-        self.pg8.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg7.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg7)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg7, self.pg8)
-        self.pg7.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg8.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-    def test_static_unknown_proto(self):
-        """ 1:1 NAT translate packet with unknown protocol """
-        nat_ip = "10.0.0.10"
-        self.snat_add_static_mapping(self.pg0.remote_ip4, nat_ip)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-             GRE() /
-             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg1.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, nat_ip)
-            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-        # out2in
-        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-             IP(src=self.pg1.remote_ip4, dst=nat_ip) /
-             GRE() /
-             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, self.pg1.remote_ip4)
-            self.assertEqual(packet[IP].dst, self.pg0.remote_ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-    def test_hairpinning_static_unknown_proto(self):
-        """ 1:1 NAT translate packet with unknown protocol - hairpinning """
-
-        host = self.pg0.remote_hosts[0]
-        server = self.pg0.remote_hosts[1]
-
-        host_nat_ip = "10.0.0.10"
-        server_nat_ip = "10.0.0.11"
-
-        self.snat_add_static_mapping(host.ip4, host_nat_ip)
-        self.snat_add_static_mapping(server.ip4, server_nat_ip)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # host to server
-        p = (Ether(dst=self.pg0.local_mac, src=host.mac) /
-             IP(src=host.ip4, dst=server_nat_ip) /
-             GRE() /
-             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, host_nat_ip)
-            self.assertEqual(packet[IP].dst, server.ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-        # server to host
-        p = (Ether(dst=self.pg0.local_mac, src=server.mac) /
-             IP(src=server.ip4, dst=host_nat_ip) /
-             GRE() /
-             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, server_nat_ip)
-            self.assertEqual(packet[IP].dst, host.ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-    def test_unknown_proto(self):
-        """ SNAT translate packet with unknown protocol """
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-             TCP(sport=self.tcp_port_in, dport=20))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg1.get_capture(1)
-
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-             GRE() /
-             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg1.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, self.snat_addr)
-            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-        # out2in
-        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-             IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
-             GRE() /
-             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, self.pg1.remote_ip4)
-            self.assertEqual(packet[IP].dst, self.pg0.remote_ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-    def test_hairpinning_unknown_proto(self):
-        """ SNAT translate packet with unknown protocol - hairpinning """
-        host = self.pg0.remote_hosts[0]
-        server = self.pg0.remote_hosts[1]
-        host_in_port = 1234
-        host_out_port = 0
-        server_in_port = 5678
-        server_out_port = 8765
-        server_nat_ip = "10.0.0.11"
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # add static mapping for server
-        self.snat_add_static_mapping(server.ip4, server_nat_ip)
-
-        # host to server
-        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
-             IP(src=host.ip4, dst=server_nat_ip) /
-             TCP(sport=host_in_port, dport=server_out_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-
-        p = (Ether(dst=self.pg0.local_mac, src=host.mac) /
-             IP(src=host.ip4, dst=server_nat_ip) /
-             GRE() /
-             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, self.snat_addr)
-            self.assertEqual(packet[IP].dst, server.ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-        # server to host
-        p = (Ether(dst=self.pg0.local_mac, src=server.mac) /
-             IP(src=server.ip4, dst=self.snat_addr) /
-             GRE() /
-             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, server_nat_ip)
-            self.assertEqual(packet[IP].dst, host.ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-    def test_output_feature(self):
-        """ S-NAT interface output feature (in2out postrouting) """
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_output_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_output_feature(self.pg1.sw_if_index,
-                                                        is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-    def test_output_feature_vrf_aware(self):
-        """ S-NAT interface output feature VRF aware (in2out postrouting) """
-        nat_ip_vrf10 = "10.0.0.10"
-        nat_ip_vrf20 = "10.0.0.20"
-
-        self.vapi.ip_add_del_route(dst_address=self.pg3.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg3.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg3.sw_if_index,
-                                   table_id=10)
-        self.vapi.ip_add_del_route(dst_address=self.pg3.remote_ip4n,
-                                   dst_address_length=32,
-                                   next_hop_address=self.pg3.remote_ip4n,
-                                   next_hop_sw_if_index=self.pg3.sw_if_index,
-                                   table_id=20)
-
-        self.snat_add_address(nat_ip_vrf10, vrf_id=10)
-        self.snat_add_address(nat_ip_vrf20, vrf_id=20)
-        self.vapi.snat_interface_add_del_output_feature(self.pg4.sw_if_index)
-        self.vapi.snat_interface_add_del_output_feature(self.pg6.sw_if_index)
-        self.vapi.snat_interface_add_del_output_feature(self.pg3.sw_if_index,
-                                                        is_inside=0)
-
-        # in2out VRF 10
-        pkts = self.create_stream_in(self.pg4, self.pg3)
-        self.pg4.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=nat_ip_vrf10)
-
-        # out2in VRF 10
-        pkts = self.create_stream_out(self.pg3, dst_ip=nat_ip_vrf10)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg4.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg4)
-
-        # in2out VRF 20
-        pkts = self.create_stream_in(self.pg6, self.pg3)
-        self.pg6.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg3.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=nat_ip_vrf20)
-
-        # out2in VRF 20
-        pkts = self.create_stream_out(self.pg3, dst_ip=nat_ip_vrf20)
-        self.pg3.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg6.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg6)
-
-    def test_output_feature_hairpinning(self):
-        """ S-NAT interface output feature hairpinning (in2out postrouting) """
-        host = self.pg0.remote_hosts[0]
-        server = self.pg0.remote_hosts[1]
-        host_in_port = 1234
-        host_out_port = 0
-        server_in_port = 5678
-        server_out_port = 8765
-
-        self.snat_add_address(self.snat_addr)
-        self.vapi.snat_interface_add_del_output_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_output_feature(self.pg1.sw_if_index,
-                                                        is_inside=0)
-
-        # add static mapping for server
-        self.snat_add_static_mapping(server.ip4, self.snat_addr,
-                                     server_in_port, server_out_port,
-                                     proto=IP_PROTOS.tcp)
-
-        # send packet from host to server
-        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
-             IP(src=host.ip4, dst=self.snat_addr) /
-             TCP(sport=host_in_port, dport=server_out_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.snat_addr)
-            self.assertEqual(ip.dst, server.ip4)
-            self.assertNotEqual(tcp.sport, host_in_port)
-            self.assertEqual(tcp.dport, server_in_port)
-            self.check_tcp_checksum(p)
-            host_out_port = tcp.sport
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", p))
-            raise
-
-        # send reply from server to host
-        p = (Ether(src=server.mac, dst=self.pg0.local_mac) /
-             IP(src=server.ip4, dst=self.snat_addr) /
-             TCP(sport=server_in_port, dport=host_out_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.snat_addr)
-            self.assertEqual(ip.dst, host.ip4)
-            self.assertEqual(tcp.sport, server_out_port)
-            self.assertEqual(tcp.dport, host_in_port)
-            self.check_tcp_checksum(p)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:"), p)
-            raise
-
-    def tearDown(self):
-        super(TestSNAT, self).tearDown()
-        if not self.vpp_dead:
-            self.logger.info(self.vapi.cli("show snat verbose"))
-            self.clear_snat()
-
-
-class TestDeterministicNAT(MethodHolder):
-    """ Deterministic NAT Test Cases """
-
-    @classmethod
-    def setUpConstants(cls):
-        super(TestDeterministicNAT, cls).setUpConstants()
-        cls.vpp_cmdline.extend(["snat", "{", "deterministic", "}"])
-
-    @classmethod
-    def setUpClass(cls):
-        super(TestDeterministicNAT, cls).setUpClass()
-
-        try:
-            cls.tcp_port_in = 6303
-            cls.tcp_external_port = 6303
-            cls.udp_port_in = 6304
-            cls.udp_external_port = 6304
-            cls.icmp_id_in = 6305
-            cls.snat_addr = '10.0.0.3'
-
-            cls.create_pg_interfaces(range(3))
-            cls.interfaces = list(cls.pg_interfaces)
-
-            for i in cls.interfaces:
-                i.admin_up()
-                i.config_ip4()
-                i.resolve_arp()
-
-            cls.pg0.generate_remote_hosts(2)
-            cls.pg0.configure_ipv4_neighbors()
-
-        except Exception:
-            super(TestDeterministicNAT, cls).tearDownClass()
-            raise
-
-    def create_stream_in(self, in_if, out_if, ttl=64):
-        """
-        Create packet stream for inside network
-
-        :param in_if: Inside interface
-        :param out_if: Outside interface
-        :param ttl: TTL of generated packets
-        """
-        pkts = []
-        # TCP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
-             TCP(sport=self.tcp_port_in, dport=self.tcp_external_port))
-        pkts.append(p)
-
-        # UDP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
-             UDP(sport=self.udp_port_in, dport=self.udp_external_port))
-        pkts.append(p)
-
-        # ICMP
-        p = (Ether(dst=in_if.local_mac, src=in_if.remote_mac) /
-             IP(src=in_if.remote_ip4, dst=out_if.remote_ip4, ttl=ttl) /
-             ICMP(id=self.icmp_id_in, type='echo-request'))
-        pkts.append(p)
-
-        return pkts
-
-    def create_stream_out(self, out_if, dst_ip=None, ttl=64):
-        """
-        Create packet stream for outside network
-
-        :param out_if: Outside interface
-        :param dst_ip: Destination IP address (Default use global SNAT address)
-        :param ttl: TTL of generated packets
-        """
-        if dst_ip is None:
-            dst_ip = self.snat_addr
-        pkts = []
-        # TCP
-        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
-             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
-             TCP(dport=self.tcp_port_out, sport=self.tcp_external_port))
-        pkts.append(p)
-
-        # UDP
-        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
-             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
-             UDP(dport=self.udp_port_out, sport=self.udp_external_port))
-        pkts.append(p)
-
-        # ICMP
-        p = (Ether(dst=out_if.local_mac, src=out_if.remote_mac) /
-             IP(src=out_if.remote_ip4, dst=dst_ip, ttl=ttl) /
-             ICMP(id=self.icmp_external_id, type='echo-reply'))
-        pkts.append(p)
-
-        return pkts
-
-    def verify_capture_out(self, capture, nat_ip=None, packet_num=3):
-        """
-        Verify captured packets on outside network
-
-        :param capture: Captured packets
-        :param nat_ip: Translated IP address (Default use global SNAT address)
-        :param same_port: Sorce port number is not translated (Default False)
-        :param packet_num: Expected number of packets (Default 3)
-        """
-        if nat_ip is None:
-            nat_ip = self.snat_addr
-        self.assertEqual(packet_num, len(capture))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, nat_ip)
-                if packet.haslayer(TCP):
-                    self.tcp_port_out = packet[TCP].sport
-                elif packet.haslayer(UDP):
-                    self.udp_port_out = packet[UDP].sport
-                else:
-                    self.icmp_external_id = packet[ICMP].id
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet "
-                                      "(outside network):", packet))
-                raise
-
-    def initiate_tcp_session(self, in_if, out_if):
-        """
-        Initiates TCP session
-
-        :param in_if: Inside interface
-        :param out_if: Outside interface
-        """
-        try:
-            # SYN packet in->out
-            p = (Ether(src=in_if.remote_mac, dst=in_if.local_mac) /
-                 IP(src=in_if.remote_ip4, dst=out_if.remote_ip4) /
-                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
-                     flags="S"))
-            in_if.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            capture = out_if.get_capture(1)
-            p = capture[0]
-            self.tcp_port_out = p[TCP].sport
-
-            # SYN + ACK packet out->in
-            p = (Ether(src=out_if.remote_mac, dst=out_if.local_mac) /
-                 IP(src=out_if.remote_ip4, dst=self.snat_addr) /
-                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
-                     flags="SA"))
-            out_if.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            in_if.get_capture(1)
-
-            # ACK packet in->out
-            p = (Ether(src=in_if.remote_mac, dst=in_if.local_mac) /
-                 IP(src=in_if.remote_ip4, dst=out_if.remote_ip4) /
-                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
-                     flags="A"))
-            in_if.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            out_if.get_capture(1)
-
-        except:
-            self.logger.error("TCP 3 way handshake failed")
-            raise
-
-    def verify_ipfix_max_entries_per_user(self, data):
-        """
-        Verify IPFIX maximum entries per user exceeded event
-
-        :param data: Decoded IPFIX data records
-        """
-        self.assertEqual(1, len(data))
-        record = data[0]
-        # natEvent
-        self.assertEqual(ord(record[230]), 13)
-        # natQuotaExceededEvent
-        self.assertEqual('\x03\x00\x00\x00', record[466])
-        # sourceIPv4Address
-        self.assertEqual(self.pg0.remote_ip4n, record[8])
-
-    def test_deterministic_mode(self):
-        """ S-NAT run deterministic mode """
-        in_addr = '172.16.255.0'
-        out_addr = '172.17.255.50'
-        in_addr_t = '172.16.255.20'
-        in_addr_n = socket.inet_aton(in_addr)
-        out_addr_n = socket.inet_aton(out_addr)
-        in_addr_t_n = socket.inet_aton(in_addr_t)
-        in_plen = 24
-        out_plen = 32
-
-        snat_config = self.vapi.snat_show_config()
-        self.assertEqual(1, snat_config.deterministic)
-
-        self.vapi.snat_add_det_map(in_addr_n, in_plen, out_addr_n, out_plen)
-
-        rep1 = self.vapi.snat_det_forward(in_addr_t_n)
-        self.assertEqual(rep1.out_addr[:4], out_addr_n)
-        rep2 = self.vapi.snat_det_reverse(out_addr_n, rep1.out_port_hi)
-        self.assertEqual(rep2.in_addr[:4], in_addr_t_n)
-
-        deterministic_mappings = self.vapi.snat_det_map_dump()
-        self.assertEqual(len(deterministic_mappings), 1)
-        dsm = deterministic_mappings[0]
-        self.assertEqual(in_addr_n, dsm.in_addr[:4])
-        self.assertEqual(in_plen, dsm.in_plen)
-        self.assertEqual(out_addr_n, dsm.out_addr[:4])
-        self.assertEqual(out_plen, dsm.out_plen)
-
-        self.clear_snat()
-        deterministic_mappings = self.vapi.snat_det_map_dump()
-        self.assertEqual(len(deterministic_mappings), 0)
-
-    def test_set_timeouts(self):
-        """ Set deterministic NAT timeouts """
-        timeouts_before = self.vapi.snat_det_get_timeouts()
-
-        self.vapi.snat_det_set_timeouts(timeouts_before.udp + 10,
-                                        timeouts_before.tcp_established + 10,
-                                        timeouts_before.tcp_transitory + 10,
-                                        timeouts_before.icmp + 10)
-
-        timeouts_after = self.vapi.snat_det_get_timeouts()
-
-        self.assertNotEqual(timeouts_before.udp, timeouts_after.udp)
-        self.assertNotEqual(timeouts_before.icmp, timeouts_after.icmp)
-        self.assertNotEqual(timeouts_before.tcp_established,
-                            timeouts_after.tcp_established)
-        self.assertNotEqual(timeouts_before.tcp_transitory,
-                            timeouts_after.tcp_transitory)
-
-    def test_det_in(self):
-        """ CGNAT translation test (TCP, UDP, ICMP) """
-
-        nat_ip = "10.0.0.10"
-
-        self.vapi.snat_add_det_map(self.pg0.remote_ip4n,
-                                   32,
-                                   socket.inet_aton(nat_ip),
-                                   32)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1, nat_ip)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in(capture, self.pg0)
-
-        # session dump test
-        sessions = self.vapi.snat_det_session_dump(self.pg0.remote_ip4n)
-        self.assertEqual(len(sessions), 3)
-
-        # TCP session
-        s = sessions[0]
-        self.assertEqual(s.ext_addr[:4], self.pg1.remote_ip4n)
-        self.assertEqual(s.in_port, self.tcp_port_in)
-        self.assertEqual(s.out_port, self.tcp_port_out)
-        self.assertEqual(s.ext_port, self.tcp_external_port)
-
-        # UDP session
-        s = sessions[1]
-        self.assertEqual(s.ext_addr[:4], self.pg1.remote_ip4n)
-        self.assertEqual(s.in_port, self.udp_port_in)
-        self.assertEqual(s.out_port, self.udp_port_out)
-        self.assertEqual(s.ext_port, self.udp_external_port)
-
-        # ICMP session
-        s = sessions[2]
-        self.assertEqual(s.ext_addr[:4], self.pg1.remote_ip4n)
-        self.assertEqual(s.in_port, self.icmp_id_in)
-        self.assertEqual(s.out_port, self.icmp_external_id)
-
-    def test_multiple_users(self):
-        """ CGNAT multiple users """
-
-        nat_ip = "10.0.0.10"
-        port_in = 80
-        external_port = 6303
-
-        host0 = self.pg0.remote_hosts[0]
-        host1 = self.pg0.remote_hosts[1]
-
-        self.vapi.snat_add_det_map(host0.ip4n,
-                                   24,
-                                   socket.inet_aton(nat_ip),
-                                   32)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        # host0 to out
-        p = (Ether(src=host0.mac, dst=self.pg0.local_mac) /
-             IP(src=host0.ip4, dst=self.pg1.remote_ip4) /
-             TCP(sport=port_in, dport=external_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, nat_ip)
-            self.assertEqual(ip.dst, self.pg1.remote_ip4)
-            self.assertEqual(tcp.dport, external_port)
-            port_out0 = tcp.sport
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", p))
-            raise
-
-        # host1 to out
-        p = (Ether(src=host1.mac, dst=self.pg0.local_mac) /
-             IP(src=host1.ip4, dst=self.pg1.remote_ip4) /
-             TCP(sport=port_in, dport=external_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, nat_ip)
-            self.assertEqual(ip.dst, self.pg1.remote_ip4)
-            self.assertEqual(tcp.dport, external_port)
-            port_out1 = tcp.sport
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", p))
-            raise
-
-        dms = self.vapi.snat_det_map_dump()
-        self.assertEqual(1, len(dms))
-        self.assertEqual(2, dms[0].ses_num)
-
-        # out to host0
-        p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
-             IP(src=self.pg1.remote_ip4, dst=nat_ip) /
-             TCP(sport=external_port, dport=port_out0))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.pg1.remote_ip4)
-            self.assertEqual(ip.dst, host0.ip4)
-            self.assertEqual(tcp.dport, port_in)
-            self.assertEqual(tcp.sport, external_port)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", p))
-            raise
-
-        # out to host1
-        p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
-             IP(src=self.pg1.remote_ip4, dst=nat_ip) /
-             TCP(sport=external_port, dport=port_out1))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        try:
-            ip = p[IP]
-            tcp = p[TCP]
-            self.assertEqual(ip.src, self.pg1.remote_ip4)
-            self.assertEqual(ip.dst, host1.ip4)
-            self.assertEqual(tcp.dport, port_in)
-            self.assertEqual(tcp.sport, external_port)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet", p))
-            raise
-
-        # session close api test
-        self.vapi.snat_det_close_session_out(socket.inet_aton(nat_ip),
-                                             port_out1,
-                                             self.pg1.remote_ip4n,
-                                             external_port)
-        dms = self.vapi.snat_det_map_dump()
-        self.assertEqual(dms[0].ses_num, 1)
-
-        self.vapi.snat_det_close_session_in(host0.ip4n,
-                                            port_in,
-                                            self.pg1.remote_ip4n,
-                                            external_port)
-        dms = self.vapi.snat_det_map_dump()
-        self.assertEqual(dms[0].ses_num, 0)
-
-    def test_tcp_session_close_detection_in(self):
-        """ CGNAT TCP session close initiated from inside network """
-        self.vapi.snat_add_det_map(self.pg0.remote_ip4n,
-                                   32,
-                                   socket.inet_aton(self.snat_addr),
-                                   32)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        self.initiate_tcp_session(self.pg0, self.pg1)
-
-        # close the session from inside
-        try:
-            # FIN packet in -> out
-            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
-                     flags="F"))
-            self.pg0.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            self.pg1.get_capture(1)
-
-            pkts = []
-
-            # ACK packet out -> in
-            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
-                 IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
-                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
-                     flags="A"))
-            pkts.append(p)
-
-            # FIN packet out -> in
-            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
-                 IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
-                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
-                     flags="F"))
-            pkts.append(p)
-
-            self.pg1.add_stream(pkts)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            self.pg0.get_capture(2)
-
-            # ACK packet in -> out
-            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
-                     flags="A"))
-            self.pg0.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            self.pg1.get_capture(1)
-
-            # Check if snat closed the session
-            dms = self.vapi.snat_det_map_dump()
-            self.assertEqual(0, dms[0].ses_num)
-        except:
-            self.logger.error("TCP session termination failed")
-            raise
-
-    def test_tcp_session_close_detection_out(self):
-        """ CGNAT TCP session close initiated from outside network """
-        self.vapi.snat_add_det_map(self.pg0.remote_ip4n,
-                                   32,
-                                   socket.inet_aton(self.snat_addr),
-                                   32)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        self.initiate_tcp_session(self.pg0, self.pg1)
-
-        # close the session from outside
-        try:
-            # FIN packet out -> in
-            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
-                 IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
-                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
-                     flags="F"))
-            self.pg1.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            self.pg0.get_capture(1)
-
-            pkts = []
-
-            # ACK packet in -> out
-            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
-                     flags="A"))
-            pkts.append(p)
-
-            # ACK packet in -> out
-            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                 TCP(sport=self.tcp_port_in, dport=self.tcp_external_port,
-                     flags="F"))
-            pkts.append(p)
-
-            self.pg0.add_stream(pkts)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            self.pg1.get_capture(2)
-
-            # ACK packet out -> in
-            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
-                 IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
-                 TCP(sport=self.tcp_external_port, dport=self.tcp_port_out,
-                     flags="A"))
-            self.pg1.add_stream(p)
-            self.pg_enable_capture(self.pg_interfaces)
-            self.pg_start()
-            self.pg0.get_capture(1)
-
-            # Check if snat closed the session
-            dms = self.vapi.snat_det_map_dump()
-            self.assertEqual(0, dms[0].ses_num)
-        except:
-            self.logger.error("TCP session termination failed")
-            raise
-
-    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
-    def test_session_timeout(self):
-        """ CGNAT session timeouts """
-        self.vapi.snat_add_det_map(self.pg0.remote_ip4n,
-                                   32,
-                                   socket.inet_aton(self.snat_addr),
-                                   32)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-
-        self.initiate_tcp_session(self.pg0, self.pg1)
-        self.vapi.snat_det_set_timeouts(5, 5, 5, 5)
-        pkts = self.create_stream_in(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        sleep(15)
-
-        dms = self.vapi.snat_det_map_dump()
-        self.assertEqual(0, dms[0].ses_num)
-
-    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
-    def test_session_limit_per_user(self):
-        """ CGNAT maximum 1000 sessions per user should be created """
-        self.vapi.snat_add_det_map(self.pg0.remote_ip4n,
-                                   32,
-                                   socket.inet_aton(self.snat_addr),
-                                   32)
-        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
-        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
-                                                 is_inside=0)
-        self.vapi.set_ipfix_exporter(collector_address=self.pg2.remote_ip4n,
-                                     src_address=self.pg2.local_ip4n,
-                                     path_mtu=512,
-                                     template_interval=10)
-        self.vapi.snat_ipfix()
-
-        pkts = []
-        for port in range(1025, 2025):
-            p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-                 IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-                 UDP(sport=port, dport=port))
-            pkts.append(p)
-
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-
-        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
-             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
-             UDP(sport=3001, dport=3002))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.assert_nothing_captured()
-
-        # verify ICMP error packet
-        capture = self.pg0.get_capture(1)
-        p = capture[0]
-        self.assertTrue(p.haslayer(ICMP))
-        icmp = p[ICMP]
-        self.assertEqual(icmp.type, 3)
-        self.assertEqual(icmp.code, 1)
-        self.assertTrue(icmp.haslayer(IPerror))
-        inner_ip = icmp[IPerror]
-        self.assertEqual(inner_ip[UDPerror].sport, 3001)
-        self.assertEqual(inner_ip[UDPerror].dport, 3002)
-
-        dms = self.vapi.snat_det_map_dump()
-
-        self.assertEqual(1000, dms[0].ses_num)
-
-        # verify IPFIX logging
-        self.vapi.cli("ipfix flush")  # FIXME this should be an API call
-        sleep(1)
-        capture = self.pg2.get_capture(2)
-        ipfix = IPFIXDecoder()
-        # first load template
-        for p in capture:
-            self.assertTrue(p.haslayer(IPFIX))
-            if p.haslayer(Template):
-                ipfix.add_template(p.getlayer(Template))
-        # verify events in data set
-        for p in capture:
-            if p.haslayer(Data):
-                data = ipfix.decode_data_set(p.getlayer(Set))
-                self.verify_ipfix_max_entries_per_user(data)
-
-    def clear_snat(self):
-        """
-        Clear SNAT configuration.
-        """
-        self.vapi.snat_ipfix(enable=0)
-        self.vapi.snat_det_set_timeouts()
-        deterministic_mappings = self.vapi.snat_det_map_dump()
-        for dsm in deterministic_mappings:
-            self.vapi.snat_add_det_map(dsm.in_addr,
-                                       dsm.in_plen,
-                                       dsm.out_addr,
-                                       dsm.out_plen,
-                                       is_add=0)
-
-        interfaces = self.vapi.snat_interface_dump()
-        for intf in interfaces:
-            self.vapi.snat_interface_add_del_feature(intf.sw_if_index,
-                                                     intf.is_inside,
-                                                     is_add=0)
-
-    def tearDown(self):
-        super(TestDeterministicNAT, self).tearDown()
-        if not self.vpp_dead:
-            self.logger.info(self.vapi.cli("show snat detail"))
-            self.clear_snat()
-
-
-class TestNAT64(MethodHolder):
-    """ NAT64 Test Cases """
-
-    @classmethod
-    def setUpClass(cls):
-        super(TestNAT64, cls).setUpClass()
-
-        try:
-            cls.tcp_port_in = 6303
-            cls.tcp_port_out = 6303
-            cls.udp_port_in = 6304
-            cls.udp_port_out = 6304
-            cls.icmp_id_in = 6305
-            cls.icmp_id_out = 6305
-            cls.nat_addr = '10.0.0.3'
-            cls.nat_addr_n = socket.inet_pton(socket.AF_INET, cls.nat_addr)
-            cls.vrf1_id = 10
-            cls.vrf1_nat_addr = '10.0.10.3'
-            cls.vrf1_nat_addr_n = socket.inet_pton(socket.AF_INET,
-                                                   cls.vrf1_nat_addr)
-
-            cls.create_pg_interfaces(range(3))
-            cls.ip6_interfaces = list(cls.pg_interfaces[0:1])
-            cls.ip6_interfaces.append(cls.pg_interfaces[2])
-            cls.ip4_interfaces = list(cls.pg_interfaces[1:2])
-
-            cls.pg_interfaces[2].set_table_ip6(cls.vrf1_id)
-
-            cls.pg0.generate_remote_hosts(2)
-
-            for i in cls.ip6_interfaces:
-                i.admin_up()
-                i.config_ip6()
-                i.configure_ipv6_neighbors()
-
-            for i in cls.ip4_interfaces:
-                i.admin_up()
-                i.config_ip4()
-                i.resolve_arp()
-
-        except Exception:
-            super(TestNAT64, cls).tearDownClass()
-            raise
-
-    def test_pool(self):
-        """ Add/delete address to NAT64 pool """
-        nat_addr = socket.inet_pton(socket.AF_INET, '1.2.3.4')
-
-        self.vapi.nat64_add_del_pool_addr_range(nat_addr, nat_addr)
-
-        addresses = self.vapi.nat64_pool_addr_dump()
-        self.assertEqual(len(addresses), 1)
-        self.assertEqual(addresses[0].address, nat_addr)
-
-        self.vapi.nat64_add_del_pool_addr_range(nat_addr, nat_addr, is_add=0)
-
-        addresses = self.vapi.nat64_pool_addr_dump()
-        self.assertEqual(len(addresses), 0)
-
-    def test_interface(self):
-        """ Enable/disable NAT64 feature on the interface """
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-
-        interfaces = self.vapi.nat64_interface_dump()
-        self.assertEqual(len(interfaces), 2)
-        pg0_found = False
-        pg1_found = False
-        for intf in interfaces:
-            if intf.sw_if_index == self.pg0.sw_if_index:
-                self.assertEqual(intf.is_inside, 1)
-                pg0_found = True
-            elif intf.sw_if_index == self.pg1.sw_if_index:
-                self.assertEqual(intf.is_inside, 0)
-                pg1_found = True
-        self.assertTrue(pg0_found)
-        self.assertTrue(pg1_found)
-
-        features = self.vapi.cli("show interface features pg0")
-        self.assertNotEqual(features.find('nat64-in2out'), -1)
-        features = self.vapi.cli("show interface features pg1")
-        self.assertNotEqual(features.find('nat64-out2in'), -1)
-
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index, is_add=0)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_add=0)
-
-        interfaces = self.vapi.nat64_interface_dump()
-        self.assertEqual(len(interfaces), 0)
-
-    def test_static_bib(self):
-        """ Add/delete static BIB entry """
-        in_addr = socket.inet_pton(socket.AF_INET6,
-                                   '2001:db8:85a3::8a2e:370:7334')
-        out_addr = socket.inet_pton(socket.AF_INET, '10.1.1.3')
-        in_port = 1234
-        out_port = 5678
-        proto = IP_PROTOS.tcp
-
-        self.vapi.nat64_add_del_static_bib(in_addr,
-                                           out_addr,
-                                           in_port,
-                                           out_port,
-                                           proto)
-        bib = self.vapi.nat64_bib_dump(IP_PROTOS.tcp)
-        static_bib_num = 0
-        for bibe in bib:
-            if bibe.is_static:
-                static_bib_num += 1
-                self.assertEqual(bibe.i_addr, in_addr)
-                self.assertEqual(bibe.o_addr, out_addr)
-                self.assertEqual(bibe.i_port, in_port)
-                self.assertEqual(bibe.o_port, out_port)
-        self.assertEqual(static_bib_num, 1)
-
-        self.vapi.nat64_add_del_static_bib(in_addr,
-                                           out_addr,
-                                           in_port,
-                                           out_port,
-                                           proto,
-                                           is_add=0)
-        bib = self.vapi.nat64_bib_dump(IP_PROTOS.tcp)
-        static_bib_num = 0
-        for bibe in bib:
-            if bibe.is_static:
-                static_bib_num += 1
-        self.assertEqual(static_bib_num, 0)
-
-    def test_set_timeouts(self):
-        """ Set NAT64 timeouts """
-        # verify default values
-        timeouts = self.vapi.nat64_get_timeouts()
-        self.assertEqual(timeouts.udp, 300)
-        self.assertEqual(timeouts.icmp, 60)
-        self.assertEqual(timeouts.tcp_trans, 240)
-        self.assertEqual(timeouts.tcp_est, 7440)
-        self.assertEqual(timeouts.tcp_incoming_syn, 6)
-
-        # set and verify custom values
-        self.vapi.nat64_set_timeouts(udp=200, icmp=30, tcp_trans=250,
-                                     tcp_est=7450, tcp_incoming_syn=10)
-        timeouts = self.vapi.nat64_get_timeouts()
-        self.assertEqual(timeouts.udp, 200)
-        self.assertEqual(timeouts.icmp, 30)
-        self.assertEqual(timeouts.tcp_trans, 250)
-        self.assertEqual(timeouts.tcp_est, 7450)
-        self.assertEqual(timeouts.tcp_incoming_syn, 10)
-
-    def test_dynamic(self):
-        """ NAT64 dynamic translation test """
-        self.tcp_port_in = 6303
-        self.udp_port_in = 6304
-        self.icmp_id_in = 6305
-
-        ses_num_start = self.nat64_get_ses_num()
-
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-
-        # in2out
-        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=self.nat_addr,
-                                dst_ip=self.pg1.remote_ip4)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        ip = IPv6(src=''.join(['64:ff9b::', self.pg1.remote_ip4]))
-        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg0.remote_ip6)
-
-        # in2out
-        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=self.nat_addr,
-                                dst_ip=self.pg1.remote_ip4)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg0.remote_ip6)
-
-        ses_num_end = self.nat64_get_ses_num()
-
-        self.assertEqual(ses_num_end - ses_num_start, 3)
-
-        # tenant with specific VRF
-        self.vapi.nat64_add_del_pool_addr_range(self.vrf1_nat_addr_n,
-                                                self.vrf1_nat_addr_n,
-                                                vrf_id=self.vrf1_id)
-        self.vapi.nat64_add_del_interface(self.pg2.sw_if_index)
-
-        pkts = self.create_stream_in_ip6(self.pg2, self.pg1)
-        self.pg2.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=self.vrf1_nat_addr,
-                                dst_ip=self.pg1.remote_ip4)
-
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.vrf1_nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg2.remote_ip6)
-
-    def test_static(self):
-        """ NAT64 static translation test """
-        self.tcp_port_in = 60303
-        self.udp_port_in = 60304
-        self.icmp_id_in = 60305
-        self.tcp_port_out = 60303
-        self.udp_port_out = 60304
-        self.icmp_id_out = 60305
-
-        ses_num_start = self.nat64_get_ses_num()
-
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-
-        self.vapi.nat64_add_del_static_bib(self.pg0.remote_ip6n,
-                                           self.nat_addr_n,
-                                           self.tcp_port_in,
-                                           self.tcp_port_out,
-                                           IP_PROTOS.tcp)
-        self.vapi.nat64_add_del_static_bib(self.pg0.remote_ip6n,
-                                           self.nat_addr_n,
-                                           self.udp_port_in,
-                                           self.udp_port_out,
-                                           IP_PROTOS.udp)
-        self.vapi.nat64_add_del_static_bib(self.pg0.remote_ip6n,
-                                           self.nat_addr_n,
-                                           self.icmp_id_in,
-                                           self.icmp_id_out,
-                                           IP_PROTOS.icmp)
-
-        # in2out
-        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=self.nat_addr,
-                                dst_ip=self.pg1.remote_ip4, same_port=True)
-
-        # out2in
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        ip = IPv6(src=''.join(['64:ff9b::', self.pg1.remote_ip4]))
-        self.verify_capture_in_ip6(capture, ip[IPv6].src, self.pg0.remote_ip6)
-
-        ses_num_end = self.nat64_get_ses_num()
-
-        self.assertEqual(ses_num_end - ses_num_start, 3)
-
-    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
-    def test_session_timeout(self):
-        """ NAT64 session timeout """
-        self.icmp_id_in = 1234
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-        self.vapi.nat64_set_timeouts(icmp=5)
-
-        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-
-        ses_num_before_timeout = self.nat64_get_ses_num()
-
-        sleep(15)
-
-        # ICMP session after timeout
-        ses_num_after_timeout = self.nat64_get_ses_num()
-        self.assertNotEqual(ses_num_before_timeout, ses_num_after_timeout)
-
-    def test_icmp_error(self):
-        """ NAT64 ICMP Error message translation """
-        self.tcp_port_in = 6303
-        self.udp_port_in = 6304
-        self.icmp_id_in = 6305
-
-        ses_num_start = self.nat64_get_ses_num()
-
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-
-        # send some packets to create sessions
-        pkts = self.create_stream_in_ip6(self.pg0, self.pg1)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture_ip4 = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture_ip4,
-                                nat_ip=self.nat_addr,
-                                dst_ip=self.pg1.remote_ip4)
-
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture_ip6 = self.pg0.get_capture(len(pkts))
-        ip = IPv6(src=''.join(['64:ff9b::', self.pg1.remote_ip4]))
-        self.verify_capture_in_ip6(capture_ip6, ip[IPv6].src,
-                                   self.pg0.remote_ip6)
-
-        # in2out
-        pkts = [Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-                IPv6(src=self.pg0.remote_ip6, dst=ip[IPv6].src) /
-                ICMPv6DestUnreach(code=1) /
-                packet[IPv6] for packet in capture_ip6]
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IP].src, self.nat_addr)
-                self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
-                self.assertEqual(packet[ICMP].type, 3)
-                self.assertEqual(packet[ICMP].code, 13)
-                inner = packet[IPerror]
-                self.assertEqual(inner.src, self.pg1.remote_ip4)
-                self.assertEqual(inner.dst, self.nat_addr)
-                self.check_icmp_checksum(packet)
-                if inner.haslayer(TCPerror):
-                    self.assertEqual(inner[TCPerror].dport, self.tcp_port_out)
-                elif inner.haslayer(UDPerror):
-                    self.assertEqual(inner[UDPerror].dport, self.udp_port_out)
-                else:
-                    self.assertEqual(inner[ICMPerror].id, self.icmp_id_out)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-        # out2in
-        pkts = [Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-                IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
-                ICMP(type=3, code=13) /
-                packet[IP] for packet in capture_ip4]
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IPv6].src, ip.src)
-                self.assertEqual(packet[IPv6].dst, self.pg0.remote_ip6)
-                icmp = packet[ICMPv6DestUnreach]
-                self.assertEqual(icmp.code, 1)
-                inner = icmp[IPerror6]
-                self.assertEqual(inner.src, self.pg0.remote_ip6)
-                self.assertEqual(inner.dst, ip.src)
-                self.check_icmpv6_checksum(packet)
-                if inner.haslayer(TCPerror):
-                    self.assertEqual(inner[TCPerror].sport, self.tcp_port_in)
-                elif inner.haslayer(UDPerror):
-                    self.assertEqual(inner[UDPerror].sport, self.udp_port_in)
-                else:
-                    self.assertEqual(inner[ICMPv6EchoRequest].id,
-                                     self.icmp_id_in)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-    def test_hairpinning(self):
-        """ NAT64 hairpinning """
-
-        client = self.pg0.remote_hosts[0]
-        server = self.pg0.remote_hosts[1]
-        server_tcp_in_port = 22
-        server_tcp_out_port = 4022
-        server_udp_in_port = 23
-        server_udp_out_port = 4023
-        client_tcp_in_port = 1234
-        client_udp_in_port = 1235
-        client_tcp_out_port = 0
-        client_udp_out_port = 0
-        ip = IPv6(src=''.join(['64:ff9b::', self.nat_addr]))
-        nat_addr_ip6 = ip.src
-
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-
-        self.vapi.nat64_add_del_static_bib(server.ip6n,
-                                           self.nat_addr_n,
-                                           server_tcp_in_port,
-                                           server_tcp_out_port,
-                                           IP_PROTOS.tcp)
-        self.vapi.nat64_add_del_static_bib(server.ip6n,
-                                           self.nat_addr_n,
-                                           server_udp_in_port,
-                                           server_udp_out_port,
-                                           IP_PROTOS.udp)
-
-        # client to server
-        pkts = []
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=client.ip6, dst=nat_addr_ip6) /
-             TCP(sport=client_tcp_in_port, dport=server_tcp_out_port))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=client.ip6, dst=nat_addr_ip6) /
-             UDP(sport=client_udp_in_port, dport=server_udp_out_port))
-        pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IPv6].src, nat_addr_ip6)
-                self.assertEqual(packet[IPv6].dst, server.ip6)
-                if packet.haslayer(TCP):
-                    self.assertNotEqual(packet[TCP].sport, client_tcp_in_port)
-                    self.assertEqual(packet[TCP].dport, server_tcp_in_port)
-                    self.check_tcp_checksum(packet)
-                    client_tcp_out_port = packet[TCP].sport
-                else:
-                    self.assertNotEqual(packet[UDP].sport, client_udp_in_port)
-                    self.assertEqual(packet[UDP].dport, server_udp_in_port)
-                    self.check_udp_checksum(packet)
-                    client_udp_out_port = packet[UDP].sport
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-        # server to client
-        pkts = []
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=server.ip6, dst=nat_addr_ip6) /
-             TCP(sport=server_tcp_in_port, dport=client_tcp_out_port))
-        pkts.append(p)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=server.ip6, dst=nat_addr_ip6) /
-             UDP(sport=server_udp_in_port, dport=client_udp_out_port))
-        pkts.append(p)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IPv6].src, nat_addr_ip6)
-                self.assertEqual(packet[IPv6].dst, client.ip6)
-                if packet.haslayer(TCP):
-                    self.assertEqual(packet[TCP].sport, server_tcp_out_port)
-                    self.assertEqual(packet[TCP].dport, client_tcp_in_port)
-                    self.check_tcp_checksum(packet)
-                else:
-                    self.assertEqual(packet[UDP].sport, server_udp_out_port)
-                    self.assertEqual(packet[UDP].dport, client_udp_in_port)
-                    self.check_udp_checksum(packet)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-        # ICMP error
-        pkts = []
-        pkts = [Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-                IPv6(src=client.ip6, dst=nat_addr_ip6) /
-                ICMPv6DestUnreach(code=1) /
-                packet[IPv6] for packet in capture]
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        for packet in capture:
-            try:
-                self.assertEqual(packet[IPv6].src, nat_addr_ip6)
-                self.assertEqual(packet[IPv6].dst, server.ip6)
-                icmp = packet[ICMPv6DestUnreach]
-                self.assertEqual(icmp.code, 1)
-                inner = icmp[IPerror6]
-                self.assertEqual(inner.src, server.ip6)
-                self.assertEqual(inner.dst, nat_addr_ip6)
-                self.check_icmpv6_checksum(packet)
-                if inner.haslayer(TCPerror):
-                    self.assertEqual(inner[TCPerror].sport, server_tcp_in_port)
-                    self.assertEqual(inner[TCPerror].dport,
-                                     client_tcp_out_port)
-                else:
-                    self.assertEqual(inner[UDPerror].sport, server_udp_in_port)
-                    self.assertEqual(inner[UDPerror].dport,
-                                     client_udp_out_port)
-            except:
-                self.logger.error(ppp("Unexpected or invalid packet:", packet))
-                raise
-
-    def test_prefix(self):
-        """ NAT64 Network-Specific Prefix """
-
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-        self.vapi.nat64_add_del_pool_addr_range(self.vrf1_nat_addr_n,
-                                                self.vrf1_nat_addr_n,
-                                                vrf_id=self.vrf1_id)
-        self.vapi.nat64_add_del_interface(self.pg2.sw_if_index)
-
-        # Add global prefix
-        global_pref64 = "2001:db8::"
-        global_pref64_n = socket.inet_pton(socket.AF_INET6, global_pref64)
-        global_pref64_len = 32
-        self.vapi.nat64_add_del_prefix(global_pref64_n, global_pref64_len)
-
-        prefix = self.vapi.nat64_prefix_dump()
-        self.assertEqual(len(prefix), 1)
-        self.assertEqual(prefix[0].prefix, global_pref64_n)
-        self.assertEqual(prefix[0].prefix_len, global_pref64_len)
-        self.assertEqual(prefix[0].vrf_id, 0)
-
-        # Add tenant specific prefix
-        vrf1_pref64 = "2001:db8:122:300::"
-        vrf1_pref64_n = socket.inet_pton(socket.AF_INET6, vrf1_pref64)
-        vrf1_pref64_len = 56
-        self.vapi.nat64_add_del_prefix(vrf1_pref64_n,
-                                       vrf1_pref64_len,
-                                       vrf_id=self.vrf1_id)
-        prefix = self.vapi.nat64_prefix_dump()
-        self.assertEqual(len(prefix), 2)
-
-        # Global prefix
-        pkts = self.create_stream_in_ip6(self.pg0,
-                                         self.pg1,
-                                         pref=global_pref64,
-                                         plen=global_pref64_len)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=self.nat_addr,
-                                dst_ip=self.pg1.remote_ip4)
-
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(len(pkts))
-        dst_ip = self.compose_ip6(self.pg1.remote_ip4,
-                                  global_pref64,
-                                  global_pref64_len)
-        self.verify_capture_in_ip6(capture, dst_ip, self.pg0.remote_ip6)
-
-        # Tenant specific prefix
-        pkts = self.create_stream_in_ip6(self.pg2,
-                                         self.pg1,
-                                         pref=vrf1_pref64,
-                                         plen=vrf1_pref64_len)
-        self.pg2.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg1.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip=self.vrf1_nat_addr,
-                                dst_ip=self.pg1.remote_ip4)
-
-        pkts = self.create_stream_out(self.pg1, dst_ip=self.vrf1_nat_addr)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        dst_ip = self.compose_ip6(self.pg1.remote_ip4,
-                                  vrf1_pref64,
-                                  vrf1_pref64_len)
-        self.verify_capture_in_ip6(capture, dst_ip, self.pg2.remote_ip6)
-
-    def test_unknown_proto(self):
-        """ NAT64 translate packet with unknown protocol """
-
-        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
-                                                self.nat_addr_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-        remote_ip6 = self.compose_ip6(self.pg1.remote_ip4, '64:ff9b::', 96)
-
-        # in2out
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=self.pg0.remote_ip6, dst=remote_ip6) /
-             TCP(sport=self.tcp_port_in, dport=20))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg1.get_capture(1)
-
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=self.pg0.remote_ip6, dst=remote_ip6, nh=47) /
-             GRE() /
-             IP(src=self.pg2.local_ip4, dst=self.pg2.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg1.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IP].src, self.nat_addr)
-            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
-            self.assertTrue(packet.haslayer(GRE))
-            self.check_ip_checksum(packet)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-        # out2in
-        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
-             IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
-             GRE() /
-             IP(src=self.pg2.remote_ip4, dst=self.pg2.local_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg1.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IPv6].src, remote_ip6)
-            self.assertEqual(packet[IPv6].dst, self.pg0.remote_ip6)
-            self.assertEqual(packet[IPv6].nh, 47)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-    def test_hairpinning_unknown_proto(self):
-        """ NAT64 translate packet with unknown protocol - hairpinning """
-
-        client = self.pg0.remote_hosts[0]
-        server = self.pg0.remote_hosts[1]
-        server_tcp_in_port = 22
-        server_tcp_out_port = 4022
-        client_tcp_in_port = 1234
-        client_tcp_out_port = 1235
-        server_nat_ip = "10.0.0.100"
-        client_nat_ip = "10.0.0.110"
-        server_nat_ip_n = socket.inet_pton(socket.AF_INET, server_nat_ip)
-        client_nat_ip_n = socket.inet_pton(socket.AF_INET, client_nat_ip)
-        server_nat_ip6 = self.compose_ip6(server_nat_ip, '64:ff9b::', 96)
-        client_nat_ip6 = self.compose_ip6(client_nat_ip, '64:ff9b::', 96)
-
-        self.vapi.nat64_add_del_pool_addr_range(server_nat_ip_n,
-                                                client_nat_ip_n)
-        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
-        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
-
-        self.vapi.nat64_add_del_static_bib(server.ip6n,
-                                           server_nat_ip_n,
-                                           server_tcp_in_port,
-                                           server_tcp_out_port,
-                                           IP_PROTOS.tcp)
-
-        self.vapi.nat64_add_del_static_bib(server.ip6n,
-                                           server_nat_ip_n,
-                                           0,
-                                           0,
-                                           IP_PROTOS.gre)
-
-        self.vapi.nat64_add_del_static_bib(client.ip6n,
-                                           client_nat_ip_n,
-                                           client_tcp_in_port,
-                                           client_tcp_out_port,
-                                           IP_PROTOS.tcp)
-
-        # client to server
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=client.ip6, dst=server_nat_ip6) /
-             TCP(sport=client_tcp_in_port, dport=server_tcp_out_port))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=client.ip6, dst=server_nat_ip6, nh=IP_PROTOS.gre) /
-             GRE() /
-             IP(src=self.pg2.local_ip4, dst=self.pg2.remote_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IPv6].src, client_nat_ip6)
-            self.assertEqual(packet[IPv6].dst, server.ip6)
-            self.assertEqual(packet[IPv6].nh, IP_PROTOS.gre)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-        # server to client
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(src=server.ip6, dst=client_nat_ip6, nh=IP_PROTOS.gre) /
-             GRE() /
-             IP(src=self.pg2.remote_ip4, dst=self.pg2.local_ip4) /
-             TCP(sport=1234, dport=1234))
-        self.pg0.add_stream(p)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        p = self.pg0.get_capture(1)
-        packet = p[0]
-        try:
-            self.assertEqual(packet[IPv6].src, server_nat_ip6)
-            self.assertEqual(packet[IPv6].dst, client.ip6)
-            self.assertEqual(packet[IPv6].nh, IP_PROTOS.gre)
-        except:
-            self.logger.error(ppp("Unexpected or invalid packet:", packet))
-            raise
-
-    def nat64_get_ses_num(self):
-        """
-        Return number of active NAT64 sessions.
-        """
-        st = self.vapi.nat64_st_dump()
-        return len(st)
-
-    def clear_nat64(self):
-        """
-        Clear NAT64 configuration.
-        """
-        self.vapi.nat64_set_timeouts()
-
-        interfaces = self.vapi.nat64_interface_dump()
-        for intf in interfaces:
-            self.vapi.nat64_add_del_interface(intf.sw_if_index,
-                                              intf.is_inside,
-                                              is_add=0)
-
-        bib = self.vapi.nat64_bib_dump(IP_PROTOS.tcp)
-        for bibe in bib:
-            if bibe.is_static:
-                self.vapi.nat64_add_del_static_bib(bibe.i_addr,
-                                                   bibe.o_addr,
-                                                   bibe.i_port,
-                                                   bibe.o_port,
-                                                   bibe.proto,
-                                                   bibe.vrf_id,
-                                                   is_add=0)
-
-        bib = self.vapi.nat64_bib_dump(IP_PROTOS.udp)
-        for bibe in bib:
-            if bibe.is_static:
-                self.vapi.nat64_add_del_static_bib(bibe.i_addr,
-                                                   bibe.o_addr,
-                                                   bibe.i_port,
-                                                   bibe.o_port,
-                                                   bibe.proto,
-                                                   bibe.vrf_id,
-                                                   is_add=0)
-
-        bib = self.vapi.nat64_bib_dump(IP_PROTOS.icmp)
-        for bibe in bib:
-            if bibe.is_static:
-                self.vapi.nat64_add_del_static_bib(bibe.i_addr,
-                                                   bibe.o_addr,
-                                                   bibe.i_port,
-                                                   bibe.o_port,
-                                                   bibe.proto,
-                                                   bibe.vrf_id,
-                                                   is_add=0)
-
-        adresses = self.vapi.nat64_pool_addr_dump()
-        for addr in adresses:
-            self.vapi.nat64_add_del_pool_addr_range(addr.address,
-                                                    addr.address,
-                                                    vrf_id=addr.vrf_id,
-                                                    is_add=0)
-
-        prefixes = self.vapi.nat64_prefix_dump()
-        for prefix in prefixes:
-            self.vapi.nat64_add_del_prefix(prefix.prefix,
-                                           prefix.prefix_len,
-                                           vrf_id=prefix.vrf_id,
-                                           is_add=0)
-
-    def tearDown(self):
-        super(TestNAT64, self).tearDown()
-        if not self.vpp_dead:
-            self.logger.info(self.vapi.cli("show nat64 pool"))
-            self.logger.info(self.vapi.cli("show nat64 interfaces"))
-            self.logger.info(self.vapi.cli("show nat64 prefix"))
-            self.logger.info(self.vapi.cli("show nat64 bib all"))
-            self.logger.info(self.vapi.cli("show nat64 session table all"))
-            self.clear_nat64()
-
-if __name__ == '__main__':
-    unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 95de0be6..61db4d6b 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -1061,41 +1061,41 @@ class VppPapiProvider(object):
              'mt_next_hop_table_id': next_hop_table_id,
              'mt_next_hop_out_label_stack': next_hop_out_label_stack})
 
-    def snat_interface_add_del_feature(
+    def nat44_interface_add_del_feature(
             self,
             sw_if_index,
             is_inside=1,
             is_add=1):
-        """Enable/disable S-NAT feature on the interface
+        """Enable/disable NAT44 feature on the interface
 
         :param sw_if_index: Software index of the interface
         :param is_inside: 1 if inside, 0 if outside (Default value = 1)
         :param is_add: 1 if add, 0 if delete (Default value = 1)
         """
         return self.api(
-            self.papi.snat_interface_add_del_feature,
+            self.papi.nat44_interface_add_del_feature,
             {'is_add': is_add,
              'is_inside': is_inside,
              'sw_if_index': sw_if_index})
 
-    def snat_interface_add_del_output_feature(
+    def nat44_interface_add_del_output_feature(
             self,
             sw_if_index,
             is_inside=1,
             is_add=1):
-        """Enable/disable S-NAT output feature on the interface
+        """Enable/disable NAT44 output feature on the interface
 
         :param sw_if_index: Software index of the interface
         :param is_inside: 1 if inside, 0 if outside (Default value = 1)
         :param is_add: 1 if add, 0 if delete (Default value = 1)
         """
         return self.api(
-            self.papi.snat_interface_add_del_output_feature,
+            self.papi.nat44_interface_add_del_output_feature,
             {'is_add': is_add,
              'is_inside': is_inside,
              'sw_if_index': sw_if_index})
 
-    def snat_add_static_mapping(
+    def nat44_add_del_static_mapping(
             self,
             local_ip,
             external_ip=0,
@@ -1105,9 +1105,8 @@ class VppPapiProvider(object):
             addr_only=1,
             vrf_id=0,
             protocol=0,
-            is_add=1,
-            is_ip4=1):
-        """Add/delete S-NAT static mapping
+            is_add=1):
+        """Add/delete NAT44 static mapping
 
         :param local_ip: Local IP address
         :param external_ip: External IP address
@@ -1118,12 +1117,10 @@ class VppPapiProvider(object):
         :param vrf_id: VRF ID
         :param protocol: IP protocol (Default value = 0)
         :param is_add: 1 if add, 0 if delete (Default value = 1)
-        :param is_ip4: 1 if address type is IPv4 (Default value = 1)
         """
         return self.api(
-            self.papi.snat_add_static_mapping,
+            self.papi.nat44_add_del_static_mapping,
             {'is_add': is_add,
-             'is_ip4': is_ip4,
              'addr_only': addr_only,
              'local_ip_address': local_ip,
              'external_ip_address': external_ip,
@@ -1133,100 +1130,96 @@ class VppPapiProvider(object):
              'vrf_id': vrf_id,
              'protocol': protocol})
 
-    def snat_add_address_range(
+    def nat44_add_del_address_range(
             self,
             first_ip_address,
             last_ip_address,
             is_add=1,
-            is_ip4=1,
             vrf_id=0xFFFFFFFF):
-        """Add/del S-NAT address range
+        """Add/del NAT44 address range
 
         :param first_ip_address: First IP address
         :param last_ip_address: Last IP address
         :param vrf_id: VRF id for the address range
         :param is_add: 1 if add, 0 if delete (Default value = 1)
-        :param is_ip4: 1 if address type is IPv4 (Default value = 1)
         """
         return self.api(
-            self.papi.snat_add_address_range,
-            {'is_ip4': is_ip4,
-             'first_ip_address': first_ip_address,
+            self.papi.nat44_add_del_address_range,
+            {'first_ip_address': first_ip_address,
              'last_ip_address': last_ip_address,
              'vrf_id': vrf_id,
              'is_add': is_add})
 
-    def snat_address_dump(self):
-        """Dump S-NAT addresses
-        :return: Dictionary of S-NAT addresses
+    def nat44_address_dump(self):
+        """Dump NAT44 addresses
+        :return: Dictionary of NAT44 addresses
         """
-        return self.api(self.papi.snat_address_dump, {})
+        return self.api(self.papi.nat44_address_dump, {})
 
-    def snat_interface_dump(self):
-        """Dump interfaces with S-NAT feature
-        :return: Dictionary of interfaces with S-NAT feature
+    def nat44_interface_dump(self):
+        """Dump interfaces with NAT44 feature
+        :return: Dictionary of interfaces with NAT44 feature
         """
-        return self.api(self.papi.snat_interface_dump, {})
+        return self.api(self.papi.nat44_interface_dump, {})
 
-    def snat_interface_output_feature_dump(self):
-        """Dump interfaces with S-NAT output feature
-        :return: Dictionary of interfaces with S-NAT output feature
+    def nat44_interface_output_feature_dump(self):
+        """Dump interfaces with NAT44 output feature
+        :return: Dictionary of interfaces with NAT44 output feature
         """
-        return self.api(self.papi.snat_interface_output_feature_dump, {})
+        return self.api(self.papi.nat44_interface_output_feature_dump, {})
 
-    def snat_static_mapping_dump(self):
-        """Dump S-NAT static mappings
-        :return: Dictionary of S-NAT static mappings
+    def nat44_static_mapping_dump(self):
+        """Dump NAT44 static mappings
+        :return: Dictionary of NAT44 static mappings
         """
-        return self.api(self.papi.snat_static_mapping_dump, {})
+        return self.api(self.papi.nat44_static_mapping_dump, {})
 
-    def snat_show_config(self):
-        """Show S-NAT config
-        :return: S-NAT config parameters
+    def nat_show_config(self):
+        """Show NAT plugin config
+        :return: NAT plugin config parameters
         """
-        return self.api(self.papi.snat_show_config, {})
+        return self.api(self.papi.nat_show_config, {})
 
-    def snat_add_interface_addr(
+    def nat44_add_interface_addr(
             self,
             sw_if_index,
             is_add=1):
-        """Add/del S-NAT address from interface
+        """Add/del NAT44 address from interface
 
         :param sw_if_index: Software index of the interface
         :param is_add: 1 if add, 0 if delete (Default value = 1)
         """
-        return self.api(self.papi.snat_add_del_interface_addr,
+        return self.api(self.papi.nat44_add_del_interface_addr,
                         {'is_add': is_add, 'sw_if_index': sw_if_index})
 
-    def snat_interface_addr_dump(self):
-        """Dump S-NAT addresses interfaces
-        :return: Dictionary of S-NAT addresses interfaces
+    def nat44_interface_addr_dump(self):
+        """Dump NAT44 addresses interfaces
+        :return: Dictionary of NAT44 addresses interfaces
         """
-        return self.api(self.papi.snat_interface_addr_dump, {})
+        return self.api(self.papi.nat44_interface_addr_dump, {})
 
-    def snat_ipfix(
+    def nat_ipfix(
             self,
             domain_id=1,
             src_port=4739,
             enable=1):
-        """Enable/disable S-NAT IPFIX logging
+        """Enable/disable NAT IPFIX logging
 
         :param domain_id: Observation domain ID (Default value = 1)
         :param src_port: Source port number (Default value = 4739)
         :param enable: 1 if enable, 0 if disable (Default value = 1)
         """
         return self.api(
-            self.papi.snat_ipfix_enable_disable,
+            self.papi.nat_ipfix_enable_disable,
             {'domain_id': domain_id,
              'src_port': src_port,
              'enable': enable})
 
-    def snat_user_session_dump(
+    def nat44_user_session_dump(
             self,
             ip_address,
-            vrf_id,
-            is_ip4=1):
-        """Dump S-NAT user's sessions
+            vrf_id):
+        """Dump NAT44 user's sessions
 
         :param ip_address: ip adress of the user to be dumped
         :param cpu_index: cpu_index on which the user is
@@ -1234,26 +1227,25 @@ class VppPapiProvider(object):
         :return: Dictionary of S-NAT sessions
         """
         return self.api(
-            self.papi.snat_user_session_dump,
+            self.papi.nat44_user_session_dump,
             {'ip_address': ip_address,
-             'vrf_id': vrf_id,
-             'is_ip4': is_ip4})
+             'vrf_id': vrf_id})
 
-    def snat_user_dump(self):
-        """Dump S-NAT users
+    def nat44_user_dump(self):
+        """Dump NAT44 users
 
-        :return: Dictionary of S-NAT users
+        :return: Dictionary of NAT44 users
         """
-        return self.api(self.papi.snat_user_dump, {})
+        return self.api(self.papi.nat44_user_dump, {})
 
-    def snat_add_det_map(
+    def nat_det_add_del_map(
             self,
             in_addr,
             in_plen,
             out_addr,
             out_plen,
             is_add=1):
-        """Add/delete S-NAT deterministic mapping
+        """Add/delete deterministic NAT mapping
 
         :param is_add - 1 if add, 0 if delete
         :param in_addr - inside IP address
@@ -1262,14 +1254,15 @@ class VppPapiProvider(object):
         :param out_plen - outside IP address prefix length
         """
         return self.api(
-            self.papi.snat_add_det_map,
+            self.papi.nat_det_add_del_map,
             {'is_add': is_add,
+             'is_nat44': 1,
              'in_addr': in_addr,
              'in_plen': in_plen,
              'out_addr': out_addr,
              'out_plen': out_plen})
 
-    def snat_det_forward(
+    def nat_det_forward(
             self,
             in_addr):
         """Get outside address and port range from inside address
@@ -1277,10 +1270,11 @@ class VppPapiProvider(object):
         :param in_addr - inside IP address
         """
         return self.api(
-            self.papi.snat_det_forward,
-            {'in_addr': in_addr})
+            self.papi.nat_det_forward,
+            {'in_addr': in_addr,
+             'is_nat44': 1})
 
-    def snat_det_reverse(
+    def nat_det_reverse(
             self,
             out_addr,
             out_port):
@@ -1290,18 +1284,18 @@ class VppPapiProvider(object):
         :param out_port - outside port
         """
         return self.api(
-            self.papi.snat_det_reverse,
+            self.papi.nat_det_reverse,
             {'out_addr': out_addr,
              'out_port': out_port})
 
-    def snat_det_map_dump(self):
-        """Dump S-NAT deterministic mappings
+    def nat_det_map_dump(self):
+        """Dump deterministic NAT mappings
 
-        :return: Dictionary of S-NAT deterministic mappings
+        :return: Dictionary of deterministic NAT mappings
         """
-        return self.api(self.papi.snat_det_map_dump, {})
+        return self.api(self.papi.nat_det_map_dump, {})
 
-    def snat_det_set_timeouts(
+    def nat_det_set_timeouts(
             self,
             udp=300,
             tcp_established=7440,
@@ -1315,78 +1309,71 @@ class VppPapiProvider(object):
         :param icmp - ICMP timeout (Default value = 60)
         """
         return self.api(
-            self.papi.snat_det_set_timeouts,
+            self.papi.nat_det_set_timeouts,
             {'udp': udp,
              'tcp_established': tcp_established,
              'tcp_transitory': tcp_transitory,
              'icmp': icmp})
 
-    def snat_det_get_timeouts(self):
+    def nat_det_get_timeouts(self):
         """Get values of timeouts for deterministic NAT
 
         :return: Timeouts for deterministic NAT (in seconds)
         """
-        return self.api(self.papi.snat_det_get_timeouts, {})
+        return self.api(self.papi.nat_det_get_timeouts, {})
 
-    def snat_det_close_session_out(
+    def nat_det_close_session_out(
             self,
             out_addr,
             out_port,
             ext_addr,
-            ext_port,
-            is_ip4=1):
-        """Close CGN session using outside address and port
+            ext_port):
+        """Close deterministic NAT session using outside address and port
 
         :param out_addr - outside IP address
         :param out_port - outside port
         :param ext_addr - external host IP address
         :param ext_port - external host port
-        :param is_ip4: 1 if address type is IPv4 (Default value = 1)
         """
         return self.api(
-            self.papi.snat_det_close_session_out,
+            self.papi.nat_det_close_session_out,
             {'out_addr': out_addr,
              'out_port': out_port,
              'ext_addr': ext_addr,
-             'ext_port': ext_port,
-             'is_ip4': is_ip4})
+             'ext_port': ext_port})
 
-    def snat_det_close_session_in(
+    def nat_det_close_session_in(
             self,
             in_addr,
             in_port,
             ext_addr,
-            ext_port,
-            is_ip4=1):
-        """Close CGN session using inside address and port
+            ext_port):
+        """Close deterministic NAT session using inside address and port
 
         :param in_addr - inside IP address
         :param in_port - inside port
         :param ext_addr - external host IP address
         :param ext_port - external host port
-        :param is_ip4: 1 if address type is IPv4 (Default value = 1)
         """
         return self.api(
-            self.papi.snat_det_close_session_in,
+            self.papi.nat_det_close_session_in,
             {'in_addr': in_addr,
              'in_port': in_port,
              'ext_addr': ext_addr,
              'ext_port': ext_port,
-             'is_ip4': is_ip4})
+             'is_nat44': 1})
 
-    def snat_det_session_dump(
+    def nat_det_session_dump(
             self,
-            user_addr,
-            is_ip4=1):
-        """Dump S-NAT deterministic sessions belonging to a user
+            user_addr):
+        """Dump deterministic NAT sessions belonging to a user
 
         :param user_addr - inside IP address of the user
-        :param is_ip4: - 1 if address type is IPv4 (Default value = 1)
-        :return: Dictionary of S-NAT deterministic sessions
+        :return: Dictionary of deterministic NAT sessions
         """
         return self.api(
-            self.papi.snat_det_session_dump,
-            {'is_ip4': is_ip4,
+            self.papi.nat_det_session_dump,
+            {'is_nat44': 1,
              'user_addr': user_addr})
 
     def nat64_add_del_pool_addr_range(
-- 
cgit 1.2.3-korg